diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2018-04-16 20:02:00 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-04-16 20:02:00 +0200 |
commit | cc5c1775020c2eac14a071927f2e67618140c6d8 (patch) | |
tree | 1893e9a55adcaa24a3fb5e84e02c4fbfbafa39c5 | |
parent | 01f4b36399ee0df0b0fde58e418ea04fab397b9b (diff) | |
parent | 417d2f873652f9cae04adf252905414711f80065 (diff) |
Merge pull request #5590 from vespa-engine/balder/remove-unused-code
Balder/remove unused code
43 files changed, 11 insertions, 1953 deletions
diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index ae2e479e8bf..b03dd66c817 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -20,7 +20,6 @@ vespa_define_module( src/vespa/documentapi/messagebus src/vespa/documentapi/messagebus/messages src/vespa/documentapi/messagebus/policies - src/vespa/documentapi/messagebus/systemstate TEST_DEPENDS messagebus_messagebus-test @@ -34,5 +33,4 @@ vespa_define_module( src/tests/priority src/tests/replymerger src/tests/routablefactory - src/tests/systemstate ) diff --git a/documentapi/src/testlist.txt b/documentapi/src/testlist.txt index 851741bd24a..270cd72f524 100644 --- a/documentapi/src/testlist.txt +++ b/documentapi/src/testlist.txt @@ -4,6 +4,5 @@ tests/policies tests/policyfactory tests/priority tests/routablefactory -tests/systemstate tests/loadtypes tests/replymerger diff --git a/documentapi/src/tests/policies/testframe.cpp b/documentapi/src/tests/policies/testframe.cpp index 024a0c2fdaa..d99d99b9ec0 100644 --- a/documentapi/src/tests/policies/testframe.cpp +++ b/documentapi/src/tests/policies/testframe.cpp @@ -322,13 +322,6 @@ TestFrame::waitSlobrok(const string &pattern, uint32_t cnt) return false; } -SystemStateHandle -TestFrame::getSystemState() -{ - mbus::IProtocol * protocol = _mbus->getProtocol(DocumentProtocol::NAME); - return SystemStateHandle(static_cast<DocumentProtocol&>(*protocol).getSystemState()); -} - void TestFrame::handleReply(mbus::Reply::UP reply) { diff --git a/documentapi/src/tests/policies/testframe.h b/documentapi/src/tests/policies/testframe.h index 15ec61aff7b..cc5101905a1 100644 --- a/documentapi/src/tests/policies/testframe.h +++ b/documentapi/src/tests/policies/testframe.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/network/identity.h> #include <vespa/messagebus/network/inetwork.h> @@ -190,12 +189,6 @@ public: */ mbus::Receptor &getReceptor() { return _handler; } - /** - * Returns the system state from contained document protocol. - * - * @return Handle to the system state. - */ - documentapi::SystemStateHandle getSystemState(); void handleReply(mbus::Reply::UP reply) override; }; diff --git a/documentapi/src/tests/systemstate/.gitignore b/documentapi/src/tests/systemstate/.gitignore deleted file mode 100644 index 3f52bc38742..00000000000 --- a/documentapi/src/tests/systemstate/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -systemstate_test -documentapi_systemstate_test_app diff --git a/documentapi/src/tests/systemstate/CMakeLists.txt b/documentapi/src/tests/systemstate/CMakeLists.txt deleted file mode 100644 index b275f2b2fd0..00000000000 --- a/documentapi/src/tests/systemstate/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(documentapi_systemstate_test_app TEST - SOURCES - systemstate.cpp - DEPENDS - documentapi -) -vespa_add_test(NAME documentapi_systemstate_test_app COMMAND documentapi_systemstate_test_app) diff --git a/documentapi/src/tests/systemstate/DESC b/documentapi/src/tests/systemstate/DESC deleted file mode 100644 index 19dbc9195f1..00000000000 --- a/documentapi/src/tests/systemstate/DESC +++ /dev/null @@ -1,3 +0,0 @@ -This is a unit test for the system state parser and the corresponding NodeState class. It mirrors the -StateParserTestCase available in the java implementation of message bus. It consists of tests that verify that parsing -works, pathing works, encoding/decoding works, and finally that the NodeState class works as intended. diff --git a/documentapi/src/tests/systemstate/FILES b/documentapi/src/tests/systemstate/FILES deleted file mode 100644 index e1d0e026d31..00000000000 --- a/documentapi/src/tests/systemstate/FILES +++ /dev/null @@ -1 +0,0 @@ -systemstate.cpp diff --git a/documentapi/src/tests/systemstate/systemstate.cpp b/documentapi/src/tests/systemstate/systemstate.cpp deleted file mode 100644 index 3e42c94950a..00000000000 --- a/documentapi/src/tests/systemstate/systemstate.cpp +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/documentapi/messagebus/systemstate/systemstate.h> -#include <vespa/documentapi/messagebus/systemstate/nodestate.h> -#include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("systemstate_test"); - -using namespace documentapi; - -class Test : public vespalib::TestApp { -public: - int Main() override; - void testParser(); - void testPathing(); - void testState(); - void testEncoding(); - void testHandle(); - void testCompact(); - -private: - void assertParser(const string &state, const string &expected = ""); -}; - -TEST_APPHOOK(Test); - -int -Test::Main() -{ - TEST_INIT("systemstate_test"); - - testParser(); TEST_FLUSH(); - testPathing(); TEST_FLUSH(); - testState(); TEST_FLUSH(); - testEncoding(); TEST_FLUSH(); - testHandle(); TEST_FLUSH(); - testCompact(); TEST_FLUSH(); - - TEST_DONE(); - return 0; -} - -void -Test::testParser() -{ - assertParser("storage"); - assertParser("storage?", "ERROR"); - assertParser("storage?a", "ERROR"); - assertParser("storage?a=", "ERROR"); - assertParser("storage?a=1"); - assertParser("storage?a=1&", "ERROR"); - assertParser("storage?a=1&b", "ERROR"); - assertParser("storage?a=1&b=2"); - assertParser("storage?a=1&b=2 search"); - assertParser("storage?a=1&b=2 search?", "ERROR"); - assertParser("storage?a=1&b=2 search?a", "ERROR"); - assertParser("storage?a=1&b=2 search?a=", "ERROR"); - assertParser("storage?a=1&b=2 search?a=1"); - assertParser("storage?a=1&b=2 search?a=1&", "ERROR"); - assertParser("storage?a=1&b=2 search?a=1&b", "ERROR"); - assertParser("storage?a=1&b=2 search?a=1&b=", "ERROR"); - assertParser("storage?a=1&b=2 search?a=1&b=2"); - - assertParser("storage"); - assertParser("storage/"); - assertParser("storage/?", "ERROR"); - assertParser("storage/?a", "ERROR"); - assertParser("storage/?a=", "ERROR"); - assertParser("storage/?a=1"); - assertParser("storage/cluster.storage"); - assertParser("storage/cluster.storage/"); - - assertParser("storage?a=1"); - assertParser("storage/?a=1"); - assertParser("storage/.?a=1"); - assertParser("storage/./?a=1"); - assertParser("storage/./cluster.storage?a=1"); - assertParser("storage/./cluster.storage/?a=1"); - assertParser("storage/./cluster.storage/..?a=1"); - assertParser("storage/./cluster.storage/../?a=1"); - assertParser("storage/./cluster.storage/../storage?a=1"); - assertParser("storage/./cluster.storage/../storage/?a=1"); -} - -void -Test::testPathing() -{ - assertParser("storage?a=1", "storage?a=1"); - assertParser("storage/?a=1", "storage?a=1"); - assertParser("storage/.?a=1", "storage?a=1"); - assertParser("storage/./?a=1", "storage?a=1"); - assertParser("storage/./cluster.storage?a=1", "storage/cluster.storage?a=1"); - assertParser("storage/./cluster.storage/?a=1", "storage/cluster.storage?a=1"); - assertParser("storage/./cluster.storage/..?a=1", "storage?a=1"); - assertParser("storage/./cluster.storage/../?a=1", "storage?a=1"); - assertParser("storage/./cluster.storage/../storage?a=1", "storage/storage?a=1"); - assertParser("storage/./cluster.storage/../storage/?a=1", "storage/storage?a=1"); - - assertParser("a?p1=1 a/b?p2=2 a/b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - assertParser("a .?p1=1 ./b?p2=2 ./b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - assertParser("a .?p1=1 ./../a/b/ .?p2=2 c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - assertParser("a/./ .?p1=1 ../a/b/c/.. .?p2=2 ./c/../c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - assertParser("a/b/c/d/ ../../ ../ ../a .?p1=1 ./b?p2=2 ./ ../a/b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - - assertParser("a/b/c/d?p1=1 a?p2=2", "a?p2=2 a/b/c/d?p1=1"); - assertParser("a/b/c/d/?p1=1 /a?p2=2", "a?p2=2 a/b/c/d?p1=1"); - assertParser("/a/b/c/d/?p1=1 /a?p2=2", "a?p2=2 a/b/c/d?p1=1"); - - assertParser("a .?p1=1", "a?p1=1"); - assertParser("a/b .?p1=1", "a/b?p1=1"); - assertParser("a/b c?p1=1 d?p2=2", "a/b/c?p1=1 a/b/d?p2=2"); -} - -void -Test::testState() -{ - NodeState state; - state - .addChild("distributor", NodeState() - .setState("n", "27")) - .addChild("storage", NodeState() - .setState("n", "170") - .addChild("2", NodeState() - .setState("s", "d")) - .addChild("13", NodeState() - .setState("s", "r") - .setState("c", "0.0"))); - - EXPECT_EQUAL("27", state.getState("distributor/n")); - EXPECT_EQUAL("170", state.getState("storage/n")); - EXPECT_EQUAL("d", state.getState("storage/2/s")); - EXPECT_EQUAL("r", state.getState("storage/13/s")); - EXPECT_EQUAL("0.0", state.getState("storage/13/c")); - - EXPECT_EQUAL("27", state.getChild("distributor")->getState("n")); - EXPECT_EQUAL("170", state.getChild("storage")->getState("n")); - EXPECT_EQUAL("d", state.getChild("storage")->getChild("2")->getState("s")); - EXPECT_EQUAL("r", state.getChild("storage")->getChild("13")->getState("s")); - EXPECT_EQUAL("0.0", state.getChild("storage")->getChild("13")->getState("c")); -} - -void -Test::testEncoding() -{ - NodeState state; - state.setState("foo", "http://search.yahoo.com/?query=bar"); - LOG(info, "'%s'", state.toString().c_str()); - EXPECT_EQUAL(".?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); - assertParser(state.toString(), state.toString()); - - state = NodeState() - .addChild("foo:bar", NodeState() - .setState("foo", "http://search.yahoo.com/?query=bar")); - LOG(info, "'%s'", state.toString().c_str()); - EXPECT_EQUAL("foo%3Abar?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); - assertParser(state.toString(), state.toString()); - - state = NodeState() - .addChild("foo/bar", NodeState() - .setState("foo", "http://search.yahoo.com/?query=bar")); - LOG(info, "'%s'", state.toString().c_str()); - EXPECT_EQUAL("foo/bar?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); - assertParser(state.toString(), state.toString()); -} - -void -Test::testHandle() -{ - SystemState::UP state(SystemState::newInstance("")); - ASSERT_TRUE(state.get() != NULL); - - SystemStateHandle handle(*state); - ASSERT_TRUE(handle.isValid()); - - SystemStateHandle hoe(std::move(handle)); - ASSERT_TRUE(!handle.isValid()); - ASSERT_TRUE(hoe.isValid()); -} - -void -Test::testCompact() -{ - NodeState state; - state - .setState("a/b0/s", "d") - .setState("a/b0/c0/s", "d") - .setState("a/b0/c1/s", "d") - .setState("a/b1/s", "d") - .setState("a/b1/c0/s", "d") - .setState("a/b1/c1/s", "d"); - EXPECT_EQUAL("a/b0?s=d a/b0/c0?s=d a/b0/c1?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); - - state.removeChild("a/b0/c0"); - EXPECT_EQUAL("a/b0?s=d a/b0/c1?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); - - state.removeState("a/b0/c1/s"); - EXPECT_EQUAL("a/b0?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); - - state.setState("a/b1/c0/s", ""); - EXPECT_EQUAL("a/b0?s=d a/b1?s=d a/b1/c1?s=d", state.toString()); - - state.removeChild("a/b1"); - EXPECT_EQUAL("a/b0?s=d", state.toString()); - - state.removeChild("a"); - EXPECT_EQUAL("", state.toString()); -} - -void -Test::assertParser(const string &state, const string &expected) -{ - SystemState::UP obj = SystemState::newInstance(state); - if (obj.get() == NULL) { - EXPECT_EQUAL("ERROR", expected); - } - else { - SystemStateHandle handle(*obj); - LOG(info, "'%s' => '%s'", state.c_str(), handle.getRoot().toString().c_str()); - if (!expected.empty()) { - EXPECT_EQUAL(expected, handle.getRoot().toString()); - } - } -} - diff --git a/documentapi/src/vespa/documentapi/CMakeLists.txt b/documentapi/src/vespa/documentapi/CMakeLists.txt index 42998182f56..46bc0239834 100644 --- a/documentapi/src/vespa/documentapi/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_add_library(documentapi $<TARGET_OBJECTS:documentapi_documentapimessagebus> $<TARGET_OBJECTS:documentapi_documentapimessages> $<TARGET_OBJECTS:documentapi_documentapipolicies> - $<TARGET_OBJECTS:documentapi_documentapisystemstate> $<TARGET_OBJECTS:documentapi_documentapiloadtypes> INSTALL lib64 DEPENDS diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index d92fbdfb941..d2661d0fe6c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -28,7 +28,6 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, const string &configId) : _routingPolicyRepository(new RoutingPolicyRepository()), _routableRepository(new RoutableRepository(loadTypes)), - _systemState(SystemState::newInstance("")), _repo(repo) { // Prepare config string for routing policy factories. diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index 0a6c10ace77..32e6dc1d95e 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/documentapi/messagebus/systemstate/systemstate.h> #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/iprotocol.h> #include <vespa/messagebus/reply.h> #include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/documentapi/common.h> namespace vespalib { class VersionSpecification; @@ -28,7 +28,6 @@ class DocumentProtocol final : public mbus::IProtocol { private: std::unique_ptr<RoutingPolicyRepository> _routingPolicyRepository; std::unique_ptr<RoutableRepository> _routableRepository; - std::unique_ptr<SystemState> _systemState; std::shared_ptr<const document::DocumentTypeRepo> _repo; public: @@ -295,13 +294,6 @@ public: */ static bool hasOnlyErrorsOfType(const mbus::Reply &reply, uint32_t errCode); - /** - * Returns the curren state of the system, as observed by this protocol. This state object may be freely - * modified by the caller. - * - * @return The system state. - */ - SystemState &getSystemState() { return *_systemState; } const mbus::string &getName() const override { return NAME; } mbus::IRoutingPolicy::UP createPolicy(const mbus::string &name, const mbus::string ¶m) const override; mbus::Blob encode(const vespalib::Version &version, const mbus::Routable &routable) const override; diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h index 05202ad0de9..e2bf5119c58 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h @@ -2,7 +2,6 @@ #pragma once #include "iroutingpolicyfactory.h" -#include <vespa/documentapi/messagebus/systemstate/systemstate.h> namespace document { class DocumentTypeRepo; } diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore b/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore deleted file mode 100644 index 5dae353d999..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.depend -Makefile diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt deleted file mode 100644 index 45ca04e2cb3..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(documentapi_documentapisystemstate OBJECT - SOURCES - nodestate.cpp - systemstate.cpp - systemstatehandle.cpp - urlencoder.cpp - DEPENDS -) diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp deleted file mode 100644 index 42c573ef09e..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "nodestate.h" -#include "urlencoder.h" - -#include <vespa/log/log.h> -LOG_SETUP(".nodestate"); - -using namespace documentapi; - -NodeState::NodeState() : - _parent(NULL), - _id(""), - _children(), - _state() -{ - // empty -} - -NodeState::NodeState(const NodeState &rhs) : - _parent(rhs._parent), - _id(rhs._id), - _children(rhs._children), - _state(rhs._state) -{ - // empty -} - -NodeState::NodeState(StateMap args) : - _parent(NULL), - _id(""), - _children(), - _state(args) -{ - // empty -} - -NodeState::~NodeState() { } - -NodeState & -NodeState::addChild(const string &key, const NodeState &child) -{ - getChild(key, true)->copy(child); - return *this; -} - -NodeState * -NodeState::getChild(const string &key, bool force) -{ - if (key.empty()) { - return this; - } - - // Find first not-self location item. - size_t from = 0, to = key.find('/'); - while (to != string::npos && key.substr(from, to - from) == ".") { - from = to + 1; - to = key.find('/', from); - } - string arr0 = to != string::npos ? key.substr(from, to - from) : key.substr(from); - string arr1 = to != string::npos ? key.substr(to + 1) : ""; - - // Reference this or parent. - if (arr0 == ".") { - return this; - } - if (arr0 == "..") { - if (_parent == NULL) { - LOG(error, "Location string '%s' requests a parent above the top-most node, returning self to avoid crash.", - key.c_str()); - return this; - } - return _parent->getChild(arr1, force); - } - - // Look for child, forcing it if requested. - ChildMap::iterator it = _children.find(arr0); - if (it == _children.end()) { - if (!force) { - return NULL; - } - _children[arr0] = NodeState::SP(new NodeState()); - _children[arr0]->setParent(*this, arr0); - } - if (to != string::npos) { - return _children[arr0]->getChild(arr1, force); - } - return _children[arr0].get(); -} - -const NodeState::ChildMap & -NodeState::getChildren() const -{ - return _children; -} - -NodeState & -NodeState::removeChild(const string &key) -{ - if (key.empty()) { - return *this; - } - size_t pos = key.find_last_of('/'); - if (pos != string::npos) { - NodeState* parent = getChild(key.substr(0, pos), false); - if (parent != NULL) { - return parent->removeChild(key.substr(pos + 1)); - } - } - else { - _children.erase(key); - } - return compact(); -} - -const string -NodeState::getState(const string &key) -{ - if (key.empty()) { - return ""; - } - size_t pos = key.find_last_of('/'); - if (pos != string::npos) { - NodeState* parent = getChild(key.substr(0, pos), false); - return parent != NULL ? parent->getState(key.substr(pos + 1)) : ""; - } - StateMap::iterator it = _state.find(key); - return it != _state.end() ? it->second : ""; -} - -NodeState & -NodeState::setState(const string &key, const string &value) -{ - if (key.empty()) { - return *this; - } - size_t pos = key.find_last_of('/'); - if (pos != string::npos) { - getChild(key.substr(0, pos), true)->setState(key.substr(pos + 1), value); - } - else { - if (value.empty()) { - return removeState(key); - } - else { - _state[key] = value; - } - } - return *this; -} - -NodeState & -NodeState::removeState(const string &key) -{ - if (key.empty()) { - return *this; - } - size_t pos = key.find_last_of('/'); - if (pos != string::npos) { - NodeState* parent = getChild(key.substr(0, pos), false); - if (parent != NULL) { - return parent->removeState(key.substr(pos + 1)); - } - } - else { - _state.erase(key); - } - return compact(); -} - -NodeState & -NodeState::compact() -{ - if (_state.empty() && _children.empty()) { - if (_parent != NULL) { - return _parent->removeChild(_id); - } - } - return *this; -} - -NodeState & -NodeState::copy(const NodeState &node) -{ - for (StateMap::const_iterator it = node._state.begin(); - it != node._state.end(); ++it) { - _state[it->first] = it->second; - } - for (ChildMap::const_iterator it = node._children.begin(); - it != node._children.end(); ++it) { - getChild(it->first, true)->copy(*it->second); - } - return *this; -} - -NodeState & -NodeState::clear() -{ - _state.clear(); - _children.clear(); - return compact(); -} - -NodeState & -NodeState::setParent(NodeState &parent, const string &id) -{ - _parent = &parent; - _id = id; - return *this; -} - -const string -NodeState::toString() const -{ - const std::string ret = toString(""); - size_t pos = ret.find_last_not_of(' '); - return pos != string::npos ? ret.substr(0, pos + 1) : ret; -} - -const string -NodeState::toString(const string &prefix) const -{ - string ret; - if (!_state.empty()) { - string str; - for (StateMap::const_iterator it = _state.begin(); - it != _state.end(); ++it) { - str += it->first + "=" + URLEncoder::encode(it->second) + "&"; - } - ret += (prefix.empty() ? ".?" : prefix + "?") + str.substr(0, str.size() - 1) + " "; - } - string pre = prefix.empty() ? "" : (prefix + "/"); - for (ChildMap::const_iterator it = _children.begin(); - it != _children.end(); ++it) { - ret += it->second->toString(pre + URLEncoder::encode(it->first)); - } - return ret; -} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h deleted file mode 100644 index eff684f5bfe..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/documentapi/common.h> -#include <map> - -namespace documentapi { - -/** - * A node state is a single node in an annotatet tree of such nodes. It contains a reference to its parent - * node, a list of named child nodes, as well as a mapping of (key, value) pairs that constitute the annotated - * state of this node. To create an instance of a node state tree, one can either use the {@link SystemState} - * factory class, or one can programmatically construct each node and use the chaining capabilities of its - * set-methods to compact the necessary code. - */ -class NodeState { -public: - typedef std::unique_ptr<NodeState> UP; - typedef std::shared_ptr<NodeState> SP; - typedef std::map<string, string> StateMap; - typedef std::map<string, NodeState::SP> ChildMap; - -private: - NodeState* _parent; - string _id; - ChildMap _children; - StateMap _state; - - /** - * Compacts the system state tree from this node upwards. This will delete itself if it has a parent, but - * no internal state and no children. - * - * @return This or the first non-null ancestor, to allow chaining. - */ - NodeState &compact(); - - /** - * Returns a string representation of this node state. - * - * @param prefix The prefix to use for this string. - * @return A string representation of this. - */ - const string toString(const string &prefix) const; - -public: - NodeState(NodeState && rhs) = default; - NodeState & operator = (NodeState && rhs) = default; - NodeState & operator = (const NodeState & rhs) = default; - /** - * Creates a node state that no internal content. - */ - NodeState(); - - /** - * Creates a node state as a copy of another. - * - * @param rhs The state to copy. - */ - NodeState(const NodeState &rhs); - - /** - * Creates a node state based on a list of argument objects. These arguments are iterated and added to - * this node's internal state map. - * - * @param args The arguments to use as state. - */ - NodeState(StateMap args); - - ~NodeState(); - - /** - * Adds a child to this node at the given location. The key can be a location string, in which case the - * necessary intermediate node states are created. - * - * @param key The location at which to add the child. - * @param child The child node to add. - * @return This, to allow chaining. - */ - NodeState &addChild(const string &key, const NodeState &child); - - /** - * Returns the child at the given location relative to this. This method can be forced to return a child - * node even if it does not exist, by adding all intermediate nodes and the target node itself. - * - * @param key The location of the child to return. - * @param force Whether or not to force a return value by creating missing nodes. - * @return The child object, null if not found. - */ - NodeState *getChild(const string &key, bool force = false); - - /** - * Returns the map of child nodes for iteration. - * - * @return The internal child map. - */ - const ChildMap &getChildren() const; - - /** - * Removes the named child node from this node, and attempts to compact the system state from this node - * upwards by removing empty nodes. - * - * @param key The child to remove. - * @return The result of invoking {@link #compact} after the remove. - */ - NodeState &removeChild(const string &key); - - /** - * Retrieves some arbitrary state information for a given key. The key can be a location string, in which - * case the necessary intermediate nodes are traversed. If the key is not found, this method returns - * null. This method can not be const because it uses the non-const method {@link #getChild} to resolve a - * pathed key. - * - * @param key The name of the state information to return. - * @return The value of the state key. - */ - const string getState(const string &key); - - /** - * Sets some arbitrary state data in this node. The key can be a location string, in which case the - * necessary intermediate nodes are traversed and even created if missing. - * - * @param key The key to set. - * @param value The value to assign to the key. - * @return This, to allow chaining. - */ - NodeState &setState(const string &key, const string &value); - - /** - * Removes the named (key, value) state pair from this node, and attempts to compact the system state from - * this node upwards by removing empty nodes. - * - * @param key The state variable to clear. - * @return The result of invoking {@link #compact} after the remove. - */ - NodeState &removeState(const string &key); - - /** - * Copies the state content of another node state object into this. - * - * @param node The node state to copy into this. - * @return This, to allow chaining. - */ - NodeState ©(const NodeState &node); - - /** - * Clears both the internal state and child list, then compacts the tree from this node upwards. - * - * @return The result of invoking {@link #compact} after the remove. - */ - NodeState &clear(); - - /** - * Sets the parent of this node. - * - * @param parent The parent node. - * @param id The identifier of this node as seen in the parent. - * @return This, to allow chaining. - */ - NodeState &setParent(NodeState &parent, const string &id); - - /** - * Returns a string representation of this node state. - * - * @return A string representation of this. - */ - const string toString() const; -}; - -} - diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp deleted file mode 100644 index 0556e859c42..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "systemstate.h" -#include "nodestate.h" -#include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <boost/spirit/include/classic_core.hpp> -#include <boost/spirit/include/classic_parse_tree.hpp> -#include <boost/spirit/include/classic_tree_to_xml.hpp> -#include <boost/spirit/include/classic_chset.hpp> -#include <boost/spirit/include/classic_escape_char.hpp> -#include <boost/spirit/include/classic_grammar_def.hpp> - -#include <vespa/log/log.h> -LOG_SETUP(".systemstate"); - -using namespace documentapi; - -/** - * This class implements a boost::spirit type parser for the system state string. All contained names - * confirm to the boost::spirit naming convention, and care should therefore be taken if one wishes - * to modify any of these. Note that all content is inlined, just as all of boost::spirit, so this is - * therefore contained in the .cpp file instead of a separate .h file. - */ -struct SystemStateGrammar : public boost::spirit::classic::grammar<SystemStateGrammar> { - enum RuleId { - id_hexChar = 1, - id_hexCode, - id_alphaNum, - id_string, - id_argument, - id_argumentList, - id_locationItem, - id_location, - id_systemState - }; - - template <typename Scanner> - struct gram_base { - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_hexChar> > rule_hexChar; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_hexCode> > rule_hexCode; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_alphaNum> > rule_alphaNum; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_string> > rule_string; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_argument> > rule_argument; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_argumentList> > rule_argumentList; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_locationItem> > rule_locationItem; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_location> > rule_location; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_systemState> > rule_systemState; - typedef boost::spirit::classic::grammar_def<rule_systemState> type; - }; - - template <typename Scanner> - struct definition : gram_base<Scanner>::type { - typename gram_base<Scanner>::rule_hexChar _hexChar; - typename gram_base<Scanner>::rule_hexCode _hexCode; - typename gram_base<Scanner>::rule_alphaNum _alphaNum; - typename gram_base<Scanner>::rule_string _string; - typename gram_base<Scanner>::rule_argument _argument; - typename gram_base<Scanner>::rule_argumentList _argumentList; - typename gram_base<Scanner>::rule_locationItem _locationItem; - typename gram_base<Scanner>::rule_location _location; - typename gram_base<Scanner>::rule_systemState _systemState; - - definition(const SystemStateGrammar &) : - _hexChar(), - _hexCode(), - _alphaNum(), - _string(), - _argument(), - _argumentList(), - _locationItem(), - _location(), - _systemState() { - _hexChar = ( boost::spirit::classic::chset<>("A-Fa-f0-9") ); - _hexCode = ( boost::spirit::classic::ch_p('%') >> _hexChar >> _hexChar); - _alphaNum = ( boost::spirit::classic::chset<>("A-Za-z0-9") | - boost::spirit::classic::ch_p('-') | boost::spirit::classic::ch_p('.') | - boost::spirit::classic::ch_p('_') | boost::spirit::classic::ch_p('~') ); - _string = ( +( boost::spirit::classic::ch_p('+') | _hexCode | _alphaNum ) ); - _argument = ( _string >> boost::spirit::classic::ch_p('=') >> _string ); - _argumentList = ( _argument >> *( boost::spirit::classic::ch_p('&') >> _argument ) ); - _locationItem = ( boost::spirit::classic::str_p("..") | boost::spirit::classic::ch_p('.') | _string ); - _location = ( !boost::spirit::classic::ch_p('/') - >> _locationItem - >> *( boost::spirit::classic::ch_p('/') >> _locationItem ) - >> !boost::spirit::classic::ch_p('/') ); - _systemState = ( +( *boost::spirit::classic::space_p >> - _location >> !( boost::spirit::classic::ch_p('?') >> _argumentList ) ) ); - this->start_parsers(_systemState); - } - }; -}; - -template<typename T> void -debugNode(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, const string &prefix = "") -{ - std::map<boost::spirit::classic::parser_id, string> names; - names[boost::spirit::classic::parser_id(grammar.id_hexChar)] = "hexChar"; - names[boost::spirit::classic::parser_id(grammar.id_hexCode)] = "hexCode"; - names[boost::spirit::classic::parser_id(grammar.id_alphaNum)] = "alphaNum"; - names[boost::spirit::classic::parser_id(grammar.id_string)] = "string"; - names[boost::spirit::classic::parser_id(grammar.id_argument)] = "argument"; - names[boost::spirit::classic::parser_id(grammar.id_argumentList)] = "argumentList"; - names[boost::spirit::classic::parser_id(grammar.id_locationItem)] = "locationItem"; - names[boost::spirit::classic::parser_id(grammar.id_location)] = "location"; - names[boost::spirit::classic::parser_id(grammar.id_systemState)] = "systemState"; - - std::cout << prefix << names[node.value.id()] << ": " << string(node.value.begin(), node.value.end()) << std::endl; - for (size_t i = 0; i < node.children.size(); i++) { - debugNode(grammar, node.children[i], vespalib::make_string("%s %d.", prefix.c_str(), (int)i)); - } -} - -template<typename T> string -parseHexChar(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_hexChar); - assert(node.children.size() == 1); - assert(node.children[0].value.id().to_long() == grammar.id_hexChar); - (void) grammar; - return string(node.children[0].value.begin(), node.children[0].value.end()); -} - -template<typename T> char -parseHexCode(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_hexCode); - assert(node.children.size() == 3); - assert(node.children[0].value.id().to_long() == grammar.id_hexCode); - assert(node.children[1].value.id().to_long() == grammar.id_hexChar); - assert(node.children[2].value.id().to_long() == grammar.id_hexChar); - string enc = parseHexChar(grammar, node.children[1]) + parseHexChar(grammar, node.children[2]); - return (char)strtol(enc.c_str(), NULL, 16); -} - -template<typename T> string -parseAlphaNum(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_alphaNum); - assert(node.children.size() == 1); - assert(node.children[0].value.id().to_long() == grammar.id_alphaNum); - (void) grammar; - return string(node.children[0].value.begin(), node.children[0].value.end()); -} - -template<typename T> string -parseString(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_string); - string ret; - for (size_t i = 0; i < node.children.size(); ++i) { - boost::spirit::classic::tree_node<T> &child = node.children[i]; - if (child.value.id().to_long() == grammar.id_string) { - ret += " "; - } - else if (child.value.id().to_long() == grammar.id_alphaNum) { - ret += parseAlphaNum(grammar, child); - } - else if (child.value.id().to_long() == grammar.id_hexCode) { - ret += parseHexCode(grammar, child); - } - } - return ret; -} - -template<typename T> void -parseArgument(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, - std::map<string, string> &arg) -{ - assert(node.value.id().to_long() == grammar.id_argument); - assert(node.children.size() == 3); - assert(node.children[0].value.id().to_long() == grammar.id_string); - assert(node.children[1].value.id().to_long() == grammar.id_argument); - assert(node.children[2].value.id().to_long() == grammar.id_string); - string key = parseString(grammar, node.children[0]); - string val = parseString(grammar, node.children[2]); - arg[key] = val; -} - -template<typename T> void -parseArgumentList(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, - std::map<string, string> &arg) -{ - assert(node.value.id().to_long() == grammar.id_argumentList); - for (size_t i = 0; i < node.children.size(); ++i) { - boost::spirit::classic::tree_node<T> &child = node.children[i]; - if (child.value.id().to_long() == grammar.id_argument) { - parseArgument(grammar, child, arg); - } - } -} - -template<typename T> string -parseLocationItem(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_locationItem); - assert(node.children.size() == 1); - - string ret; - boost::spirit::classic::tree_node<T> &child = node.children[0]; - if (child.value.id().to_long() == grammar.id_locationItem) { - ret = string(child.value.begin(), child.value.end()); - } - else if (child.value.id().to_long() == grammar.id_string) { - ret = parseString(grammar, child); - } - return ret; -} - -template<typename T> string -parseLocation(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_location); - string ret; - for (size_t i = 0; i < node.children.size(); ++i) { - boost::spirit::classic::tree_node<T> &child = node.children[i]; - if (child.value.id().to_long() == grammar.id_locationItem) { - ret += parseLocationItem(grammar, child) + "/"; - } - } - return ret.substr(0, ret.size() - 1); -} - -template<typename T> NodeState::UP -parseSystemState(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_systemState); - NodeState::UP ret(new NodeState()); - string loc, pwd; - std::map<string, string> arg; - for (size_t i = 0; i < node.children.size(); ++i) { - boost::spirit::classic::tree_node<T> &child = node.children[i]; - if (child.value.id().to_long() == grammar.id_systemState) { - if (string(child.value.begin(), child.value.end()) != "?") { - if (!arg.empty()) { - ret->addChild(!loc.empty() ? loc : pwd, NodeState(arg)); - } - else { - pwd = loc; - } - loc.clear(); - arg.clear(); - } - } - else if (child.value.id().to_long() == grammar.id_location) { - if (!pwd.empty()) { - loc = pwd + "/"; - } - loc += parseLocation(grammar, child); - } - else if (child.value.id().to_long() == grammar.id_argumentList) { - parseArgumentList(grammar, child, arg); - } - } - if (!arg.empty()) { - ret->addChild(!loc.empty() ? loc : pwd, NodeState(arg)); - } - return ret; -} - -namespace { - vespalib::Lock _G_parseLock; -} - -SystemState::UP -SystemState::newInstance(const string &state) -{ - if (state.empty()) { - return SystemState::UP(new SystemState(NodeState::UP(new NodeState()))); - } - try { - vespalib::LockGuard guard(_G_parseLock); - SystemStateGrammar grammar; - boost::spirit::classic::tree_parse_info<> info = - boost::spirit::classic::pt_parse(static_cast<const char *>(&*state.begin()), - static_cast<const char *>(&*state.end()), - grammar.use_parser<0>()); - if (!info.full) { - string unexpected(info.stop); - unsigned int position = state.size() - unexpected.size(); - if (unexpected.size() > 10) { - unexpected = unexpected.substr(0, 10); - } - LOG(error, "Unexpected token at position %u ('%s') in query '%s'.", - position, unexpected.c_str(), state.c_str()); - } - else if (info.trees.size() != 1) { - LOG(error, "Parser returned %u trees, expected 1.", - (uint32_t)info.trees.size()); - } - else { - return SystemState::UP(new SystemState(parseSystemState(grammar, info.trees[0]))); - } - } - catch(std::exception& e) { - LOG(fatal, "SystemState::parse() internal error: %s", e.what()); - } - return SystemState::UP(); -} - -SystemState::SystemState(NodeState::UP root) : - _root(std::move(root)), - _lock(std::make_unique<vespalib::Lock>()) -{} - -SystemState::~SystemState() {} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h deleted file mode 100644 index 26f1b7fd713..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/documentapi/common.h> - -namespace vespalib { class Lock; } -namespace documentapi { - -class NodeState; - -/** - * This class is a factory to create a tree of {@link NodeState} objects from a parseable node state - * string. The naming of this class is intended to capture the fact that this annotated service tree actually - * contains the state of each service in the system. - */ -class SystemState { -private: - std::unique_ptr<NodeState> _root; - std::unique_ptr<vespalib::Lock> _lock; - - friend class SystemStateHandle; - - /** - * Constructs a new system state object to encapsulate a given root node state. This method is private; the only way - * to create a new instance is through the {@link #create} method. - * - * @param root The root node state. - */ - SystemState(std::unique_ptr<NodeState> root); - -public: - ~SystemState(); - SystemState(const SystemState &) = delete; - SystemState & operator = (const SystemState &) = delete; - /** - * Convenience typedefs. - */ - typedef std::unique_ptr<SystemState> UP; - - /** - * Creates a system state expression from a system state string. - * - * @param state The string to parse as a system state. - * @return The created node state tree. - * @throws RuntimeException Thrown if the string could not be parsed. - */ - static SystemState::UP newInstance(const string &state); -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp deleted file mode 100644 index 9ccaece4511..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "systemstatehandle.h" - -using namespace documentapi; - -SystemStateHandle::SystemStateHandle(SystemState &state) : - _state(&state), - _guard(*state._lock) -{} - -SystemStateHandle::SystemStateHandle(SystemStateHandle &&rhs) : - _state(rhs._state), - _guard(std::move(rhs._guard)) -{ - rhs._state = nullptr; -} - -SystemStateHandle & -SystemStateHandle::operator=(SystemStateHandle &&rhs) -{ - if (this != &rhs) { - _state = rhs._state; - _guard = std::move(rhs._guard); - rhs._state = nullptr; - } - return *this; -} - -SystemStateHandle::~SystemStateHandle() {} - diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h deleted file mode 100644 index f0342cfb2de..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "systemstate.h" -#include <vespa/vespalib/util/sync.h> - -namespace documentapi { - -/** - * Implements a handle to grant synchronized access to the content of a system state object. - */ -class SystemStateHandle { -private: - SystemState *_state; // The associated system state for which this object is a handler. - vespalib::LockGuard _guard; // The lock guard for the system state's lock. - - SystemStateHandle &operator=(const SystemStateHandle &) = delete; - SystemStateHandle(const SystemStateHandle &) = delete; - -public: - /** - * Creates a new system state handler object that grants access to the content of the supplied system - * state object. This handle is required to make sure that all access to the system state content is - * locked. - */ - SystemStateHandle(SystemState &state); - - /** - * Implements the move constructor. - * - * @param rhs The handle to move to this. - */ - SystemStateHandle(SystemStateHandle &&rhs); - - SystemStateHandle &operator=(SystemStateHandle &&rhs); - /** - * Destructor. Releases the contained lock on the associated system state object. There is no unlock() - * mechanism provided, since this will happen automatically as soon as this handle goes out of scope. - */ - ~SystemStateHandle(); - - /** Returns whether or not this handle is valid. */ - bool isValid() const { return _state != NULL; } - - /** Returns a reference to the root node of the associated system state. */ - NodeState &getRoot() { return *_state->_root; } - - /** Returns a const reference to the root node of the associated system state. */ - const NodeState &getRoot() const { return *_state->_root; } -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp deleted file mode 100644 index 13bd0750c31..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "urlencoder.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/stllike/asciistream.h> - -using namespace documentapi; - -const string -URLEncoder::encode(const string &str) -{ - vespalib::asciistream out; - for (size_t i = 0; i < str.size(); i++) { - char c = str[i]; - if ((c >= 48 && c <= 57) || // The range '0'-'9'. - (c >= 65 && c <= 90) || // The range 'A'-'Z'. - (c >= 97 && c <= 122) || // The range 'a'-'z'. - (c == '-' || c == '.' || c == '*' || c == '_')) { - out << c; - } - else if (c == ' ') { - out << '+'; - } - else { - out << "%" << vespalib::make_string("%02X", c & 0xff); - } - } - return out.str(); -} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h deleted file mode 100644 index 5b09327c0a0..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/documentapi/common.h> - -namespace documentapi { - -/** - * <p>Utility class for HTML form encoding. This class contains static methods for converting a String to the - * application/x-www-form-urlencoded MIME format. For more information about HTML form encoding, consult the - * HTML specification.</p> - * - * <p>When encoding a String, the following rules apply:</p> - * <ul> - * <li>The alphanumeric characters "a" through "z", "A" through "Z" and "0" through "9" remain the - * same.</li> - * <li>The special characters ".", "-", "*", and "_" remain the same.</li> - * <li>The space character " " is converted into a plus sign "+".</li> - * <li>All other characters are unsafe and are first converted into one or more bytes using some encoding - * scheme. Then each byte is represented by the 3-character string "%xy", where xy is the two-digit - * hexadecimal representation of the byte. The recommended encoding scheme to use is UTF-8. However, for - * compatibility reasons, if an encoding is not specified, then the default encoding of the platform is - * used.</li> - * </ul> - * - * <p>For example using UTF-8 as the encoding scheme the string "The string �@foo-bar" would get converted to - * "The+string+%C3%BC%40foo-bar" because in UTF-8 the character � is encoded as two bytes C3 (hex) and BC - * (hex), and the character @ is encoded as one byte 40 (hex).</p> - */ -class URLEncoder { -public: - /** - * Translates a string into application/x-www-form-urlencoded format using a UTF-8 encoding. - * - * @param str The string to be translated. - * @return The translated string. - */ - static const string encode(const string &str); -}; - -} - diff --git a/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt b/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt index df2022f2e70..f22e6c8b241 100644 --- a/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt +++ b/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt @@ -4,5 +4,3 @@ vespa_add_executable(searchcore_vespa-dump-feed_app vespa-dump-feed.cpp DEPENDS ) -vespa_add_target_system_dependency(searchcore_vespa-dump-feed_app boost boost_system${VESPA_BOOST_LIB_SUFFIX}) -vespa_add_target_system_dependency(searchcore_vespa-dump-feed_app boost boost_filesystem${VESPA_BOOST_LIB_SUFFIX}) diff --git a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp index dd9a0a732af..a6a76bf3e7b 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp +++ b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp @@ -35,6 +35,7 @@ ThreeLevelCountWriteBuffers(EC &sse, EC &spe, EC &pe) assert(_pe.getWriteOffset() == 0); } +ThreeLevelCountWriteBuffers::~ThreeLevelCountWriteBuffers() = default; void ThreeLevelCountWriteBuffers::flush() @@ -110,6 +111,6 @@ ThreeLevelCountReadBuffers::ThreeLevelCountReadBuffers(DC &ssd, DC &spd, DC &pd) pd.setReadContext(&_rcpd); } -ThreeLevelCountReadBuffers::~ThreeLevelCountReadBuffers() {} +ThreeLevelCountReadBuffers::~ThreeLevelCountReadBuffers() = default; } diff --git a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h index 40cf598ed3a..496b8e896cb 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h +++ b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h @@ -27,15 +27,12 @@ public: uint64_t _pFileBitSize; ThreeLevelCountWriteBuffers(EC &sse, EC &spe, EC &pe); + ~ThreeLevelCountWriteBuffers(); - void - flush(); + void flush(); // unit test method. Just pads without writing proper header - void - startPad(uint32_t ssHeaderLen, - uint32_t spHeaderLen, - uint32_t pHeaderLen); + void startPad(uint32_t ssHeaderLen, uint32_t spHeaderLen, uint32_t pHeaderLen); }; diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt index 5aa4e18cb84..13c9863aa8e 100644 --- a/storage/src/tests/bucketdb/CMakeLists.txt +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(storage_testbucketdb TEST SOURCES bucketinfotest.cpp bucketmanagertest.cpp - distribution_hash_normalizer_test.cpp initializertest.cpp judyarraytest.cpp judymultimaptest.cpp diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index 0c7122547db..d42330086f0 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -82,7 +82,6 @@ public: CPPUNIT_TEST(testConflictSetOnlyClearedAfterAllBucketRequestsDone); CPPUNIT_TEST(testRejectRequestWithMismatchingDistributionHash); CPPUNIT_TEST(testDbNotIteratedWhenAllRequestsRejected); - CPPUNIT_TEST(testReceivedDistributionHashIsNormalized); // FIXME(vekterli): test is not deterministic and enjoys failing // sporadically when running under Valgrind. See bug 5932891. @@ -1311,28 +1310,4 @@ BucketManagerTest::testDbNotIteratedWhenAllRequestsRejected() auto replies = fixture.awaitAndGetReplies(1); } -/** - * Accept bucket info requests if their distribution hash is a valid permutation - * of our own config (i.e. they are set-wise identical even though the - * ordering of nodes may differ). See VESPA-1980 for context. - */ -void -BucketManagerTest::testReceivedDistributionHashIsNormalized() -{ - ConcurrentOperationFixture fixture(*this); - document::BucketId bucket(17, 0); - fixture.setUp(WithBuckets().add(bucket, api::BucketInfo(50, 100, 200))); - - // Test is configured with 10 nodes in increasing order. Jumble the order - // around. - auto infoCmd = fixture.createFullFetchCommandWithHash( - "(0;2;1;3;9;6;4;5;8;7;0)"); - _top->sendDown(infoCmd); - auto replies = fixture.awaitAndGetReplies(1); - auto& reply = dynamic_cast<api::RequestBucketInfoReply&>(*replies[0]); - // Should NOT have been rejected despite hash not matching config order - // verbatim. - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, reply.getResult().getResult()); -} - } // storage diff --git a/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp b/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp deleted file mode 100644 index da7db8c6f4c..00000000000 --- a/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/storage/bucketdb/distribution_hash_normalizer.h> -#include <string> - -namespace storage { - -using Normalizer = DistributionHashNormalizer; - -class DistributionHashNormalizerTest : public CppUnit::TestFixture { -public: - CPPUNIT_TEST_SUITE(DistributionHashNormalizerTest); - CPPUNIT_TEST(orderNonHierarchicRootGroupNodesByDistributionKey); - CPPUNIT_TEST(mayHaveSameGroupIndexAsNodeIndex); - CPPUNIT_TEST(emitOptionalCapacityForRootGroup); - CPPUNIT_TEST(emitOptionalCapacityForSubGroups); - CPPUNIT_TEST(hierarchicGroupsAreOrderedByGroupIndex); - CPPUNIT_TEST(subgroupsOrderedOnEachNestingLevel); - CPPUNIT_TEST(distributionSpecIsCopiedVerbatim); - CPPUNIT_TEST(emptyInputYieldsEmptyOutput); - CPPUNIT_TEST(parseFailureReturnsInputVerbatim); - CPPUNIT_TEST_SUITE_END(); - - void orderNonHierarchicRootGroupNodesByDistributionKey(); - void mayHaveSameGroupIndexAsNodeIndex(); - void emitOptionalCapacityForRootGroup(); - void emitOptionalCapacityForSubGroups(); - void hierarchicGroupsAreOrderedByGroupIndex(); - void subgroupsOrderedOnEachNestingLevel(); - void distributionSpecIsCopiedVerbatim(); - void emptyInputYieldsEmptyOutput(); - void parseFailureReturnsInputVerbatim(); - -private: - DistributionHashNormalizer _normalizer; -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(DistributionHashNormalizerTest); - -void -DistributionHashNormalizerTest::orderNonHierarchicRootGroupNodesByDistributionKey() -{ - // Group index is first in list. - CPPUNIT_ASSERT_EQUAL(vespalib::string("(1;0;2;3;4;7)"), - _normalizer.normalize("(1;4;7;2;0;3)")); -} - -void -DistributionHashNormalizerTest::mayHaveSameGroupIndexAsNodeIndex() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0;0;2;3;4;7)"), - _normalizer.normalize("(0;4;7;2;0;3)")); -} - -void -DistributionHashNormalizerTest::emitOptionalCapacityForRootGroup() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0c12.5;1;2;3;4;7)"), - _normalizer.normalize("(0c12.5;1;4;7;2;3)")); -} - -void -DistributionHashNormalizerTest::emitOptionalCapacityForSubGroups() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(1c5.5;1)(2;2)(3c7;3))"), - _normalizer.normalize("(0d1|*(2;2)(1c5.5;1)(3c7;3))")); -} - -void -DistributionHashNormalizerTest::hierarchicGroupsAreOrderedByGroupIndex() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(0;0)(1;1)(3;3))"), - _normalizer.normalize("(0d1|*(3;3)(1;1)(0;0))")); -} - -void -DistributionHashNormalizerTest::subgroupsOrderedOnEachNestingLevel() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(1d3|*(2;2)(3;3))" - "(4;1)(7d2|*(5;5)(6;6)))"), - _normalizer.normalize("(0d1|*(7d2|*(6;6)(5;5))" - "(1d3|*(2;2)(3;3))(4;1))")); -} - -void -DistributionHashNormalizerTest::distributionSpecIsCopiedVerbatim() -{ - // Definitely don't want to do any ordering of the distribution spec. - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d3|2|1|*(0;0)(1;1)(3;3))"), - _normalizer.normalize("(0d3|2|1|*(3;3)(1;1)(0;0))")); -} - -void -DistributionHashNormalizerTest::emptyInputYieldsEmptyOutput() -{ - // Technically a parse failure (only 4.2 has this behavior), but it's - // explicitly checked for in BucketManager, so let's test it explicitly - // here as well. - CPPUNIT_ASSERT_EQUAL(vespalib::string(""), _normalizer.normalize("")); -} - -// In the (unlikely) case that the parser somehow fails to capture all possible -// valid values of the distribution hash, fall back to returning the non- -// normalized string. A log warning will also be emitted (though that's not -// testable). -void -DistributionHashNormalizerTest::parseFailureReturnsInputVerbatim() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("onkel skrue"), - _normalizer.normalize("onkel skrue")); -} - -} // storage - diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 28fccf438ff..a9b28bd2542 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <boost/assign/std/vector.hpp> // for 'operator+=()' #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storageapi/message/persistence.h> diff --git a/storage/src/vespa/storage/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/bucketdb/CMakeLists.txt index 1b59ea2290b..8200884de17 100644 --- a/storage/src/vespa/storage/bucketdb/CMakeLists.txt +++ b/storage/src/vespa/storage/bucketdb/CMakeLists.txt @@ -6,7 +6,6 @@ vespa_add_library(storage_bucketdb OBJECT bucketinfo.cpp bucketmanager.cpp bucketmanagermetrics.cpp - distribution_hash_normalizer.cpp judyarray.cpp lockablemap.cpp mapbucketdatabase.cpp diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 733cc490bd3..551f6fc726d 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bucketmanager.h" -#include "distribution_hash_normalizer.h" #include "minimumusedbitstracker.h" #include "lockablemap.hpp" #include <iomanip> @@ -522,10 +521,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac lib::ClusterState::CSP clusterState(clusterStateBundle->getDerivedClusterState(bucketSpace)); assert(clusterState.get()); - DistributionHashNormalizer normalizer; - - const auto our_hash = normalizer.normalize( - distribution->getNodeGraph().getDistributionConfigHash()); + const auto our_hash = distribution->getNodeGraph().getDistributionConfigHash(); LOG(debug, "Processing %" PRIu64 " queued request bucket info commands. " "Using cluster state '%s' and distribution hash '%s'", @@ -537,8 +533,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac for (auto it = reqs.rbegin(); it != reqs.rend(); ++it) { // Currently small requests should not be forwarded to worker thread assert((*it)->hasSystemState()); - const auto their_hash = normalizer.normalize( - (*it)->getDistributionHash()); + const auto their_hash = (*it)->getDistributionHash(); std::ostringstream error; if ((*it)->getSystemState().getVersion() > _lastClusterStateSeen) { @@ -570,8 +565,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac // If we get here, message should be failed auto reply = std::make_shared<api::RequestBucketInfoReply>(**it); - reply->setResult(api::ReturnCode( - api::ReturnCode::REJECTED, error.str())); + reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, error.str())); LOG(debug, "Rejecting request from distributor %u: %s", (*it)->getDistributor(), error.str().c_str()); diff --git a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp deleted file mode 100644 index 1e6825555b2..00000000000 --- a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "distribution_hash_normalizer.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <boost/spirit/include/qi.hpp> -#include <boost/spirit/include/phoenix_core.hpp> -#include <boost/spirit/include/phoenix_object.hpp> -#include <boost/fusion/include/adapt_struct.hpp> -#include <boost/optional.hpp> -#include <boost/variant/recursive_wrapper.hpp> -#include <vector> -#include <algorithm> -#include <iterator> -#include <functional> - -#include <vespa/log/bufferedlogger.h> -LOG_SETUP(".storage.bucketdb.distribution_hash_normalizer"); - -// TODO -// This code can be removed once we have a model out which ensures consistent -// ordering of nodes in the stor-distribution config. - -namespace qi = boost::spirit::qi; -namespace ascii = boost::spirit::ascii; -namespace phoenix = boost::phoenix; - -namespace { - -struct GroupSet; - -using Children = boost::variant< - std::vector<unsigned int>, - boost::recursive_wrapper<GroupSet> ->; - -struct Group { - uint16_t index; - boost::optional<double> capacity; - Children children; - ~Group() {} -}; - -struct GroupSet { - std::string distribution_spec; - std::vector<Group> subgroups; -}; - -} // anon ns - -// Fusion adaptations must be in global scope. -BOOST_FUSION_ADAPT_STRUCT( - ::Group, - (uint16_t, index) - (boost::optional<double>, capacity) - (Children, children) -) - -BOOST_FUSION_ADAPT_STRUCT( - ::GroupSet, - (std::string, distribution_spec) - (std::vector<Group>, subgroups) -) - -namespace storage { -namespace { - -// Boost.Spirit v2 grammar for parsing the output of lib::Group::getConfigHash. -template <typename Iterator> -struct HashGrammar - : qi::grammar<Iterator, Group()> -{ - HashGrammar() - : HashGrammar::base_type(group) - { - using qi::uint_; - using qi::double_; - using ascii::char_; - /* - * This grammar makes the (reasonable) assumption that you can't have - * empty groups. - * - * Quick Spirit PEG DSL syntax primer for any two arbitrary parsers - * a and b (all subcomponents of parsers are themselves parsers): - * - * 'X' : character literal match parser - * a >> b : a must be followed by b ("a b" in EBNF) - * -a : optional ("a?" in EBNF) - * a | b : a or b must match (same as in EBNF) - * +a : match 1 or more times ("a+" in EBNF) - * *a : kleene star; 0 or more times ("a*" in EBNF) - * a - b : difference; a but not b - * - * Please see Boost.Spirit docs on how these map to parser attributes - * (optional maps to boost::optional of nested attribute, + or kleene - * star maps to an iterable range (std::vector) of nested attributes, - * a | b maps to a boost::variant of the attributes of a and b, - * a >> b maps to a boost::tuple of the attributes and so on; usually - * fairly intuitive). - */ - group = - '(' - >> uint_ - >> -('c' >> double_) - >> ( +(';' >> uint_) - | subgroups - ) - >> ')'; - - subgroups = ('d' >> distr_spec >> +group); - - distr_spec = +(char_ - '('); // Match everything until open paren. - } - - qi::rule<Iterator, Group()> group; - qi::rule<Iterator, GroupSet()> subgroups; - qi::rule<Iterator, std::string()> distr_spec; -}; - -template <typename Range, typename Predicate> -auto ordered_by(const Range& range, Predicate pred) { - std::vector<typename Range::value_type> copy( - std::begin(range), std::end(range)); - std::sort(copy.begin(), copy.end(), pred); - return copy; -} - -void emit_normalized_groups(vespalib::asciistream& out, const Group& g); - -struct InOrderGroupVisitor : boost::static_visitor<void> { - vespalib::asciistream& _out; - InOrderGroupVisitor(vespalib::asciistream& out) - : _out(out) - { - } - - void operator()(const std::vector<unsigned int>& nodes) const { - for (uint16_t node : ordered_by(nodes, std::less<void>())) { - _out << ';' << node; - } - } - - void operator()(const GroupSet& gs) const { - _out << 'd' << gs.distribution_spec; - auto index_less_than = [](auto& lhs, auto& rhs) { - return lhs.index < rhs.index; - }; - // Ordering will also copy nested subgroups, but the number of known - // Vespa installations with nested subgroups is currently somewhere - // around the high end of zero. - for (auto& g : ordered_by(gs.subgroups, index_less_than)) { - emit_normalized_groups(_out, g); - } - } -}; - -void emit_normalized_groups(vespalib::asciistream& out, const Group& g) { - out << '(' << g.index; - if (g.capacity) { - out << 'c' << *g.capacity; - } - boost::apply_visitor(InOrderGroupVisitor(out), g.children); - out << ')'; -} - -} // anon ns - -// We keep the grammar around across multiple normalized() calls because -// constructing the grammar object(s) isn't free. -struct DistributionHashNormalizer::ParserImpl { - using Iterator = vespalib::string::const_iterator; - HashGrammar<Iterator> grammar; -}; - -DistributionHashNormalizer::DistributionHashNormalizer() - : _impl(std::make_unique<ParserImpl>()) -{ -} - -// Required here because of incomplete ParserImpl in header. -DistributionHashNormalizer::~DistributionHashNormalizer() -{ -} - -vespalib::string -DistributionHashNormalizer::normalize(vespalib::stringref hash) const -{ - Group root; - - auto iter = hash.begin(); - const bool ok = qi::parse(iter, hash.end(), _impl->grammar, root); - if (!ok || iter != hash.end()) { - vespalib::string hash_str = hash; // stringref might not be zero-term'd. - LOGBT(warning, hash_str.c_str(), - "Unable to parse compact distribution config " - "representation: '%s'", - hash_str.c_str()); - return hash; // Fallback to input on parse failure. - } - - vespalib::asciistream out; - emit_normalized_groups(out, root); - - return out.str(); -} - -} // storage - diff --git a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.h b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.h deleted file mode 100644 index a3a79542265..00000000000 --- a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/string.h> - -namespace storage { - -/** - * Utility class for "normalizing" a received distribution hash string into - * a representation that is ordering invariant across group and node indices. - * - * All group indices and node indices will be returned in increasing order. - * - * In the case of a parser error the original string will be returned verbatim. - */ -class DistributionHashNormalizer { - // PIMPL the parser to avoid Spirit deps in header file. - struct ParserImpl; - std::unique_ptr<ParserImpl> _impl; -public: - DistributionHashNormalizer(); - ~DistributionHashNormalizer(); - - vespalib::string normalize(vespalib::stringref hash) const; -}; - -} // storage - diff --git a/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h b/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h index 888b6d248bd..41878f09014 100644 --- a/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h +++ b/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "prioritizedbucket.h" #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <vespa/storage/distributor/maintenance/prioritizedbucket.h> #include <boost/iterator/iterator_facade.hpp> -namespace storage { -namespace distributor { +namespace storage::distributor { class BucketPriorityDatabase { @@ -69,7 +68,4 @@ public: }; } -} - - diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 8a4e5d5b336..ed61d730629 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -17,7 +17,6 @@ vespa_define_module( APPS src/vespa/vespaclient/spoolmaster src/vespa/vespaclient/vdsstates - src/vespa/vespaclient/vespadoclocator src/vespa/vespaclient/vesparoute ) diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore b/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore deleted file mode 100644 index 52e1033ec5b..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -vespa-doclocator -vespa-doclocator-bin diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt deleted file mode 100644 index 69bffc28092..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespaclient_vespadoclocator_app - SOURCES - application.cpp - locator.cpp - main.cpp - OUTPUT_NAME vespa-doclocator-bin - INSTALL bin - DEPENDS -) -vespa_add_target_system_dependency(vespaclient_vespadoclocator_app boost boost_program_options${VESPA_BOOST_LIB_SUFFIX}) diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp deleted file mode 100644 index 1d7d84e2b01..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "application.h" -#include <boost/program_options.hpp> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/config/common/exceptions.h> -#include <vespa/document/base/idstringexception.h> -#include <iostream> - -#include <vespa/log/log.h> -LOG_SETUP("vespadoclocator"); - - -bool -Application::printDocumentLocation(Locator &locator, const std::string &docIdStr) -{ - try { - document::DocumentId docId(docIdStr); - std::cout << "DocumentId(" << docIdStr << ") " - << "BucketId(" << locator.getBucketId(docId).getId() << ") " - << "SearchColumn(" << locator.getSearchColumn(docId) << ")" - << std::endl; - } catch (document::IdParseException &e) { - std::cerr << e.getMessage() << std::endl; - return false; - } catch (vespalib::IllegalArgumentException &e) { - std::cerr << e.getMessage() << std::endl; - return false; - } - return true; -} - -int -Application::Main() -{ - // Configure locator object. - using namespace boost::program_options; - - uint32_t numColumns = 0; - std::string configId; - std::string clusterName; - std::vector<std::string> docIds; - - options_description desc("This is a tool for resolving the target column number of a document." - "\n\n" - "The options are"); - desc.add_options() - ( "config-id,i", - value<std::string>(&configId)->default_value("client"), - "The identifier to use when subscribing to configuration." ) - - ( "cluster-name,c", - value<std::string>(&clusterName), - "The name of the search cluster in which to resolve document location." ) - - ( "document-id,d", - value< std::vector<std::string> >(&docIds), - "The identifiers of the documents to locate. " - "These can also be passed as arguments without the option prefix. " - "If none is given, this tool parses identifiers from standard in." ) - - ( "help,h", - "Shows this help page." ) - - ( "num-columns,n", - value<uint32_t>(&numColumns), - "The number of columns in the search cluster. By providing this, no configuration " - "is required, meaning you can run this tool outside of a vespa cluster." ); - - positional_options_description pos; - pos.add("document-id", -1); - - variables_map vm; - try { - store(command_line_parser(_argc, _argv).options(desc).positional(pos).run(), vm); - notify(vm); - } catch (unknown_option &e) { - std::cout << e.what() << std::endl; - return EXIT_FAILURE; - } - - if (vm.count("help") != 0) { - std::cout << desc << std::endl; - return EXIT_SUCCESS; - } - - Locator locator(numColumns); - if (vm.count("num-columns") == 0) { - try { - locator.configure(configId, clusterName); - } catch (config::InvalidConfigException &e) { - std::cerr << e.getMessage() << std::endl; - return EXIT_FAILURE; - } - } - - // Locate the documents provided. - if (docIds.empty()) { - char buf[4096]; - while (!std::cin.getline(buf, 4096).eof()) { - std::string in(buf); - if (!printDocumentLocation(locator, in)) { - return EXIT_FAILURE; - } - } - } else { - for (std::vector<std::string>::iterator it = docIds.begin(); - it != docIds.end(); ++it) - { - if (!printDocumentLocation(locator, *it)) { - return EXIT_FAILURE; - } - } - } - return EXIT_SUCCESS; -} diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h deleted file mode 100644 index 58104ec407c..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "locator.h" -#include <vespa/fastos/app.h> - -class Application : public FastOS_Application { -private: - /** - * Locates and outputs the whereabouts of the given document id. If there is a problem parsing the given - * document identifier, this method returns false. - * - * @param locator The locator to use. - * @param docId The document to locate. - * @return True if the document was located. - */ - bool printDocumentLocation(Locator &locator, const std::string &docId); - -public: - int Main() override; -}; - diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp deleted file mode 100644 index 638c83ca456..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "locator.h" -#include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/messagebus/configagent.h> -#include <vespa/messagebus/iconfighandler.h> -#include <vespa/messagebus/routing/routingspec.h> -#include <vespa/vdslib/bucketdistribution.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/config/helper/configgetter.hpp> -#include <vespa/config/common/exceptions.h> -#include <vespa/config/subscription/configuri.h> -#include <boost/tokenizer.hpp> - -typedef std::map<std::string, uint32_t> ClusterMap; -using namespace config; - -namespace { - - void - processHop(const mbus::HopSpec &hop, ClusterMap &clusters) - { - typedef boost::char_separator<char> CharSeparator; - typedef boost::tokenizer<CharSeparator> Tokenizer; - - int colIdx = -1; - for (uint32_t r = 0, len = hop.getNumRecipients(); r < len; ++r) { - Tokenizer tokens(hop.getRecipient(r), CharSeparator("/")); - Tokenizer::iterator token = tokens.begin(); - for (uint32_t t = 0; t < 2 && token != tokens.end(); ++t, ++token) { - // empty - } - if (token != tokens.end()) { - colIdx = std::max(colIdx, atoi(&token->c_str()[1])); - } - } - if (colIdx < 0) { - throw config::InvalidConfigException(vespalib::make_string("Failed to process cluster '%s'.", - hop.getName().c_str())); - } - clusters.insert(ClusterMap::value_type(hop.getName().substr(15), colIdx + 1)); - } - - void - processTable(const mbus::RoutingTableSpec &table, ClusterMap &clusters) - { - clusters.clear(); - for (uint32_t i = 0, len = table.getNumHops(); i < len; ++i) { - const mbus::HopSpec &hop = table.getHop(i); - if (hop.getName().find("search/cluster.") == 0) { - processHop(hop, clusters); - } - } - if (clusters.empty()) { - throw config::InvalidConfigException("No search clusters found to resolve document location for."); - } - } - - void - processRouting(const mbus::RoutingSpec &routing, ClusterMap &clusters) - { - const mbus::RoutingTableSpec *table = NULL; - for (uint32_t i = 0, len = routing.getNumTables(); i < len; ++i) { - const mbus::RoutingTableSpec &ref = routing.getTable(i); - if (ref.getProtocol() == documentapi::DocumentProtocol::NAME) { - table = &ref; - break; - } - } - if (table == NULL) { - throw config::InvalidConfigException("No routing table available to derive config from."); - } - processTable(*table, clusters); - } - - uint32_t - getNumColumns(const mbus::RoutingSpec &routing, const std::string &clusterName) - { - ClusterMap clusters; - processRouting(routing, clusters); - - if (clusterName.empty() && clusters.size() == 1) { - return clusters.begin()->second; - } - - ClusterMap::iterator it = clusters.find(clusterName); - if (it == clusters.end()) { - std::string str = "Cluster name must be one of "; - int i = 0, len = clusters.size(); - for (it = clusters.begin(); it != clusters.end(); ++it, ++i) - { - str.append("'").append(it->first).append("'"); - if (i < len - 2) { - str.append(", "); - } else if (i == len - 2) { - str.append(" or "); - } - } - str.append("."); - throw config::InvalidConfigException(str); - } - - return it->second; - } -} - -Locator::Locator(uint32_t numColumns) : - _factory(), - _numColumns(numColumns) -{ - // empty -} - -void -Locator::configure(const std::string &configId, const std::string &clusterName) -{ - config::ConfigUri configUri(configId); - // Configure by inspecting routing config. - struct MyCB : public mbus::IConfigHandler { - mbus::RoutingSpec mySpec; - MyCB() : mySpec() {} - bool setupRouting(const mbus::RoutingSpec &spec) override { - mySpec = spec; - return true; - } - } myCB; - mbus::ConfigAgent agent(myCB); - agent.configure(ConfigGetter<messagebus::MessagebusConfig>::getConfig(configUri.getConfigId(), configUri.getContext())); - _numColumns = getNumColumns(myCB.mySpec, clusterName); -} - -document::BucketId -Locator::getBucketId(document::DocumentId &docId) -{ - return _factory.getBucketId(docId); -} - -uint32_t -Locator::getSearchColumn(document::DocumentId &docId) -{ - vdslib::BucketDistribution dist(_numColumns, 16u); - return dist.getColumn(getBucketId(docId)); -} diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h deleted file mode 100644 index 02df3a9916f..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/document/base/documentid.h> -#include <vespa/document/bucket/bucketidfactory.h> - -class Locator { -private: - document::BucketIdFactory _factory; - uint32_t _numColumns; - -public: - /** - * Constructs a new locator object. - */ - Locator(uint32_t numColumns = 0); - - /** - * Configures this locator using the supplied configuration id and cluster name. This method will - * subscribe to some known config and attempt to retrieve the number of columns of the given search - * cluster from that. - * - * This method throws an exception if it could not be configured. - * - * @param configId The config identifier to subscribe to. - * @param clusterName The name of the search cluster to resolve locations in. - */ - void configure(const std::string &configId, - const std::string &clusterName); - - /** - * Returns the bucket id to which a document id belongs. - * - * @param docId The document id to resolve. - * @return The corresponding bucket id. - */ - document::BucketId getBucketId(document::DocumentId &docId); - - /** - * Returns the column in which the given document id belongs. - * - * @param docId The document id to resolve. - * @return The corresponding column. - */ - uint32_t getSearchColumn(document::DocumentId &docId); -}; - diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp deleted file mode 100644 index da20eb5c283..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "application.h" - -int -main(int argc, char **argv) -{ - Application app; - return app.Entry(argc, argv); -} |