diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /configutil/src/lib |
Publish
Diffstat (limited to 'configutil/src/lib')
-rw-r--r-- | configutil/src/lib/.gitignore | 2 | ||||
-rw-r--r-- | configutil/src/lib/CMakeLists.txt | 9 | ||||
-rw-r--r-- | configutil/src/lib/configstatus.cpp | 237 | ||||
-rw-r--r-- | configutil/src/lib/configstatus.h | 31 | ||||
-rw-r--r-- | configutil/src/lib/modelinspect.cpp | 366 | ||||
-rw-r--r-- | configutil/src/lib/modelinspect.h | 52 | ||||
-rw-r--r-- | configutil/src/lib/tags.cpp | 39 | ||||
-rw-r--r-- | configutil/src/lib/tags.h | 12 |
8 files changed, 748 insertions, 0 deletions
diff --git a/configutil/src/lib/.gitignore b/configutil/src/lib/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/configutil/src/lib/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/configutil/src/lib/CMakeLists.txt b/configutil/src/lib/CMakeLists.txt new file mode 100644 index 00000000000..290957bf9f3 --- /dev/null +++ b/configutil/src/lib/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(configutil_util STATIC + SOURCES + tags.cpp + modelinspect.cpp + configstatus.cpp + DEPENDS + configdefinitions +) diff --git a/configutil/src/lib/configstatus.cpp b/configutil/src/lib/configstatus.cpp new file mode 100644 index 00000000000..f889c436a97 --- /dev/null +++ b/configutil/src/lib/configstatus.cpp @@ -0,0 +1,237 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("vespa-config-status"); +#include <iostream> + +#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vbench/http/http_result_handler.h> +#include <vbench/http/server_spec.h> +#include <vbench/http/http_client.h> + +#include <lib/tags.h> + +#include "configstatus.h" + +using configdefinitions::tagsContain; + +struct ComponentTraverser : public vespalib::slime::ObjectTraverser +{ + const std::string _configId; + std::string _component; + enum { + ROOT, + COMPONENT + } _state; + std::map<std::string, int64_t> &_generations; + + ComponentTraverser(std::string configId, + std::map<std::string, int64_t> &generations) + : _configId(configId), _state(ROOT), _generations(generations) + {} + + void object(const vespalib::slime::Inspector &inspector) { + inspector.traverse(*this); + } + + static void collect(const std::string configId, const vespalib::Slime &slime, + std::map<std::string, int64_t> &generations) { + ComponentTraverser traverser(configId, generations); + slime.get()["config"].traverse(traverser); + } + + virtual void + field(const vespalib::slime::Memory &symbol_name, const vespalib::slime::Inspector &inspector) { + switch (_state) { + case ROOT: + _component = symbol_name.make_string(); + _state = COMPONENT; + inspector.traverse(*this); + _state = ROOT; + break; + case COMPONENT: + const std::string key = symbol_name.make_string(); + int64_t value; + if (key == "generation") { + if (inspector.type().getId() == vespalib::slime::DOUBLE::ID) { + value = (int64_t) inspector.asDouble(); + _generations[_component] = value; + } else if (inspector.type().getId() == vespalib::slime::LONG::ID) { + value = inspector.asLong(); + _generations[_component] = value; + } else { + value = 0; + std::cerr << _configId << ":" << _component << + "Generation has wrong type" << std::endl; + } + } + + break; + } + } +}; + +class MyHttpHandler : public vbench::HttpResultHandler { +private: + std::string _json; + std::string _error; + std::string _configId; + +public: + + MyHttpHandler(std::string configId) + : _json(), _error(), _configId(configId) + {} + + virtual void handleHeader(const vbench::string &name, const vbench::string &value) { + (void) name; + (void) value; + } + + virtual void handleContent(const vbench::Memory &data) { + _json += std::string(data.data, data.size); + } + + virtual void handleFailure(const vbench::string &reason) { + std::cerr << _configId << ": Failed to fetch json: " << reason << std::endl; + _error = reason; + } + + bool failed() { + return(_error.size() > 0); + } + + std::string getJson() { + return _json; + } +}; + +ConfigStatus::ConfigStatus(Flags flags, const config::ConfigUri uri) + : _cfg(), _flags(flags), _generation(0) +{ + if (_flags.verbose) { + std::cerr << "Subscribing to model config with config id " << + uri.getConfigId() << std::endl; + } + try { + config::ConfigSubscriber subscriber(uri.getContext()); + config::ConfigHandle<cloud::config::ModelConfig>::UP handle = + subscriber.subscribe<cloud::config::ModelConfig>(uri.getConfigId()); + subscriber.nextConfig(0); + _cfg = handle->getConfig(); + _generation = subscriber.getGeneration(); + } catch(config::ConfigRuntimeException &e) { + std::cerr << e.getMessage() << std::endl; + } + + if (_cfg.get() == NULL) { + std::cerr << "FATAL ERROR: failed to get model configuration." << std::endl; + exit(1); + } +} + +int +ConfigStatus::action() +{ + bool allUpToDate = true; + + for (size_t i = 0; i < _cfg->hosts.size(); i++) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + + for (size_t j = 0; j < hconf.services.size(); j++) { + const cloud::config::ModelConfig::Hosts::Services &svc = hconf.services[j]; + + if (svc.type == "configserver") { + continue; + } + + for (size_t k = 0; k < svc.ports.size(); k++) { + std::string tags = svc.ports[k].tags; + if (tagsContain(tags, "http") && + tagsContain(tags, "state")) { + bool upToDate = checkServiceGeneration(svc.configid, hconf.name, + svc.ports[k].number, + "/state/v1/config"); + + if (!upToDate) { + if (svc.type == "searchnode" || + svc.type == "filedistributorservice" || + svc.type == "topleveldispatch") + { + std::cerr << "[generation not up-to-date ignored]" << std::endl; + } else { + allUpToDate = false; + } + } + break; + } + } + } + } + + return allUpToDate ? 0 : 1; +} + +bool +ConfigStatus::fetch_json(std::string configId, std::string host, int port, + std::string path, std::string &data) +{ + MyHttpHandler myHandler(configId); + bool ok = vbench::HttpClient::fetch(vbench::ServerSpec(host, port), path, myHandler); + + if (ok) { + data = myHandler.getJson(); + return true; + } else { + return false; + } +} + +bool +ConfigStatus::checkServiceGeneration(std::string configId, std::string host, int port, std::string path) +{ + std::string data; + vespalib::Slime slime; + + if (!fetch_json(configId, host, port, path, data)) { + return false; + } + + size_t size = vespalib::slime::JsonFormat::decode(data, slime); + + if (size == 0) { + std::cerr << configId << ": JSON parsing failed" << std::endl; + return false; + } + + vespalib::slime::SimpleBuffer buf; + vespalib::slime::JsonFormat::encode(slime, buf, false); + + if (slime.get()["config"].valid()) { + std::map<std::string, int64_t> generations; + ComponentTraverser::collect(configId, slime, generations); + bool upToDate = true; + + std::map<std::string, int64_t>::iterator iter; + for (iter = generations.begin(); iter != generations.end(); iter++) { + if (iter->second != _generation) { + std::cout << configId << ":" << iter->first << " has generation " << + iter->second << " not " << _generation << std::endl; + upToDate = false; + } else { + if (_flags.verbose) { + std::cout << configId << ":" << iter->first << + " has the latest generation " << iter->second << std::endl; + } + } + } + + return upToDate; + } else { + std::cerr << configId << ": No valid config object" << std::endl; + + return false; + } +} diff --git a/configutil/src/lib/configstatus.h b/configutil/src/lib/configstatus.h new file mode 100644 index 00000000000..41e3d2fe782 --- /dev/null +++ b/configutil/src/lib/configstatus.h @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config-model.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/config.h> + +class ConfigStatus +{ +public: + struct Flags { + bool verbose; + Flags() + : verbose(false) + {} + }; + + ConfigStatus(Flags flags, const config::ConfigUri uri); + int action(); + +private: + std::unique_ptr<cloud::config::ModelConfig> _cfg; + Flags _flags; + int64_t _generation; + + bool fetch_json(std::string configId, std::string host, int port, std::string path, + std::string &data); + bool checkServiceGeneration(std::string configId, std::string host, int port, + std::string path); +}; + diff --git a/configutil/src/lib/modelinspect.cpp b/configutil/src/lib/modelinspect.cpp new file mode 100644 index 00000000000..69fd2b804df --- /dev/null +++ b/configutil/src/lib/modelinspect.cpp @@ -0,0 +1,366 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <ctype.h> +#include <vespa/log/log.h> +LOG_SETUP("vespa-model-inspect"); +#include <iostream> +#include "modelinspect.h" +#include <set> +#include <lib/tags.h> + +using configdefinitions::tagsContain; +using configdefinitions::upcase; + +ModelInspect::ModelInspect(Flags flags, const config::ConfigUri uri, std::ostream &out) + : _cfg(), _flags(flags), _out(out) +{ + if (_flags.verbose) { + std::cerr << "subscribing to model config with configid " << uri.getConfigId() << "\n"; + } + + try { + _cfg = config::ConfigGetter<cloud::config::ModelConfig>::getConfig(uri.getConfigId(), + uri.getContext()); + } catch(config::ConfigRuntimeException &e) { + std::cerr << e.getMessage() << "\n"; + } + if (_cfg.get() != NULL) { + if (_flags.verbose) std::cerr << "success!\n"; + } else { + std::cerr << "FATAL ERROR: failed to get model configuration.\n"; + exit(1); + } +} + +ModelInspect::~ModelInspect() +{ +} + +void +ModelInspect::printPort(const vespalib::string &host, int port, + const vespalib::string &tags) +{ + if (_flags.tagfilt) { + for (size_t i = 0; i < _flags.tagFilter.size(); ++i) { + if (! tagsContain(tags, _flags.tagFilter[i])) { + return; + } + } + } + if (_flags.makeuri && tagsContain(tags, "HTTP")) { + _out << " http://" << host << ":" << port << "/"; + } else { + _out << " tcp/" << host << ":" << port; + } + if (_flags.tagfilt) { + _out << "\n"; + } else { + vespalib::string upper = upcase(tags); + _out << " (" << upper << ")\n"; + } +} + +void +ModelInspect::printService(const cloud::config::ModelConfig::Hosts::Services &svc, + const vespalib::string &host) +{ + if (!_flags.tagfilt) { + _out << svc.name << " @ " << host << " : " << svc.clustertype << std::endl; + _out << svc.configid << std::endl; + } + for (size_t i = 0; i < svc.ports.size(); ++i) { + printPort(host, svc.ports[i].number, svc.ports[i].tags); + } +} + +int +ModelInspect::action(int cnt, char **argv) +{ + const vespalib::string cmd = *argv++; + if (cnt == 1) { + if (cmd == "yamldump") { + yamlDump(); + return 0; + } + if (cmd == "hosts") { + listHosts(); + return 0; + } + if (cmd == "services") { + listServices(); + return 0; + } + if (cmd == "clusters") { + listClusters(); + return 0; + } + if (cmd == "configids") { + listConfigIds(); + return 0; + } + if (cmd == "filter:hosts") { + if (!_flags.tagfilt) { + std::cerr << "filter needs some filter options" << std::endl; + return 1; + } + std::cerr << "not implemented" << std::endl; + return 1; + } + if (cmd == "filter:ports") { + if (!_flags.tagfilt) { + std::cerr << "filter needs some filter options" << std::endl; + return 1; + } + return listAllPorts(); + } + } + if (cnt == 2) { + vespalib::string arg = *argv++; + if (cmd == "host") { + return listHost(arg); + } + if (cmd == "cluster") { + return listCluster(arg); + } + if (cmd == "service") { + size_t colon = arg.find(':'); + if (colon != vespalib::string::npos) { + return listService(arg.substr(0, colon), + arg.substr(colon + 1)); + } else { + return listService(arg); + } + } + if (cmd == "configid") { + return listConfigId(arg); + } + } + if (cnt == 3) { + vespalib::string arg1 = *argv++; + vespalib::string arg2 = *argv++; + if (cmd == "get-index-of") { + return getIndexOf(arg1, arg2); + } + }; + std::cerr << "bad args '" << cmd << "' (got " << cnt << " arguments)" << std::endl; + return 1; +} + +void +ModelInspect::dumpService(const cloud::config::ModelConfig::Hosts::Services &svc, + const vespalib::string &host) +{ + _out << "- servicename: " << svc.name << "\n"; + _out << " servicetype: " << svc.type << "\n"; + _out << " clustertype: " << svc.clustertype << "\n"; + _out << " clustername: " << svc.clustername << "\n"; + _out << " index: " << svc.index << "\n"; + _out << " hostname: " << host << "\n"; + _out << " config-id: " << svc.configid << "\n"; + + if (svc.ports.size() > 0) { + _out << " ports: \n"; + for (size_t i = 0; i < svc.ports.size(); ++i) { + _out << " - " << svc.ports[i].number << "\n"; + } + } +} + +void +ModelInspect::yamlDump() +{ + _out << "--- \n"; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + dumpService(hconf.services[j], hconf.name); + } + } +} + +void +ModelInspect::listHosts() +{ + std::vector<vespalib::string> hosts; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + hosts.push_back(hconf.name); + } + std::sort(hosts.begin(), hosts.end()); + for (size_t i = 0; i < hosts.size(); ++i) { + _out << hosts[i] << std::endl; + } +} + +void +ModelInspect::listServices() +{ + typedef std::set<vespalib::string> Set; + Set services; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + services.insert(hconf.services[j].type); + } + } + for (Set::const_iterator it = services.begin(); it != services.end(); ++it) { + _out << (*it) << std::endl; + } +} + +void +ModelInspect::listClusters() +{ + typedef std::set<vespalib::string> Set; + Set clusters; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + clusters.insert(hconf.services[j].clustername); + } + } + for (Set::const_iterator it = clusters.begin(); it != clusters.end(); ++it) { + _out << (*it) << std::endl; + } +} + +void +ModelInspect::listConfigIds() +{ + std::vector<vespalib::string> configids; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + configids.push_back(hconf.services[j].configid); + } + } + std::sort(configids.begin(), configids.end()); + for (size_t i = 0; i < configids.size(); ++i) { + _out << configids[i] << std::endl; + } +} + +int +ModelInspect::listHost(const vespalib::string host) +{ + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + if (host == hconf.name) { + for (size_t j = 0; j < hconf.services.size(); ++j) { + printService(hconf.services[j], host); + } + return 0; + } + } + std::cerr << "no config found for host '" << host << "'\n"; + return 1; +} + +int +ModelInspect::listCluster(const vespalib::string cluster) +{ + bool found = false; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + if (cluster == hconf.services[j].clustername) { + found = true; + printService(hconf.services[j], hconf.name); + } + } + } + if (found) return 0; + std::cerr << "no config found for cluster '" << cluster << "'\n"; + return 1; +} + +int +ModelInspect::listAllPorts() +{ + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + printService(hconf.services[j], hconf.name); + } + } + return 0; +} + +int +ModelInspect::listService(const vespalib::string svctype) +{ + bool found = false; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + if (svctype == hconf.services[j].type) { + found = true; + printService(hconf.services[j], hconf.name); + } + } + } + if (found) return 0; + std::cerr << "no services found with type '" << svctype << "'\n"; + return 1; +} + + +int +ModelInspect::listService(const vespalib::string cluster, + const vespalib::string svctype) +{ + bool found = false; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + if (cluster == hconf.services[j].clustername && + svctype == hconf.services[j].type) + { + found = true; + printService(hconf.services[j], hconf.name); + } + } + } + if (found) return 0; + std::cerr << "no services found with type '" << svctype << "' in cluster '" << cluster << "'\n"; + return 1; +} + +int +ModelInspect::listConfigId(const vespalib::string configid) +{ + bool found = false; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + for (size_t j = 0; j < hconf.services.size(); ++j) { + if (configid == hconf.services[j].configid) { + found = true; + printService(hconf.services[j], hconf.name); + } + } + } + if (found) return 0; + std::cerr << "no services found with configid '" << configid << "'\n"; + return 1; +} + +int +ModelInspect::getIndexOf(const vespalib::string service, const vespalib::string host) +{ + bool found = false; + for (size_t i = 0; i < _cfg->hosts.size(); ++i) { + const cloud::config::ModelConfig::Hosts &hconf = _cfg->hosts[i]; + if (host == hconf.name) { + for (size_t j = 0; j < hconf.services.size(); ++j) { + if (service == hconf.services[j].type) { + found = true; + _out << hconf.services[j].index << std::endl; + } + } + } + } + if (found) return 0; + std::cerr << "no service of type '" << service << "' found for host '" << host << "'\n"; + return 1; +} diff --git a/configutil/src/lib/modelinspect.h b/configutil/src/lib/modelinspect.h new file mode 100644 index 00000000000..b50ea04e197 --- /dev/null +++ b/configutil/src/lib/modelinspect.h @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/config-model.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/config.h> + +class ModelInspect + +{ +public: + struct Flags { + bool verbose; + bool makeuri; + bool tagfilt; + std::vector<vespalib::string> tagFilter; + Flags() + : verbose(false), makeuri(false), tagfilt(false), + tagFilter() + {} + }; + + ModelInspect(Flags flags, const config::ConfigUri uri, std::ostream &out); + virtual ~ModelInspect(); + + int action(int cnt, char *argv[]); + + virtual void yamlDump(); + virtual void listHosts(); + virtual void listServices(); + virtual void listClusters(); + virtual void listConfigIds(); + virtual int listHost(const vespalib::string host); + virtual int listCluster(const vespalib::string cluster); + virtual int listAllPorts(); + virtual int listService(const vespalib::string svctype); + virtual int listService(const vespalib::string cluster, + const vespalib::string svctype); + virtual int listConfigId(const vespalib::string configid); + virtual int getIndexOf(const vespalib::string service, const vespalib::string host); + +private: + std::unique_ptr<cloud::config::ModelConfig> _cfg; + Flags _flags; + std::ostream &_out; + + void printService(const cloud::config::ModelConfig::Hosts::Services &svc, + const vespalib::string &host); + void printPort(const vespalib::string &host, int port, + const vespalib::string &tags); + void dumpService(const cloud::config::ModelConfig::Hosts::Services &svc, + const vespalib::string &host); + +}; diff --git a/configutil/src/lib/tags.cpp b/configutil/src/lib/tags.cpp new file mode 100644 index 00000000000..50b01c80ba2 --- /dev/null +++ b/configutil/src/lib/tags.cpp @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> + +#include <vespa/vespalib/stllike/string.h> + +#include "tags.h" + +namespace configdefinitions { + +vespalib::string upcase(const vespalib::string &orig) +{ + vespalib::string upper(orig); + for (size_t i = 0; i < orig.size(); ++i) { + int l = (unsigned char)orig[i]; + upper[i] = (unsigned char)toupper(l); + } + return upper; +} + +bool tagsContain(const vespalib::string &tags, const vespalib::string &tag) +{ + vespalib::string allupper = upcase(tags); + vespalib::string tagupper = upcase(tag); + + for (;;) { + size_t pos = allupper.rfind(' '); + if (pos == vespalib::string::npos) { + break; + } + if (allupper.substr(pos+1) == tagupper) { + return true; + } + allupper.resize(pos); + } + return (allupper == tagupper); +} + +} diff --git a/configutil/src/lib/tags.h b/configutil/src/lib/tags.h new file mode 100644 index 00000000000..11764fe7af1 --- /dev/null +++ b/configutil/src/lib/tags.h @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace configdefinitions { + +vespalib::string upcase(const vespalib::string &orig); +bool tagsContain(const vespalib::string &tags, const vespalib::string &tag); + +} + |