diff options
author | Haavard <havardpe@yahoo-inc.com> | 2017-07-11 10:27:39 +0000 |
---|---|---|
committer | Haavard <havardpe@yahoo-inc.com> | 2017-07-11 10:27:39 +0000 |
commit | a649575bc025ebbc9e46c475ac2fb445d3bc6276 (patch) | |
tree | 7e3e9de970900833c22069a77c210db42e8ceff1 /vespalib | |
parent | def65208db4f09716cea530e5032febdca3f89c1 (diff) |
remove lazy resolver
Diffstat (limited to 'vespalib')
-rw-r--r-- | vespalib/CMakeLists.txt | 1 | ||||
-rw-r--r-- | vespalib/src/tests/net/lazy_resolver/CMakeLists.txt | 8 | ||||
-rw-r--r-- | vespalib/src/tests/net/lazy_resolver/lazy_resolver_test.cpp | 174 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/net/CMakeLists.txt | 1 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/net/lazy_resolver.cpp | 205 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/net/lazy_resolver.h | 108 |
6 files changed, 0 insertions, 497 deletions
diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index d7f3f9bb73a..93ac717d9ef 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -48,7 +48,6 @@ vespa_define_module( src/tests/make_fixture_macros src/tests/memory src/tests/net/async_resolver - src/tests/net/lazy_resolver src/tests/net/selector src/tests/net/socket src/tests/net/socket_spec diff --git a/vespalib/src/tests/net/lazy_resolver/CMakeLists.txt b/vespalib/src/tests/net/lazy_resolver/CMakeLists.txt deleted file mode 100644 index 440ee7ab873..00000000000 --- a/vespalib/src/tests/net/lazy_resolver/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_lazy_resolver_test_app TEST - SOURCES - lazy_resolver_test.cpp - DEPENDS - vespalib -) -vespa_add_test(NAME vespalib_lazy_resolver_test_app COMMAND vespalib_lazy_resolver_test_app) diff --git a/vespalib/src/tests/net/lazy_resolver/lazy_resolver_test.cpp b/vespalib/src/tests/net/lazy_resolver/lazy_resolver_test.cpp deleted file mode 100644 index ba448c32cb6..00000000000 --- a/vespalib/src/tests/net/lazy_resolver/lazy_resolver_test.cpp +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/net/lazy_resolver.h> -#include <vespa/vespalib/net/socket_spec.h> - -using namespace vespalib; - -TEST("require that lazy resolver internal duration type is appropriate") { - LazyResolver::seconds my_secs = std::chrono::milliseconds(500); - EXPECT_EQUAL(my_secs.count(), 0.5); -} - -TEST("require that lazy resolver can be used to resolve connect spec") { - vespalib::string spec("tcp/localhost:123"); - auto resolver = LazyResolver::create(); - auto address = resolver->make_address(spec); - auto resolved = address->resolve(); - fprintf(stderr, "resolver(spec:%s) -> '%s'\n", spec.c_str(), resolved.c_str()); - EXPECT_EQUAL(spec, address->spec()); - EXPECT_NOT_EQUAL(resolved, address->spec()); - EXPECT_EQUAL(resolved, SocketSpec(spec).client_address().spec()); - EXPECT_EQUAL(resolved, SocketAddress::select_remote(123, "localhost").spec()); -} - -TEST("require that lazy resolver can be used to resolve host name") { - vespalib::string host_name("localhost"); - auto resolver = LazyResolver::create(); - auto host = resolver->make_host(host_name); - auto resolved = host->resolve(); - fprintf(stderr, "resolver(host_name:%s) -> '%s'\n", host_name.c_str(), resolved.c_str()); - EXPECT_EQUAL(host_name, host->host_name()); - EXPECT_NOT_EQUAL(resolved, host->host_name()); - EXPECT_EQUAL(resolved, SocketSpec("tcp/localhost:123").client_address().ip_address()); - EXPECT_EQUAL(resolved, SocketAddress::select_remote(123, "localhost").ip_address()); - EXPECT_EQUAL(resolved, LazyResolver::default_resolve_host(host_name)); -} - -vespalib::string dummy_resolve_host(const vespalib::string &) { return "ip.addr"; } - -TEST("require that host name resolve function can be overridden (bonus: slow resolve warning)") { - LazyResolver::Params params; - params.resolve_host = dummy_resolve_host; - params.max_resolve_time = LazyResolver::seconds(0); - auto resolver = LazyResolver::create(params); - EXPECT_EQUAL(resolver->make_address("tcp/host_name:123")->resolve(), "tcp/ip.addr:123"); -} - -struct ResolveFixture { - std::mutex ip_lock; - std::map<vespalib::string,vespalib::string> ip_map; - std::map<vespalib::string, size_t> ip_cnt; - LazyResolver::SP resolver; - void set_ip_addr(const vespalib::string &host, const vespalib::string &ip_addr) { - std::lock_guard<std::mutex> guard(ip_lock); - ip_map[host] = ip_addr; - } - vespalib::string get_ip_addr(const vespalib::string &host) { - std::lock_guard<std::mutex> guard(ip_lock); - ++ip_cnt[host]; - return ip_map[host]; - } - size_t get_cnt(const vespalib::string &host) { - std::lock_guard<std::mutex> guard(ip_lock); - return ip_cnt[host]; - } - size_t get_total_cnt() { - size_t total = 0; - std::lock_guard<std::mutex> guard(ip_lock); - for (const auto &entry: ip_cnt) { - total += entry.second; - } - return total; - } - ResolveFixture(double max_age) : ip_lock(), ip_map(), ip_cnt(), resolver() { - LazyResolver::Params params; - params.resolve_host = [this](const vespalib::string &host_name){ return get_ip_addr(host_name); }; - params.max_result_age = LazyResolver::seconds(max_age); - resolver = LazyResolver::create(std::move(params)); - set_ip_addr("localhost", "127.0.0.1"); - set_ip_addr("127.0.0.1", "127.0.0.1"); - } - LazyResolver::Address::SP make(const vespalib::string &spec) { return resolver->make_address(spec); } -}; - -TEST_F("require that lazy resolver can be used to resolve connect specs without host names", ResolveFixture(300)) { - EXPECT_EQUAL(f1.make("this is bogus")->resolve(), "this is bogus"); - EXPECT_EQUAL(f1.make("tcp/123")->resolve(), "tcp/123"); - EXPECT_EQUAL(f1.make("ipc/file:my_socket")->resolve(), "ipc/file:my_socket"); - EXPECT_EQUAL(f1.make("ipc/name:my_socket")->resolve(), "ipc/name:my_socket"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(f1.get_total_cnt(), 0u); -} - -TEST_F("require that resolved hosts can be shared between addresses", ResolveFixture(300)) { - auto addr1 = f1.make("tcp/localhost:123"); - auto addr2 = f1.make("tcp/localhost:456"); - EXPECT_EQUAL(addr1->resolve(), "tcp/127.0.0.1:123"); - EXPECT_EQUAL(addr2->resolve(), "tcp/127.0.0.1:456"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(f1.get_cnt("localhost"), 1u); - EXPECT_EQUAL(f1.get_total_cnt(), 1u); -} - -TEST_F("require that resolved hosts are discarded when not used", ResolveFixture(300)) { - EXPECT_EQUAL(f1.make("tcp/localhost:123")->resolve(), "tcp/127.0.0.1:123"); - EXPECT_EQUAL(f1.make("tcp/localhost:456")->resolve(), "tcp/127.0.0.1:456"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(f1.get_cnt("localhost"), 2u); - EXPECT_EQUAL(f1.get_total_cnt(), 2u); -} - -TEST_F("require that host names resolving to themselves (ip addresses) are not shared", ResolveFixture(300)) { - auto addr1 = f1.make("tcp/127.0.0.1:123"); - auto addr2 = f1.make("tcp/127.0.0.1:456"); - EXPECT_EQUAL(addr1->resolve(), "tcp/127.0.0.1:123"); - EXPECT_EQUAL(addr2->resolve(), "tcp/127.0.0.1:456"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(f1.get_cnt("127.0.0.1"), 2u); - EXPECT_EQUAL(f1.get_total_cnt(), 2u); -} - -TEST_F("require that resolve changes can be detected", ResolveFixture(0)) { - auto addr = f1.make("tcp/localhost:123"); - f1.set_ip_addr("localhost", "127.0.0.2"); - EXPECT_EQUAL(addr->resolve(), "tcp/127.0.0.1:123"); - f1.resolver->wait_for_pending_updates(); - f1.set_ip_addr("localhost", "127.0.0.3"); - EXPECT_EQUAL(addr->resolve(), "tcp/127.0.0.2:123"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(addr->resolve(), "tcp/127.0.0.3:123"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(f1.get_cnt("localhost"), 4u); - EXPECT_EQUAL(f1.get_total_cnt(), 4u); -} - -TEST_F("require that resolve changes are not detected when old results are still fresh", ResolveFixture(300)) { - auto addr = f1.make("tcp/localhost:123"); - f1.set_ip_addr("localhost", "127.0.0.2"); - EXPECT_EQUAL(addr->resolve(), "tcp/127.0.0.1:123"); - f1.resolver->wait_for_pending_updates(); - f1.set_ip_addr("localhost", "127.0.0.3"); - EXPECT_EQUAL(addr->resolve(), "tcp/127.0.0.1:123"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(addr->resolve(), "tcp/127.0.0.1:123"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(f1.get_cnt("localhost"), 1u); - EXPECT_EQUAL(f1.get_total_cnt(), 1u); -} - -TEST_F("require that missing ip address gives invalid spec", ResolveFixture(300)) { - f1.set_ip_addr("localhost", ""); - auto addr = f1.make("tcp/localhost:123"); - EXPECT_EQUAL(addr->resolve(), "invalid"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(f1.get_cnt("localhost"), 1u); - EXPECT_EQUAL(f1.get_total_cnt(), 1u); -} - -TEST_F("require that all ip address results are treated equally (including empty ones)", ResolveFixture(0)) { - auto addr = f1.make("tcp/localhost:123"); - f1.set_ip_addr("localhost", ""); - EXPECT_EQUAL(addr->resolve(), "tcp/127.0.0.1:123"); - f1.resolver->wait_for_pending_updates(); - f1.set_ip_addr("localhost", "127.0.0.2"); - EXPECT_EQUAL(addr->resolve(), "invalid"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(addr->resolve(), "tcp/127.0.0.2:123"); - f1.resolver->wait_for_pending_updates(); - EXPECT_EQUAL(f1.get_cnt("localhost"), 4u); - EXPECT_EQUAL(f1.get_total_cnt(), 4u); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/net/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/CMakeLists.txt index 42023cb73b0..f652be53560 100644 --- a/vespalib/src/vespa/vespalib/net/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_add_library(vespalib_vespalib_net OBJECT SOURCES async_resolver.cpp - lazy_resolver.cpp selector.cpp server_socket.cpp socket.cpp diff --git a/vespalib/src/vespa/vespalib/net/lazy_resolver.cpp b/vespalib/src/vespa/vespalib/net/lazy_resolver.cpp deleted file mode 100644 index ef2b44958c9..00000000000 --- a/vespalib/src/vespa/vespalib/net/lazy_resolver.cpp +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "lazy_resolver.h" -#include "socket_spec.h" - -#include <vespa/log/log.h> -LOG_SETUP(".vespalib.net.lazy_resolver"); - -namespace vespalib { - -VESPA_THREAD_STACK_TAG(lazy_resolver_executor_thread); - -LazyResolver::Params::Params() - : resolve_host(default_resolve_host), - max_result_age(seconds(300.0)), - max_resolve_time(seconds(1.0)) -{ -} - -//----------------------------------------------------------------------------- - -LazyResolver::Host::Host(const vespalib::string &host_name, LazyResolver::SP resolver, - const vespalib::string &ip_address) - : _host_name(host_name), - _resolver(std::move(resolver)), - _ip_lock(), - _ip_pending(false), - _ip_address(ip_address), - _ip_updated(clock::now()) -{ -} - -void -LazyResolver::Host::update_ip_address(const vespalib::string &ip_address) -{ - std::lock_guard<std::mutex> guard(_ip_lock); - _ip_pending = false; - _ip_address = ip_address; - _ip_updated = clock::now(); -} - -LazyResolver::Host::~Host() -{ - // clean up weak_ptr to this - _resolver->try_lookup_host(_host_name); -} - -vespalib::string -LazyResolver::Host::resolve() -{ - std::lock_guard<std::mutex> guard(_ip_lock); - if (!_ip_pending && _resolver->should_request_update(_ip_updated)) { - // TODO(havardpe): switch to weak_from_this() when available - _ip_pending = _resolver->try_request_update(shared_from_this()); - } - return _ip_address; -} - -//----------------------------------------------------------------------------- - -vespalib::string -LazyResolver::Address::resolve() -{ - if (_host) { - return SocketSpec(_spec).replace_host(_host->resolve()).spec(); - } - return _spec; -} - -//----------------------------------------------------------------------------- - -void -LazyResolver::UpdateTask::run() -{ - if (Host::SP host = weak_host.lock()) { - host->update_ip_address(resolver.resolve_host_now(host->host_name())); - } -} - -//----------------------------------------------------------------------------- - -LazyResolver::LazyResolver(Params params) - : _host_lock(), - _host_map(), - _params(std::move(params)), - _executor(1, 128*1024, lazy_resolver_executor_thread, 4096) -{ -} - -LazyResolver::Host::SP -LazyResolver::try_lookup_host(const vespalib::string &host_name, - const std::lock_guard<std::mutex> &guard) -{ - (void) guard; - auto pos = _host_map.find(host_name); - if (pos != _host_map.end()) { - Host::SP host = pos->second.lock(); - if (host) { - return host; - } else { - _host_map.erase(pos); - } - } - return Host::SP(nullptr); -} - -LazyResolver::Host::SP -LazyResolver::try_lookup_host(const vespalib::string &host_name) -{ - std::lock_guard<std::mutex> guard(_host_lock); - return try_lookup_host(host_name, guard); -} - -LazyResolver::Host::SP -LazyResolver::insert_host(const vespalib::string &host_name, const vespalib::string &ip_address) -{ - std::lock_guard<std::mutex> guard(_host_lock); - Host::SP host = try_lookup_host(host_name, guard); - if (!host) { - host.reset(new Host(host_name, shared_from_this(), ip_address)); - _host_map.emplace(host_name, host); - } - return host; -} - -vespalib::string -LazyResolver::resolve_host_now(const vespalib::string &host_name) -{ - auto before = clock::now(); - vespalib::string ip_address = _params.resolve_host(host_name); - seconds resolve_time = (clock::now() - before); - if (resolve_time >= _params.max_resolve_time) { - LOG(warning, "slow resolve time: '%s' -> '%s' (%g s)", - host_name.c_str(), ip_address.c_str(), resolve_time.count()); - } - if (ip_address.empty()) { - LOG(warning, "could not resolve host name: '%s'", host_name.c_str()); - } - return ip_address; -} - -bool -LazyResolver::should_request_update(clock::time_point ip_updated) -{ - seconds result_age = (clock::now() - ip_updated); - return (result_age >= _params.max_result_age); -} - -bool -LazyResolver::try_request_update(std::weak_ptr<Host> self) -{ - Executor::Task::UP task(new UpdateTask(*this, std::move(self))); - auto rejected = _executor.execute(std::move(task)); - return !rejected; -} - -//----------------------------------------------------------------------------- - -LazyResolver::~LazyResolver() -{ - _executor.shutdown().sync(); -} - -LazyResolver::Host::SP -LazyResolver::make_host(const vespalib::string &host_name) -{ - if (host_name.empty()) { - return Host::SP(nullptr); - } - Host::SP host = try_lookup_host(host_name); - if (host) { - return host; - } - vespalib::string ip_address = resolve_host_now(host_name); - if (ip_address == host_name) { - return Host::SP(nullptr); - } - return insert_host(host_name, ip_address); -} - -LazyResolver::Address::SP -LazyResolver::make_address(const vespalib::string &spec_str) -{ - SocketSpec spec(spec_str); - if (!spec.valid()) { - LOG(warning, "invalid socket spec: '%s'\n", spec_str.c_str()); - } - return Address::SP(new Address(spec_str, make_host(spec.host()))); -} - -//----------------------------------------------------------------------------- - -vespalib::string -LazyResolver::default_resolve_host(const vespalib::string &host_name) -{ - return SocketAddress::select_remote(80, host_name.c_str()).ip_address(); -} - -std::shared_ptr<LazyResolver> -LazyResolver::create(Params params) -{ - return std::shared_ptr<LazyResolver>(new LazyResolver(std::move(params))); -} - -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/lazy_resolver.h b/vespalib/src/vespa/vespalib/net/lazy_resolver.h deleted file mode 100644 index e740f3de463..00000000000 --- a/vespalib/src/vespa/vespalib/net/lazy_resolver.h +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "socket_address.h" -#include "socket_spec.h" -#include <vespa/vespalib/util/threadstackexecutor.h> -#include <chrono> -#include <memory> -#include <mutex> -#include <map> - -namespace vespalib { - -/** - * Component used to perform lazy re-resolving of host names. The goal - * of this class is to allow applications to (re-)connect from within - * a network thread without stalling everything due to slow dns - * responses while still being able to pick up on dns changes - * (eventually). The idea is that the make_address function is called - * up front during configuration from a non-critical thread. It will - * (potentially) perform an initial synchronous resolve and return an - * Address object that can later be used to obtain a string-based - * connect spec where any host names have been replaced by ip - * addresses without blocking. The host names are resolved under the - * assumption that they will be used to connect to a remote server. - **/ -class LazyResolver : public std::enable_shared_from_this<LazyResolver> -{ -public: - using resolve_host_t = std::function<vespalib::string(const vespalib::string &)>; - using clock = std::chrono::steady_clock; - using seconds = std::chrono::duration<double>; - using SP = std::shared_ptr<LazyResolver>; - - struct Params - { - resolve_host_t resolve_host; - seconds max_result_age; - seconds max_resolve_time; - Params(); - }; - - class Host : public std::enable_shared_from_this<Host> - { - private: - friend class LazyResolver; - vespalib::string _host_name; - LazyResolver::SP _resolver; - std::mutex _ip_lock; - bool _ip_pending; - vespalib::string _ip_address; - clock::time_point _ip_updated; - Host(const vespalib::string &host_name, LazyResolver::SP resolver, - const vespalib::string &ip_address); - void update_ip_address(const vespalib::string &ip_address); - public: - ~Host(); - using SP = std::shared_ptr<Host>; - const vespalib::string &host_name() const { return _host_name; } - vespalib::string resolve(); - }; - - class Address - { - private: - friend class LazyResolver; - vespalib::string _spec; - Host::SP _host; - Address(const vespalib::string &spec, Host::SP host) - : _spec(spec), _host(std::move(host)) {} - public: - using SP = std::shared_ptr<Address>; - const vespalib::string &spec() const { return _spec; } - vespalib::string resolve(); - }; - -private: - struct UpdateTask : Executor::Task { - LazyResolver &resolver; - std::weak_ptr<Host> weak_host; - UpdateTask(LazyResolver &resolver_in, std::weak_ptr<Host> weak_host_in) - : resolver(resolver_in), weak_host(std::move(weak_host_in)) {} - void run() override; - }; - - std::mutex _host_lock; - std::map<vespalib::string, std::weak_ptr<Host> > _host_map; - Params _params; - ThreadStackExecutor _executor; - LazyResolver(Params params); - Host::SP try_lookup_host(const vespalib::string &host_name, - const std::lock_guard<std::mutex> &guard); - Host::SP try_lookup_host(const vespalib::string &host_name); - Host::SP insert_host(const vespalib::string &host_name, const vespalib::string &ip_address); - vespalib::string resolve_host_now(const vespalib::string &host_name); - bool should_request_update(clock::time_point ip_updated); - bool try_request_update(std::weak_ptr<Host> self); -public: - ~LazyResolver(); - void wait_for_pending_updates() { _executor.sync(); } - Host::SP make_host(const vespalib::string &host_name); - Address::SP make_address(const vespalib::string &spec); - static vespalib::string default_resolve_host(const vespalib::string &host_name); - static SP create(Params params = Params()); -}; - -} // namespace vespalib |