add rpc method for connectivity report

8 files changed, 180 insertions, 24 deletions

diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt
index 43b4f79a0b2..0999314c8ce 100644
--- a/configd/src/apps/sentinel/CMakeLists.txt
+++ b/configd/src/apps/sentinel/CMakeLists.txt
@@ -12,6 +12,7 @@ vespa_add_executable(configd_config-sentinel_app
     output-connection.cpp
     outward-check.cpp
     peer-check.cpp
+    report-connectivity.cpp
     rpchooks.cpp
     rpcserver.cpp
     sentinel.cpp
diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp
index 8d1aa0e9673..12b645dd589 100644
--- a/configd/src/apps/sentinel/connectivity.cpp
+++ b/configd/src/apps/sentinel/connectivity.cpp
@@ -45,28 +45,6 @@ std::string spec(const SpecMap::value_type &host_and_port) {
     return fmt("tcp/%s:%d", host_and_port.first.c_str(), host_and_port.second);
 }
 
-SpecMap specsFrom(const ModelConfig &model) {
-    SpecMap checkSpecs;
-    for (const auto & h : model.hosts) {
-        bool foundSentinelPort = false;
-        for (const auto & s : h.services) {
-            if (s.name == "config-sentinel") {
-                for (const auto & p : s.ports) {
-                    if (p.tags.find("rpc") != p.tags.npos) {
-                        checkSpecs[h.name] = p.number;
-                        foundSentinelPort = true;
-                    }
-                }
-            }
-        }
-        if (! foundSentinelPort) {
-            LOG(warning, "Did not find 'config-sentinel' RPC port in model for host %s [%zd services]",
-                h.name.c_str(), h.services.size());
-        }
-    }
-    return checkSpecs;
-}
-
 void classifyConnFails(ConnectivityMap &connectivityMap,
                        const SpecMap &specMap,
                        RpcServer &rpcServer)
@@ -118,6 +96,28 @@ void classifyConnFails(ConnectivityMap &connectivityMap,
 
 } // namespace <unnamed>
 
+SpecMap Connectivity::specsFrom(const ModelConfig &model) {
+    SpecMap checkSpecs;
+    for (const auto & h : model.hosts) {
+        bool foundSentinelPort = false;
+        for (const auto & s : h.services) {
+            if (s.name == "config-sentinel") {
+                for (const auto & p : s.ports) {
+                    if (p.tags.find("rpc") != p.tags.npos) {
+                        checkSpecs[h.name] = p.number;
+                        foundSentinelPort = true;
+                    }
+                }
+            }
+        }
+        if (! foundSentinelPort) {
+            LOG(warning, "Did not find 'config-sentinel' RPC port in model for host %s [%zd services]",
+                h.name.c_str(), h.services.size());
+        }
+    }
+    return checkSpecs;
+}
+
 void Connectivity::configure(const SentinelConfig::Connectivity &config) {
     _config = config;
     LOG(config, "connectivity.maxBadReverseCount = %d", _config.maxBadReverseCount);
diff --git a/configd/src/apps/sentinel/connectivity.h b/configd/src/apps/sentinel/connectivity.h
index a1e454a255a..440d7df84c0 100644
--- a/configd/src/apps/sentinel/connectivity.h
+++ b/configd/src/apps/sentinel/connectivity.h
@@ -26,6 +26,7 @@ public:
     ~Connectivity();
     void configure(const SentinelConfig::Connectivity &config);
     bool checkConnectivity(RpcServer &rpcServer);
+    static SpecMap specsFrom(const ModelConfig &model);
 private:
     struct Accumulated {
         size_t numIssues = 0;
diff --git a/configd/src/apps/sentinel/peer-check.cpp b/configd/src/apps/sentinel/peer-check.cpp
index 60c3d9c96c9..46196f36bf1 100644
--- a/configd/src/apps/sentinel/peer-check.cpp
+++ b/configd/src/apps/sentinel/peer-check.cpp
@@ -33,7 +33,7 @@ void PeerCheck::RequestDone(FRT_RPCRequest *req) {
     LOG_ASSERT(req == _req);
     bool statusOk = false;
     if (req->IsError()) {
-        LOG(warning, "error on ping to %s [port %d]: %s (%d)", _hostname.c_str(), _portnum,
+        LOG(debug, "error on ping to %s [port %d]: %s (%d)", _hostname.c_str(), _portnum,
             req->GetErrorMessage(), req->GetErrorCode());
     } else {
         LOG(debug, "OK ping to %s [port %d]", _hostname.c_str(), _portnum);
diff --git a/configd/src/apps/sentinel/report-connectivity.cpp b/configd/src/apps/sentinel/report-connectivity.cpp
new file mode 100644
index 00000000000..8e855f296c1
--- /dev/null
+++ b/configd/src/apps/sentinel/report-connectivity.cpp
@@ -0,0 +1,71 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "report-connectivity.h"
+#include "connectivity.h"
+#include <vespa/log/log.h>
+
+LOG_SETUP(".report-connectivity");
+
+using cloud::config::ModelConfig;
+
+namespace config::sentinel {
+
+ConnectivityCheckResult::~ConnectivityCheckResult() = default;
+
+void ConnectivityCheckResult::returnStatus(bool ok) {
+    status = ok ? "ok" : "ping failed";
+    LOG(debug, "peer %s [port %d] -> %s", peerName.c_str(), peerPort, status.c_str());
+    parent.requestDone();
+}
+
+ConnectivityReportResult::~ConnectivityReportResult() = default;
+
+
+ReportConnectivity::ReportConnectivity(FRT_RPCRequest *req, FRT_Supervisor &orb)
+  : _parentRequest(req),
+    _orb(orb),
+    _result(),
+    _configFetcher()
+{
+    _configFetcher.subscribe<ModelConfig>("admin/model", this);
+    _configFetcher.start();
+}
+
+ReportConnectivity::~ReportConnectivity() = default;
+
+void ReportConnectivity::requestDone() {
+    {
+        std::lock_guard<std::mutex> guard(_lock);
+        if (--_remaining != 0) {
+            return;
+        }
+    }
+    finish();
+}
+
+void ReportConnectivity::configure(std::unique_ptr<ModelConfig> config) {
+    _configFetcher.close();
+    auto map = Connectivity::specsFrom(*config);
+    for (const auto & [ hostname, port ] : map) {
+        _result.peers.emplace_back(*this, hostname, port);
+    }
+    LOG(debug, "making connectivity report for %zd peers", _result.peers.size());
+    _remaining = _result.peers.size();
+    for (auto & peer : _result.peers) {
+        peer.check = std::make_unique<PeerCheck>(peer, peer.peerName, peer.peerPort, _orb, 2500);
+    }
+}
+
+
+void ReportConnectivity::finish() const {
+    FRT_Values *dst = _parentRequest->GetReturn();
+    FRT_StringValue *pt_hn = dst->AddStringArray(_result.peers.size());
+    FRT_StringValue *pt_ss = dst->AddStringArray(_result.peers.size());
+    for (const auto & peer : _result.peers) {
+        dst->SetString(pt_hn++, peer.peerName.c_str());
+        dst->SetString(pt_ss++, peer.status.c_str());
+    }
+    _parentRequest->Return();
+}
+
+}
diff --git a/configd/src/apps/sentinel/report-connectivity.h b/configd/src/apps/sentinel/report-connectivity.h
new file mode 100644
index 00000000000..ac62627fe41
--- /dev/null
+++ b/configd/src/apps/sentinel/report-connectivity.h
@@ -0,0 +1,67 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fnet/frt/rpcrequest.h>
+#include <vespa/fnet/frt/supervisor.h>
+#include <vespa/config-model.h>
+#include <vespa/config/helper/configfetcher.h>
+#include "peer-check.h"
+#include "status-callback.h"
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace config::sentinel {
+
+class ReportConnectivity;
+
+struct ConnectivityCheckResult : StatusCallback {
+    ReportConnectivity& parent;
+    std::string peerName;
+    int peerPort;
+    std::string status;
+    std::unique_ptr<PeerCheck> check;
+
+    ConnectivityCheckResult(ReportConnectivity& owner, const std::string &hostname, int port)
+      : parent(owner),
+        peerName(hostname),
+        peerPort(port),
+        status("unknown"),
+        check(nullptr)
+    {}
+
+    ConnectivityCheckResult(ConnectivityCheckResult &&) = default;
+    ConnectivityCheckResult(const ConnectivityCheckResult &) = default;
+
+    virtual ~ConnectivityCheckResult();
+    void returnStatus(bool ok) override;
+};
+
+struct ConnectivityReportResult {
+    std::vector<ConnectivityCheckResult> peers;
+    ~ConnectivityReportResult();
+};
+
+
+class ReportConnectivity : public config::IFetcherCallback<cloud::config::ModelConfig>
+{
+public:
+    ReportConnectivity(FRT_RPCRequest *req, FRT_Supervisor &orb);
+    ~ReportConnectivity();
+    void requestDone();
+    /** from IFetcherCallback */
+    void configure(std::unique_ptr<cloud::config::ModelConfig> config) override;
+private:
+    void finish() const;
+    FRT_RPCRequest *_parentRequest;
+    FRT_Supervisor &_orb;
+    ConnectivityReportResult _result;
+    config::ConfigFetcher _configFetcher;
+    std::mutex _lock;
+    size_t _remaining;
+};
+
+}
diff --git a/configd/src/apps/sentinel/rpchooks.cpp b/configd/src/apps/sentinel/rpchooks.cpp
index 24e3cd53509..98386beac3c 100644
--- a/configd/src/apps/sentinel/rpchooks.cpp
+++ b/configd/src/apps/sentinel/rpchooks.cpp
@@ -1,9 +1,10 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "rpchooks.h"
-#include "cmdq.h"
 #include "check-completion-handler.h"
+#include "cmdq.h"
 #include "peer-check.h"
+#include "report-connectivity.h"
 #include <vespa/fnet/frt/supervisor.h>
 #include <vespa/fnet/frt/rpcrequest.h>
 
@@ -53,6 +54,12 @@ RPCHooks::initRPC(FRT_Supervisor *supervisor)
     rb.ParamDesc("timeout", "Timeout for check in milliseconds");
     rb.ReturnDesc("status", "Status (ok, bad, or unknown) for peer");
     //-------------------------------------------------------------------------
+    rb.DefineMethod("sentinel.report.connectivity", "", "SS",
+                    FRT_METHOD(RPCHooks::rpc_reportConnectivity), this);
+    rb.MethodDesc("report connectivity for peer sentinels");
+    rb.ReturnDesc("hostnames", "Names of peers checked");
+    rb.ReturnDesc("peerstatus", "Status description for each peer");
+    //-------------------------------------------------------------------------
 }
 
 void
@@ -106,4 +113,12 @@ RPCHooks::rpc_checkConnectivity(FRT_RPCRequest *req)
     req->getStash().create<PeerCheck>(completionHandler, hostname, portnum, _orb, timeout);
 }
 
+void
+RPCHooks::rpc_reportConnectivity(FRT_RPCRequest *req)
+{
+    LOG(debug, "got reportConnectivity");
+    req->Detach();
+    req->getStash().create<ReportConnectivity>(req, _orb);
+}
+
 } // namespace slobrok
diff --git a/configd/src/apps/sentinel/rpchooks.h b/configd/src/apps/sentinel/rpchooks.h
index 67f5804dcf7..5b6cf878f26 100644
--- a/configd/src/apps/sentinel/rpchooks.h
+++ b/configd/src/apps/sentinel/rpchooks.h
@@ -36,6 +36,7 @@ private:
     void rpc_stopService(FRT_RPCRequest *req);
     void rpc_startService(FRT_RPCRequest *req);
     void rpc_checkConnectivity(FRT_RPCRequest *req);
+    void rpc_reportConnectivity(FRT_RPCRequest *req);
 };
 
 } // namespace config::sentinel