Merge pull request #18197 from vespa-engine/arnej/add-report-connectivity-rpc

add rpc method for connectivity report

18 files changed, 270 insertions, 78 deletions

diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt
index 43b4f79a0b2..0323df2864f 100644
--- a/configd/src/apps/sentinel/CMakeLists.txt
+++ b/configd/src/apps/sentinel/CMakeLists.txt
@@ -9,9 +9,11 @@ vespa_add_executable(configd_config-sentinel_app
     line-splitter.cpp
     manager.cpp
     metrics.cpp
+    model-owner.cpp
     output-connection.cpp
     outward-check.cpp
     peer-check.cpp
+    report-connectivity.cpp
     rpchooks.cpp
     rpcserver.cpp
     sentinel.cpp
diff --git a/configd/src/apps/sentinel/config-owner.cpp b/configd/src/apps/sentinel/config-owner.cpp
index 840c5b1add1..665cf07d3e7 100644
--- a/configd/src/apps/sentinel/config-owner.cpp
+++ b/configd/src/apps/sentinel/config-owner.cpp
@@ -10,7 +10,7 @@ LOG_SETUP(".config-owner");
 
 namespace config::sentinel {
 
-ConfigOwner::ConfigOwner() : _subscriber() {}
+ConfigOwner::ConfigOwner() = default;
 
 ConfigOwner::~ConfigOwner() = default;
 
@@ -42,29 +42,4 @@ ConfigOwner::checkForConfigUpdate() {
     return false;
 }
 
-std::unique_ptr<ModelConfig>
-ConfigOwner::fetchModelConfig(std::chrono::milliseconds timeout)
-{
-    std::unique_ptr<ModelConfig> modelConfig;
-    ConfigSubscriber tempSubscriber;
-    try {
-        ConfigHandle<ModelConfig>::UP modelHandle =
-            tempSubscriber.subscribe<ModelConfig>("admin/model", timeout);
-        if (tempSubscriber.nextGenerationNow()) {
-            modelConfig = modelHandle->getConfig();
-            LOG(config, "Sentinel got model info [version %s] for %zd hosts [config generation %" PRId64 "]",
-                modelConfig->vespaVersion.c_str(), modelConfig->hosts.size(),
-                tempSubscriber.getGeneration());
-        }
-    } catch (ConfigTimeoutException & ex) {
-        LOG(warning, "Timeout getting model config: %s [skipping connectivity checks]", ex.getMessage().c_str());
-    } catch (InvalidConfigException& ex) {
-        LOG(warning, "Invalid model config: %s [skipping connectivity checks]", ex.getMessage().c_str());
-    } catch (ConfigRuntimeException& ex) {
-        LOG(warning, "Runtime exception getting model config: %s [skipping connectivity checks]", ex.getMessage().c_str());
-
-    }
-    return modelConfig;
-}
-
 }
diff --git a/configd/src/apps/sentinel/config-owner.h b/configd/src/apps/sentinel/config-owner.h
index 2850e6b3904..b72aed59271 100644
--- a/configd/src/apps/sentinel/config-owner.h
+++ b/configd/src/apps/sentinel/config-owner.h
@@ -7,10 +7,6 @@
 #include <vespa/config/config.h>
 
 using cloud::config::SentinelConfig;
-using cloud::config::ModelConfig;
-
-using config::ConfigSubscriber;
-using config::ConfigHandle;
 
 namespace config::sentinel {
 
@@ -19,9 +15,9 @@ namespace config::sentinel {
  **/
 class ConfigOwner {
 private:
-    ConfigSubscriber _subscriber;
-    ConfigHandle<SentinelConfig>::UP _sentinelHandle;
-
+    config::ConfigSubscriber _subscriber;
+    config::ConfigHandle<SentinelConfig>::UP _sentinelHandle;
+    
     int64_t _currGeneration = -1;
     std::unique_ptr<SentinelConfig> _currConfig;
 
@@ -37,7 +33,6 @@ public:
     bool hasConfig() const { return _currConfig.get() != nullptr; }
     const SentinelConfig& getConfig() const { return *_currConfig; }
     int64_t getGeneration() const { return _currGeneration; }
-    static std::unique_ptr<ModelConfig> fetchModelConfig(std::chrono::milliseconds timeout);
 };
 
 }
diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp
index 0e91b583868..c1c49e3068a 100644
--- a/configd/src/apps/sentinel/connectivity.cpp
+++ b/configd/src/apps/sentinel/connectivity.cpp
@@ -17,8 +17,6 @@ using namespace std::chrono_literals;
 
 namespace config::sentinel {
 
-constexpr std::chrono::milliseconds MODEL_TIMEOUT_MS = 60s;
-
 Connectivity::Connectivity() = default;
 Connectivity::~Connectivity() = default;
 
@@ -45,28 +43,6 @@ std::string spec(const SpecMap::value_type &host_and_port) {
     return fmt("tcp/%s:%d", host_and_port.first.c_str(), host_and_port.second);
 }
 
-SpecMap specsFrom(const ModelConfig &model) {
-    SpecMap checkSpecs;
-    for (const auto & h : model.hosts) {
-        bool foundSentinelPort = false;
-        for (const auto & s : h.services) {
-            if (s.name == "config-sentinel") {
-                for (const auto & p : s.ports) {
-                    if (p.tags.find("rpc") != p.tags.npos) {
-                        checkSpecs[h.name] = p.number;
-                        foundSentinelPort = true;
-                    }
-                }
-            }
-        }
-        if (! foundSentinelPort) {
-            LOG(warning, "Did not find 'config-sentinel' RPC port in model for host %s [%zd services]",
-                h.name.c_str(), h.services.size());
-        }
-    }
-    return checkSpecs;
-}
-
 void classifyConnFails(ConnectivityMap &connectivityMap,
                        const SpecMap &specMap,
                        RpcServer &rpcServer)
@@ -118,13 +94,35 @@ void classifyConnFails(ConnectivityMap &connectivityMap,
 
 } // namespace <unnamed>
 
-void Connectivity::configure(const SentinelConfig::Connectivity &config) {
+SpecMap Connectivity::specsFrom(const ModelConfig &model) {
+    SpecMap checkSpecs;
+    for (const auto & h : model.hosts) {
+        bool foundSentinelPort = false;
+        for (const auto & s : h.services) {
+            if (s.name == "config-sentinel") {
+                for (const auto & p : s.ports) {
+                    if (p.tags.find("rpc") != p.tags.npos) {
+                        checkSpecs[h.name] = p.number;
+                        foundSentinelPort = true;
+                    }
+                }
+            }
+        }
+        if (! foundSentinelPort) {
+            LOG(warning, "Did not find 'config-sentinel' RPC port in model for host %s [%zd services]",
+                h.name.c_str(), h.services.size());
+        }
+    }
+    return checkSpecs;
+}
+
+void Connectivity::configure(const SentinelConfig::Connectivity &config,
+                             const ModelConfig &model)
+{
     _config = config;
     LOG(config, "connectivity.maxBadCount = %d", _config.maxBadCount);
     LOG(config, "connectivity.minOkPercent = %d", _config.minOkPercent);
-    if (auto up = ConfigOwner::fetchModelConfig(MODEL_TIMEOUT_MS)) {
-        _checkSpecs = specsFrom(*up);
-    }
+    _checkSpecs = specsFrom(model);
 }
 
 bool
diff --git a/configd/src/apps/sentinel/connectivity.h b/configd/src/apps/sentinel/connectivity.h
index c1e41249dd5..8d923387ffa 100644
--- a/configd/src/apps/sentinel/connectivity.h
+++ b/configd/src/apps/sentinel/connectivity.h
@@ -24,8 +24,10 @@ public:
 
     Connectivity();
     ~Connectivity();
-    void configure(const SentinelConfig::Connectivity &config);
+    void configure(const SentinelConfig::Connectivity &config,
+                   const ModelConfig &model);
     bool checkConnectivity(RpcServer &rpcServer);
+    static SpecMap specsFrom(const ModelConfig &model);
 private:
     class Accumulator {
     private:
diff --git a/configd/src/apps/sentinel/env.cpp b/configd/src/apps/sentinel/env.cpp
index 5bbbfd8f0bd..f857f95a496 100644
--- a/configd/src/apps/sentinel/env.cpp
+++ b/configd/src/apps/sentinel/env.cpp
@@ -36,6 +36,7 @@ constexpr int maxConnectivityRetries = 100;
 
 Env::Env()
   : _cfgOwner(),
+    _modelOwner("admin/model"),
     _rpcCommandQueue(),
     _rpcServer(),
     _stateApi(),
@@ -52,6 +53,7 @@ Env::~Env() = default;
 void Env::boot(const std::string &configId) {
     LOG(debug, "Reading configuration for ID: %s", configId.c_str());
     _cfgOwner.subscribe(configId, CONFIG_TIMEOUT_MS);
+    _modelOwner.start(CONFIG_TIMEOUT_MS, true);
     // subscribe() should throw if something is not OK
     Connectivity checker;
     for (int retry = 0; retry < maxConnectivityRetries; ++retry) {
@@ -64,7 +66,11 @@ void Env::boot(const std::string &configId) {
                 configId.c_str(), cfg.port.telnet, cfg.port.rpc);
             rpcPort(cfg.port.rpc);
             statePort(cfg.port.telnet);
-            checker.configure(cfg.connectivity);
+            _modelOwner.checkForUpdates();
+            auto model = _modelOwner.getModelConfig();
+            if (model.has_value()) {
+                checker.configure(cfg.connectivity, model.value());
+            }
         }
         if (checker.checkConnectivity(*_rpcServer)) {
             _stateApi.myHealth.setOk();
@@ -94,7 +100,7 @@ void Env::rpcPort(int port) {
     if (_rpcServer && port == _rpcServer->getPort()) {
         return; // ok already
     }
-    _rpcServer = std::make_unique<RpcServer>(port, _rpcCommandQueue);
+    _rpcServer = std::make_unique<RpcServer>(port, _rpcCommandQueue, _modelOwner);
 }
 
 void Env::statePort(int port) {
@@ -116,7 +122,6 @@ void Env::statePort(int port) {
 void Env::notifyConfigUpdated() {
     vespalib::ComponentConfigProducer::Config current("sentinel", _cfgOwner.getGeneration(), "ok");
     _stateApi.myComponents.addConfig(current);
-
 }
 
 void Env::respondAsEmpty() {
diff --git a/configd/src/apps/sentinel/env.h b/configd/src/apps/sentinel/env.h
index f71fb537068..1bd3a7380ba 100644
--- a/configd/src/apps/sentinel/env.h
+++ b/configd/src/apps/sentinel/env.h
@@ -5,6 +5,7 @@
 #include "cmdq.h"
 #include "config-owner.h"
 #include "metrics.h"
+#include "model-owner.h"
 #include "rpcserver.h"
 #include "state-api.h"
 #include <vespa/vespalib/net/state_server.h>
@@ -22,6 +23,7 @@ public:
     ~Env();
 
     ConfigOwner &configOwner() { return _cfgOwner; }
+    ModelOwner &modelOwner() { return _modelOwner; }
     CommandQueue &commandQueue() { return _rpcCommandQueue; }
     StartMetrics &metrics() { return _startMetrics; }
 
@@ -33,6 +35,7 @@ public:
 private:
     void respondAsEmpty();
     ConfigOwner _cfgOwner;
+    ModelOwner _modelOwner;
     CommandQueue _rpcCommandQueue;
     std::unique_ptr<RpcServer> _rpcServer;
     StateApi _stateApi;
diff --git a/configd/src/apps/sentinel/manager.cpp b/configd/src/apps/sentinel/manager.cpp
index 6e0ed78211c..0c13292d704 100644
--- a/configd/src/apps/sentinel/manager.cpp
+++ b/configd/src/apps/sentinel/manager.cpp
@@ -116,6 +116,7 @@ Manager::doWork()
     if (_env.configOwner().checkForConfigUpdate()) {
         doConfigure();
     }
+    _env.modelOwner().checkForUpdates();
     handleRestarts();
     handleCommands();
     handleOutputs();
diff --git a/configd/src/apps/sentinel/model-owner.cpp b/configd/src/apps/sentinel/model-owner.cpp
new file mode 100644
index 00000000000..cfa9f1f6bf5
--- /dev/null
+++ b/configd/src/apps/sentinel/model-owner.cpp
@@ -0,0 +1,66 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "model-owner.h"
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/config/common/exceptions.h>
+#include <string>
+#include <chrono>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".sentinel.model-owner");
+
+using namespace std::chrono_literals;
+
+namespace config::sentinel {
+
+std::optional<ModelConfig> ModelOwner::getModelConfig() {
+    std::lock_guard<std::mutex> guard(_lock);
+    if (_modelConfig) {
+        return ModelConfig(*_modelConfig);
+    } else {
+        return {};
+    }
+}
+
+
+ModelOwner::ModelOwner(const std::string &configId)
+  : _configId(configId)
+{}
+
+ModelOwner::~ModelOwner() = default;
+
+void
+ModelOwner::start(std::chrono::milliseconds timeout, bool firstTime) {
+    try {
+        _modelHandle =_subscriber.subscribe<ModelConfig>(_configId, timeout);
+    } catch (ConfigTimeoutException & ex) {
+        if (firstTime) {
+            LOG(warning, "Timeout getting model config: %s [skipping connectivity checks]", ex.message());
+        }
+    } catch (InvalidConfigException& ex) {
+        if (firstTime) {
+            LOG(warning, "Invalid model config: %s [skipping connectivity checks]", ex.message());
+        }
+    } catch (ConfigRuntimeException& ex) {
+        if (firstTime) {
+            LOG(warning, "Runtime exception getting model config: %s [skipping connectivity checks]", ex.message());
+        }
+    }
+}
+
+void
+ModelOwner::checkForUpdates() {
+    if (! _modelHandle) {
+        start(250ms, false);
+    }
+    if (_modelHandle && _subscriber.nextGenerationNow()) {
+        if (auto newModel = _modelHandle->getConfig()) {
+            LOG(config, "Sentinel got model info [version %s] for %zd hosts [config generation %" PRId64 "]",
+                newModel->vespaVersion.c_str(), newModel->hosts.size(), _subscriber.getGeneration());
+            std::lock_guard<std::mutex> guard(_lock);
+            _modelConfig = std::move(newModel);
+        }
+    }
+}
+
+}
diff --git a/configd/src/apps/sentinel/model-owner.h b/configd/src/apps/sentinel/model-owner.h
new file mode 100644
index 00000000000..0513463e955
--- /dev/null
+++ b/configd/src/apps/sentinel/model-owner.h
@@ -0,0 +1,32 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/config-model.h>
+#include <vespa/config/config.h>
+#include <optional>
+#include <mutex>
+
+using cloud::config::ModelConfig;
+
+namespace config::sentinel {
+
+/**
+ * Handles config subscription and has a snapshot of current config.
+ **/
+class ModelOwner {
+private:
+    std::string _configId;
+    config::ConfigSubscriber _subscriber;
+    config::ConfigHandle<ModelConfig>::UP _modelHandle;
+    std::mutex _lock;
+    std::unique_ptr<ModelConfig> _modelConfig;
+public:
+    ModelOwner(const std::string &configId);
+    ~ModelOwner();
+    void start(std::chrono::milliseconds timeout, bool firstTime);
+    void checkForUpdates();
+    std::optional<ModelConfig> getModelConfig();
+};
+
+}
diff --git a/configd/src/apps/sentinel/peer-check.cpp b/configd/src/apps/sentinel/peer-check.cpp
index 60c3d9c96c9..daab80eefc7 100644
--- a/configd/src/apps/sentinel/peer-check.cpp
+++ b/configd/src/apps/sentinel/peer-check.cpp
@@ -15,7 +15,8 @@ PeerCheck::PeerCheck(StatusCallback &callback, const std::string &host, int port
     _hostname(host),
     _portnum(port),
     _target(nullptr),
-    _req(nullptr)
+    _req(nullptr),
+    _statusOk(false)
 {
     auto spec = fmt("tcp/%s:%d", _hostname.c_str(), _portnum);
     _target = orb.GetTarget(spec.c_str());
@@ -31,20 +32,19 @@ PeerCheck::~PeerCheck() {
 
 void PeerCheck::RequestDone(FRT_RPCRequest *req) {
     LOG_ASSERT(req == _req);
-    bool statusOk = false;
     if (req->IsError()) {
-        LOG(warning, "error on ping to %s [port %d]: %s (%d)", _hostname.c_str(), _portnum,
+        LOG(debug, "error on ping to %s [port %d]: %s (%d)", _hostname.c_str(), _portnum,
             req->GetErrorMessage(), req->GetErrorCode());
     } else {
         LOG(debug, "OK ping to %s [port %d]", _hostname.c_str(), _portnum);
-        statusOk = true;
+        _statusOk = true;
     }
     _req->SubRef();
     _req = nullptr;
     _target->SubRef();
     _target = nullptr;
     // Note: will delete this object, so must be called as final step:
-    _callback.returnStatus(statusOk);
+    _callback.returnStatus(_statusOk);
 }
 
 }
diff --git a/configd/src/apps/sentinel/peer-check.h b/configd/src/apps/sentinel/peer-check.h
index 096f304467b..ac124106387 100644
--- a/configd/src/apps/sentinel/peer-check.h
+++ b/configd/src/apps/sentinel/peer-check.h
@@ -17,6 +17,9 @@ public:
     PeerCheck(StatusCallback &callback, const std::string &host, int portnum, FRT_Supervisor &orb, int timeout_ms);
     ~PeerCheck();
 
+    bool okStatus() const { return _statusOk; }
+    const std::string& getHostname() const { return _hostname; }
+
     PeerCheck(const PeerCheck &) = delete;
     PeerCheck(PeerCheck &&) = delete;
     PeerCheck& operator= (const PeerCheck &) = delete;
@@ -30,6 +33,7 @@ private:
     int             _portnum;
     FRT_Target     *_target;
     FRT_RPCRequest *_req;
+    bool            _statusOk;
 };
 
 }
diff --git a/configd/src/apps/sentinel/report-connectivity.cpp b/configd/src/apps/sentinel/report-connectivity.cpp
new file mode 100644
index 00000000000..c1e519a4a9a
--- /dev/null
+++ b/configd/src/apps/sentinel/report-connectivity.cpp
@@ -0,0 +1,53 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "report-connectivity.h"
+#include "connectivity.h"
+#include <vespa/config/common/exceptions.h>
+#include <vespa/log/log.h>
+#include <chrono>
+
+LOG_SETUP(".report-connectivity");
+
+using cloud::config::ModelConfig;
+using namespace std::chrono_literals;
+
+namespace config::sentinel {
+
+ReportConnectivity::ReportConnectivity(FRT_RPCRequest *req, int timeout_ms, FRT_Supervisor &orb, ModelOwner &modelOwner)
+  : _parentRequest(req),
+    _checks()
+{
+    auto cfg = modelOwner.getModelConfig();
+    if (cfg.has_value()) {
+        auto map = Connectivity::specsFrom(cfg.value());
+        LOG(debug, "making connectivity report for %zd peers", map.size());
+        _remaining = map.size();
+        for (const auto & [ hostname, port ] : map) {
+            _checks.emplace_back(std::make_unique<PeerCheck>(*this, hostname, port, orb, timeout_ms));
+        }
+    } else {
+        _parentRequest->SetError(FRTE_RPC_METHOD_FAILED, "failed getting model config");
+        _parentRequest->Return();
+    }
+}
+
+ReportConnectivity::~ReportConnectivity() = default;
+
+void ReportConnectivity::returnStatus(bool) {
+    if (--_remaining == 0) {
+        finish();
+    }
+}
+
+void ReportConnectivity::finish() const {
+    FRT_Values *dst = _parentRequest->GetReturn();
+    FRT_StringValue *pt_hn = dst->AddStringArray(_checks.size());
+    FRT_StringValue *pt_ss = dst->AddStringArray(_checks.size());
+    for (const auto & peer : _checks) {
+        dst->SetString(pt_hn++, peer->getHostname().c_str());
+        dst->SetString(pt_ss++, peer->okStatus() ? "ok" : "ping failed");
+    }
+    _parentRequest->Return();
+}
+
+}
diff --git a/configd/src/apps/sentinel/report-connectivity.h b/configd/src/apps/sentinel/report-connectivity.h
new file mode 100644
index 00000000000..1f243b73028
--- /dev/null
+++ b/configd/src/apps/sentinel/report-connectivity.h
@@ -0,0 +1,33 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/fnet/frt/rpcrequest.h>
+#include <vespa/fnet/frt/supervisor.h>
+#include <vespa/config-model.h>
+#include <vespa/config/helper/configfetcher.h>
+#include "model-owner.h"
+#include "peer-check.h"
+#include "status-callback.h"
+
+#include <atomic>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace config::sentinel {
+
+class ReportConnectivity : public StatusCallback
+{
+public:
+    ReportConnectivity(FRT_RPCRequest *req, int timeout_ms, FRT_Supervisor &orb, ModelOwner &modelOwner);
+    virtual ~ReportConnectivity();
+    void returnStatus(bool ok) override;
+private:
+    void finish() const;
+    FRT_RPCRequest *_parentRequest;
+    std::vector<std::unique_ptr<PeerCheck>> _checks;
+    std::atomic<size_t> _remaining;
+};
+
+}
diff --git a/configd/src/apps/sentinel/rpchooks.cpp b/configd/src/apps/sentinel/rpchooks.cpp
index 24e3cd53509..0d49063db84 100644
--- a/configd/src/apps/sentinel/rpchooks.cpp
+++ b/configd/src/apps/sentinel/rpchooks.cpp
@@ -1,9 +1,10 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "rpchooks.h"
-#include "cmdq.h"
 #include "check-completion-handler.h"
+#include "cmdq.h"
 #include "peer-check.h"
+#include "report-connectivity.h"
 #include <vespa/fnet/frt/supervisor.h>
 #include <vespa/fnet/frt/rpcrequest.h>
 
@@ -12,9 +13,10 @@ LOG_SETUP(".rpchooks");
 
 namespace config::sentinel {
 
-RPCHooks::RPCHooks(CommandQueue &commands, FRT_Supervisor &supervisor)
+RPCHooks::RPCHooks(CommandQueue &commands, FRT_Supervisor &supervisor, ModelOwner &modelOwner)
   : _commands(commands),
-    _orb(supervisor)
+    _orb(supervisor),
+    _modelOwner(modelOwner)
 {
     initRPC(&_orb);
 }
@@ -53,6 +55,13 @@ RPCHooks::initRPC(FRT_Supervisor *supervisor)
     rb.ParamDesc("timeout", "Timeout for check in milliseconds");
     rb.ReturnDesc("status", "Status (ok, bad, or unknown) for peer");
     //-------------------------------------------------------------------------
+    rb.DefineMethod("sentinel.report.connectivity", "i", "SS",
+                    FRT_METHOD(RPCHooks::rpc_reportConnectivity), this);
+    rb.MethodDesc("report connectivity for peer sentinels");
+    rb.ParamDesc("timeout", "Timeout for check in milliseconds");
+    rb.ReturnDesc("hostnames", "Names of peers checked");
+    rb.ReturnDesc("peerstatus", "Status description for each peer");
+    //-------------------------------------------------------------------------
 }
 
 void
@@ -106,4 +115,14 @@ RPCHooks::rpc_checkConnectivity(FRT_RPCRequest *req)
     req->getStash().create<PeerCheck>(completionHandler, hostname, portnum, _orb, timeout);
 }
 
+void
+RPCHooks::rpc_reportConnectivity(FRT_RPCRequest *req)
+{
+    LOG(debug, "got reportConnectivity");
+    FRT_Values &args  = *req->GetParams();
+    int timeout = args[0]._intval32;
+    req->Detach();
+    req->getStash().create<ReportConnectivity>(req, timeout, _orb, _modelOwner);
+}
+
 } // namespace slobrok
diff --git a/configd/src/apps/sentinel/rpchooks.h b/configd/src/apps/sentinel/rpchooks.h
index 67f5804dcf7..292e8198b55 100644
--- a/configd/src/apps/sentinel/rpchooks.h
+++ b/configd/src/apps/sentinel/rpchooks.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include "model-owner.h"
 #include <vespa/fnet/frt/invokable.h>
 #include <memory>
 
@@ -25,8 +26,9 @@ class RPCHooks : public FRT_Invokable
 private:
     CommandQueue &_commands;
     FRT_Supervisor &_orb;
+    ModelOwner &_modelOwner;
 public:
-    RPCHooks(CommandQueue &commands, FRT_Supervisor &supervisor);
+    RPCHooks(CommandQueue &commands, FRT_Supervisor &supervisor, ModelOwner &modelOwner);
     ~RPCHooks() override;
 private:
     void initRPC(FRT_Supervisor *supervisor);
@@ -36,6 +38,7 @@ private:
     void rpc_stopService(FRT_RPCRequest *req);
     void rpc_startService(FRT_RPCRequest *req);
     void rpc_checkConnectivity(FRT_RPCRequest *req);
+    void rpc_reportConnectivity(FRT_RPCRequest *req);
 };
 
 } // namespace config::sentinel
diff --git a/configd/src/apps/sentinel/rpcserver.cpp b/configd/src/apps/sentinel/rpcserver.cpp
index 80c3c81c826..be867ae95db 100644
--- a/configd/src/apps/sentinel/rpcserver.cpp
+++ b/configd/src/apps/sentinel/rpcserver.cpp
@@ -7,9 +7,9 @@ LOG_SETUP(".rpcserver");
 
 namespace config::sentinel {
 
-RpcServer::RpcServer(int portNumber, CommandQueue &cmdQ)
+RpcServer::RpcServer(int portNumber, CommandQueue &cmdQ, ModelOwner &modelOwner)
     : _server(),
-      _rpcHooks(cmdQ, _server.supervisor()),
+      _rpcHooks(cmdQ, _server.supervisor(), modelOwner),
       _port(portNumber)
 {
     if (_server.supervisor().Listen(portNumber)) {
diff --git a/configd/src/apps/sentinel/rpcserver.h b/configd/src/apps/sentinel/rpcserver.h
index 4c6dea00ddf..8f60acce1ca 100644
--- a/configd/src/apps/sentinel/rpcserver.h
+++ b/configd/src/apps/sentinel/rpcserver.h
@@ -5,6 +5,7 @@
 #include <memory>
 
 #include "cmdq.h"
+#include "model-owner.h"
 #include "rpchooks.h"
 #include <vespa/fnet/frt/supervisor.h>
 
@@ -18,7 +19,7 @@ private:
     int _port;
 
 public:
-    RpcServer(int port, CommandQueue &cmdQ);
+    RpcServer(int port, CommandQueue &cmdQ, ModelOwner &modelOwner);
     ~RpcServer();
 
     int getPort() const { return _port; }