Match engine should be up if cluster state for any bucketspace says

that node is up.

6 files changed, 26 insertions, 9 deletions

diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp
index 5b50ec49844..3840cfb7869 100644
--- a/searchcore/src/apps/tests/persistenceconformance_test.cpp
+++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp
@@ -286,7 +286,7 @@ protected:
 class MyPersistenceEngineOwner : public IPersistenceEngineOwner
 {
     virtual void
-    setClusterState(const storage::spi::ClusterState &calc) override
+    setClusterState(BucketSpace, const storage::spi::ClusterState &calc) override
     {
         (void) calc;
     }
diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
index 4b73e4ca115..a016a56edf7 100644
--- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
+++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
@@ -393,7 +393,7 @@ HandlerSet::prepareGetModifiedBuckets()
 
 class SimplePersistenceEngineOwner : public IPersistenceEngineOwner
 {
-    void setClusterState(const storage::spi::ClusterState &calc) override { (void) calc; }
+    void setClusterState(BucketSpace, const storage::spi::ClusterState &calc) override { (void) calc; }
 };
 
 struct SimpleResourceWriteFilter : public IResourceWriteFilter
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h
index e8ca0bdce9b..bf07aaa82d8 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h
@@ -16,7 +16,7 @@ public:
     }
 
     virtual void
-    setClusterState(const storage::spi::ClusterState &calc) = 0;
+    setClusterState(document::BucketSpace bucketSpace, const storage::spi::ClusterState &calc) = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
index e2b389fb898..0950a596f5b 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
@@ -285,7 +285,7 @@ PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState &
         handler->handleSetClusterState(calc, resultHandler);
     }
     resultHandler.await();
-    _owner.setClusterState(calc);
+    _owner.setClusterState(bucketSpace, calc);
     return resultHandler.getResult();
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 533d0737ca2..799aafae25b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -201,7 +201,9 @@ Proton::Proton(const config::ConfigUri & configUri,
       _initStarted(false),
       _initComplete(false),
       _initDocumentDbsInSequence(false),
-      _documentDBReferenceRegistry()
+      _documentDBReferenceRegistry(),
+      _nodeUpLock(),
+      _nodeUp()
 {
     _documentDBReferenceRegistry = std::make_shared<DocumentDBReferenceRegistry>();
 }
@@ -763,15 +765,28 @@ Proton::getConfigGeneration()
     return _protonConfigurer.getActiveConfigSnapshot()->getBootstrapConfig()->getGeneration();
 }
 
+bool
+Proton::updateNodeUp(BucketSpace bucketSpace, bool nodeUpInBucketSpace)
+{
+    std::lock_guard guard(_nodeUpLock);
+    if (nodeUpInBucketSpace) {
+        _nodeUp.insert(bucketSpace);
+    } else {
+        _nodeUp.erase(bucketSpace);
+    }
+    return !_nodeUp.empty();
+}
 
 void
-Proton::setClusterState(const storage::spi::ClusterState &calc)
+Proton::setClusterState(BucketSpace bucketSpace, const storage::spi::ClusterState &calc)
 {
+    (void) bucketSpace;
     // forward info sent by cluster controller to persistence engine
     // about whether node is supposed to be up or not.  Match engine
     // needs to know this in order to stop serving queries.
-    bool nodeUp(calc.nodeUp());
+    bool nodeUpInBucketSpace(calc.nodeUp());
     bool nodeRetired(calc.nodeRetired());
+    bool nodeUp = updateNodeUp(bucketSpace, nodeUpInBucketSpace);
     _matchEngine->setNodeUp(nodeUp);
     if (_memoryFlushConfigUpdater) {
         _memoryFlushConfigUpdater->setNodeRetired(nodeRetired);
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
index 501b5ab8c62..e7e7153be8b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -56,7 +56,6 @@ private:
     typedef std::map<DocTypeName, DocumentDB::SP>         DocumentDBMap;
     typedef BootstrapConfig::ProtonConfigSP               ProtonConfigSP;
     using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
-    using lock_guard = std::lock_guard<std::mutex>;
     using BucketSpace = document::BucketSpace;
 
     struct MetricsUpdateHook : metrics::UpdateHook
@@ -123,6 +122,8 @@ private:
     bool                            _initComplete;
     bool                            _initDocumentDbsInSequence;
     std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry;
+    std::mutex                      _nodeUpLock;
+    std::set<BucketSpace>           _nodeUp;   // bucketspaces where node is up
 
     IDocumentDBConfigOwner *
     addDocumentDB(const DocTypeName & docTypeName, BucketSpace bucketSpace, const vespalib::string & configid,
@@ -145,6 +146,7 @@ private:
     uint32_t getDistributionKey() const override { return _distributionKey; }
     BootstrapConfig::SP getActiveConfigSnapshot() const;
     std::shared_ptr<IDocumentDBReferenceRegistry> getDocumentDBReferenceRegistry() const override;
+    bool updateNodeUp(BucketSpace bucketSpace, bool nodeUpInBucketSpace);
 public:
     typedef std::unique_ptr<Proton> UP;
     typedef std::shared_ptr<Proton> SP;
@@ -177,7 +179,7 @@ public:
     bool prepareRestart();
 
     void getComponentConfig(Consumer &consumer) override;
-    void setClusterState(const storage::spi::ClusterState &calc) override;
+    void setClusterState(BucketSpace bucketSpace, const storage::spi::ClusterState &calc) override;
 
     // Return the oldest active config generation used by proton.
     int64_t getConfigGeneration();