Control valdation with a config.

6 files changed, 21 insertions, 8 deletions

diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index f895bd0c88d..0501ab6ed7c 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -31,6 +31,11 @@ numsummarythreads int default=16 restart
 ## Stop on io errors ?
 stoponioerrors bool default=false restart
 
+## Perform extra validation of stored data on startup
+## It requires a restart to be turned, but no restart to turned off.
+## Hence it must always be followed by a manual restart.
+validate_and_sanitize_docstore enum {NO, YES} default = NO
+
 ## Maximum number of concurrent flushes outstanding.
 flush.maxconcurrent int default=2 restart
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
index 69afeb37277..5931f7ca26a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
@@ -6,6 +6,7 @@
 #include <vespa/searchlib/common/bitvector.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".server.docstorevalidator");
@@ -98,10 +99,10 @@ DocStoreValidator::killOrphans(search::IDocumentStore &store,
 }
 
 
-LidVectorContext::SP
+std::shared_ptr<LidVectorContext>
 DocStoreValidator::getInvalidLids() const
 {
-    LidVectorContext::SP res(new LidVectorContext(_docIdLimit));
+    auto res = std::make_unique<LidVectorContext>(_docIdLimit);
     assert(_invalid->size() == _docIdLimit);
     for (search::DocumentIdT lid(_invalid->getFirstTrueBit(1));
          lid < _docIdLimit;
@@ -127,7 +128,7 @@ void DocStoreValidator::performRemoves(FeedHandler & feedHandler, const search::
             assert(document);
             LOG(info, "Removing document with id %s and lid %u with gid %s in bucket %s", document->getId().toString().c_str(), lid, metaData.gid.toString().c_str(), metaData.bucketId.toString().c_str());
             std::unique_ptr<RemoveOperation> remove = std::make_unique<RemoveOperation>(metaData.bucketId, metaData.timestamp, document->getId());
-            feedHandler.performOperation(FeedToken::UP(), std::move(remove));
+            feedHandler.performOperation(FeedToken(), std::move(remove));
         }
     }
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h
index e27f6ba6c6d..d51d924655a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h
+++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h
@@ -4,12 +4,12 @@
 #include <vespa/searchlib/common/serialnum.h>
 #include <vespa/searchlib/docstore/idocumentstore.h>
 #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
-#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
 
 namespace search { class BitVector; }
 namespace proton {
 
 class FeedHandler;
+class LidVectorContext;
 
 class DocStoreValidator : public search::IDocumentStoreReadVisitor
 {
@@ -32,7 +32,7 @@ public:
     uint32_t getOrphanCount() const;
     uint32_t getVisitCount() const { return _visitCount; }
     uint32_t getVisitEmptyCount() const { return _visitEmptyCount; }
-    LidVectorContext::SP getInvalidLids() const;
+    std::shared_ptr<LidVectorContext> getInvalidLids() const;
     void performRemoves(FeedHandler & feedHandler, const search::IDocumentStore &store, const document::DocumentTypeRepo & repo) const;
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 8dd5c8a44db..55f95ce0518 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -147,6 +147,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
       _activeConfigSnapshot(),
       _activeConfigSnapshotGeneration(0),
       _activeConfigSnapshotSerialNum(0u),
+      _validateAndSanitizeDocStore(protonCfg.validateAndSanitizeDocstore == vespa::config::search::core::ProtonConfig::ValidateAndSanitizeDocstore::YES),
       _initGate(),
       _clusterStateHandler(_writeService.master()),
       _bucketHandler(_writeService.master()),
@@ -661,7 +662,7 @@ DocumentDB::onTransactionLogReplayDone()
         // must signal that all existing buckets must be checked.
         notifyAllBucketsChanged();
     }
-    if (true) {
+    if (_validateAndSanitizeDocStore) {
         LOG(info, "Validating documentdb %s", getName().c_str());
         SerialNum serialNum = _feedHandler.getSerialNum();
         sync(serialNum);
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 1a1d97a657b..917d753683a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -112,6 +112,7 @@ private:
     DocumentDBConfig::SP          _activeConfigSnapshot;
     int64_t                       _activeConfigSnapshotGeneration;
     SerialNum                     _activeConfigSnapshotSerialNum;
+    const bool                    _validateAndSanitizeDocStore;
 
     vespalib::Gate                _initGate;
 
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
index 6c11ebbbf6b..7358a78de61 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
@@ -31,7 +31,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB
 
     DummyDocumentSubDb(std::shared_ptr<BucketDBOwner> bucketDB, uint32_t subDbId)
         : _subDbId(subDbId),
-          _metaStoreCtx(bucketDB),
+          _metaStoreCtx(std::move(bucketDB)),
           _summaryManager(),
           _indexManager(),
           _summaryAdapter(),
@@ -40,7 +40,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB
           _writeService(std::make_unique<ExecutorThreadingService>(_sharedExecutor, 1))
     {
     }
-    ~DummyDocumentSubDb() {}
+    ~DummyDocumentSubDb() override { }
     void close() override { }
     uint32_t getSubDbId() const override { return _subDbId; }
     vespalib::string getName() const override { return "dummysubdb"; }
@@ -64,6 +64,11 @@ struct DummyDocumentSubDb : public IDocumentSubDB
     proton::IAttributeManager::SP getAttributeManager() const override {
         return proton::IAttributeManager::SP();
     }
+
+    void validateDocStore(FeedHandler &, SerialNum ) const override {
+
+    }
+
     const IIndexManager::SP &getIndexManager() const override { return _indexManager; }
     const ISummaryAdapter::SP &getSummaryAdapter() const override { return _summaryAdapter; }
     const IIndexWriter::SP &getIndexWriter() const override { return _indexWriter; }