Add skeleton for proton disk layout, to better handle removal of

unused directories in proton directory tree.

9 files changed, 178 insertions, 19 deletions

diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
index a89f2e3f0ff..70e1db879da 100644
--- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
+++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
@@ -16,6 +16,7 @@
 #include <vespa/searchcore/proton/server/proton_config_snapshot.h>
 #include <vespa/searchcore/proton/server/proton_configurer.h>
 #include <vespa/searchcore/proton/server/i_proton_configurer_owner.h>
+#include <vespa/searchcore/proton/server/i_proton_disk_layout.h>
 #include <vespa/searchsummary/config/config-juniperrc.h>
 #include <vespa/searchcore/config/config-ranking-constants.h>
 #include <vespa/vespalib/testkit/testapp.h>
@@ -224,18 +225,33 @@ struct MyDocumentDBConfigOwner : public DocumentDBConfigOwner
     void reconfigure(const DocumentDBConfig::SP & config) override;
 };
 
-struct MyProtonConfigurerOwner : public IProtonConfigurerOwner
+struct MyLog
+{
+    std::vector<vespalib::string> _log;
+
+    MyLog()
+        : _log()
+    {
+    }
+
+    void appendLog(vespalib::string logEntry)
+    {
+        _log.emplace_back(logEntry);
+    }
+};
+
+struct MyProtonConfigurerOwner : public IProtonConfigurerOwner,
+                                 public MyLog
 {
     using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
     vespalib::ThreadStackExecutor _executor;
     std::map<DocTypeName, std::shared_ptr<MyDocumentDBConfigOwner>> _dbs;
-    std::vector<vespalib::string> _log;
 
     MyProtonConfigurerOwner()
         : IProtonConfigurerOwner(),
+          MyLog(),
           _executor(1, 128 * 1024),
-          _dbs(),
-          _log()
+          _dbs()
     {
     }
     virtual ~MyProtonConfigurerOwner() { }
@@ -286,17 +302,48 @@ MyDocumentDBConfigOwner::reconfigure(const DocumentDBConfig::SP & config)
     _owner.reconfigureDocumentDB(_name, config);
 }
 
+struct MyProtonDiskLayout : public IProtonDiskLayout
+{
+    MyLog &_log;
+
+    MyProtonDiskLayout(MyLog &myLog)
+        : _log(myLog)
+    {
+    }
+    void remove(const DocTypeName &docTypeName) override {
+        std::ostringstream os;
+        os << "remove dbdir " << docTypeName.getName();
+        _log.appendLog(os.str());
+    }
+    void init(const std::set<DocTypeName> &docTypeNames) override {
+        std::ostringstream os;
+        os << "initial dbs ";
+        bool first = true;
+        for (const auto &docTypeName : docTypeNames) {
+            if (!first) {
+                os << ",";
+            }
+            first = false;
+            os << docTypeName.getName();
+        }
+        _log.appendLog(os.str());
+    }
+};
+
 struct Fixture
 {
     MyProtonConfigurerOwner _owner;
     ConfigFixture _config;
+    std::unique_ptr<IProtonDiskLayout> _diskLayout;
     ProtonConfigurer _configurer;
 
     Fixture()
         : _owner(),
           _config("test"),
-          _configurer(_owner._executor, _owner)
+          _diskLayout(),
+          _configurer(_owner._executor, _owner, _diskLayout)
     {
+        _diskLayout = std::make_unique<MyProtonDiskLayout>(_owner);
     }
     ~Fixture() { }
 
@@ -338,14 +385,14 @@ TEST_F("require that nothing is applied before initial config", Fixture())
 TEST_F("require that initial config is applied", Fixture())
 {
     f.applyInitialConfig();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}));
 }
 
 TEST_F("require that new config is blocked", Fixture())
 {
     f.applyInitialConfig();
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}));
 }
 
 TEST_F("require that new config can be unblocked", Fixture())
@@ -353,14 +400,14 @@ TEST_F("require that new config can be unblocked", Fixture())
     f.applyInitialConfig();
     f.reconfigure();
     f.allowReconfig();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}));
 }
 
 TEST_F("require that initial config is not reapplied due to config unblock", Fixture())
 {
     f.applyInitialConfig();
     f.allowReconfig();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}));
 }
 
 TEST_F("require that we can add document db", Fixture())
@@ -369,7 +416,7 @@ TEST_F("require that we can add document db", Fixture())
     f.allowReconfig();
     f.addDocType("foobar");
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"}));
 }
 
 TEST_F("require that we can remove document db", Fixture())
@@ -379,7 +426,7 @@ TEST_F("require that we can remove document db", Fixture())
     f.allowReconfig();
     f.removeDocType("foobar");
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar", "remove dbdir foobar"}));
 }
 
 TEST_F("require that document db adds and reconfigs are intermingled", Fixture())
@@ -392,7 +439,7 @@ TEST_F("require that document db adds and reconfigs are intermingled", Fixture()
     f.addDocType("foobar");
     f.addDocType("zbar");
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"}));
 }
 
 TEST_F("require that document db removes are applied at end", Fixture())
@@ -403,7 +450,7 @@ TEST_F("require that document db removes are applied at end", Fixture())
     f.allowReconfig();
     f.removeDocType("abar");
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,abar,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar", "remove dbdir abar"}));
 }
 
 TEST_F("require that new configs can be blocked again", Fixture())
@@ -413,7 +460,7 @@ TEST_F("require that new configs can be blocked again", Fixture())
     f.allowReconfig();
     f.disableReconfig();
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}));
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
index 87aa19fb1b2..2df34312b52 100644
--- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
@@ -75,6 +75,7 @@ vespa_add_library(searchcore_server STATIC
     proton_config_fetcher.cpp
     proton_config_snapshot.cpp
     proton_configurer.cpp
+    proton_disk_layout.cpp
     prune_session_cache_job.cpp
     pruneremoveddocumentsjob.cpp
     putdonecontext.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h
new file mode 100644
index 00000000000..dc38c38cc1c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h
@@ -0,0 +1,23 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <set>
+
+namespace proton {
+
+class DocTypeName;
+
+/**
+ * Interface class with utility functions for handling the disk
+ * directory layout for proton instance.
+ */
+class IProtonDiskLayout
+{
+public:
+    virtual ~IProtonDiskLayout() = default;
+    virtual void remove(const DocTypeName &docTypeName) = 0;
+    virtual void init(const std::set<DocTypeName> &docTypeNames) = 0;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 9a7d5a5141e..ab0abde6ccd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -9,6 +9,7 @@
 #include "prepare_restart_handler.h"
 #include "proton.h"
 #include "proton_config_snapshot.h"
+#include "proton_disk_layout.h"
 #include "resource_usage_explorer.h"
 #include "searchhandlerproxy.h"
 #include "simpleflush.h"
@@ -193,7 +194,8 @@ Proton::Proton(const config::ConfigUri & configUri,
       // This executor can only have 1 thread as it is used for
       // serializing startup.
       _executor(1, 128 * 1024),
-      _protonConfigurer(_executor, *this),
+      _protonDiskLayout(),
+      _protonConfigurer(_executor, *this, _protonDiskLayout),
       _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout),
       _warmupExecutor(),
       _summaryExecutor(),
@@ -269,7 +271,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
         strategy = std::make_shared<SimpleFlush>();
         break;
     }
-    vespalib::mkdir(protonConfig.basedir + "/documents", true);
+    _protonDiskLayout = std::make_unique<ProtonDiskLayout>(protonConfig.basedir, protonConfig.tlsspec);
     vespalib::chdir(protonConfig.basedir);
     _tls->start();
     _flushEngine = std::make_unique<FlushEngine>(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()),
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
index a3c596d7b0b..6e07ddcecdb 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -37,6 +37,7 @@ namespace proton {
 
 class DiskMemUsageSampler;
 class IDocumentDBReferenceRegistry;
+class IProtonDiskLayout;
 class PrepareRestartHandler;
 class SummaryEngine;
 class DocsumBySlime;
@@ -111,6 +112,7 @@ private:
     std::unique_ptr<vespalib::StateServer>  _stateServer;
     std::unique_ptr<TransportServer>        _fs4Server;
     vespalib::ThreadStackExecutor   _executor;
+    std::unique_ptr<IProtonDiskLayout> _protonDiskLayout;
     ProtonConfigurer                _protonConfigurer;
     ProtonConfigFetcher             _protonConfigFetcher;
     std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
index f871f36b042..1f565e14889 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
@@ -6,6 +6,7 @@
 #include "i_proton_configurer_owner.h"
 #include "document_db_config_owner.h"
 #include "document_db_directory_holder.h"
+#include "i_proton_disk_layout.h"
 #include <vespa/vespalib/util/lambdatask.h>
 #include <vespa/vespalib/util/threadstackexecutorbase.h>
 #include <vespa/document/bucket/fixed_bucket_spaces.h>
@@ -39,7 +40,8 @@ getBucketSpace(const BootstrapConfig &bootstrapConfig, const DocTypeName &name)
 
 
 ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor,
-                                   IProtonConfigurerOwner &owner)
+                                   IProtonConfigurerOwner &owner,
+                                   const std::unique_ptr<IProtonDiskLayout> &diskLayout)
     : IProtonConfigurer(),
       _executor(executor),
       _owner(owner),
@@ -48,7 +50,8 @@ ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor,
       _activeConfigSnapshot(),
       _mutex(),
       _allowReconfig(false),
-      _componentConfig()
+      _componentConfig(),
+      _diskLayout(diskLayout)
 {
 }
 
@@ -135,6 +138,9 @@ ProtonConfigurer::applyConfig(std::shared_ptr<ProtonConfigSnapshot> configSnapsh
     }
     const auto &bootstrapConfig = configSnapshot->getBootstrapConfig();
     const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig();
+    if (initialConfig && _diskLayout) {
+        pruneInitialDocumentDBDirs(*configSnapshot);
+    }
     _owner.applyConfig(bootstrapConfig);
     for (const auto &ddbConfig : protonConfig.documentdb) {
         DocTypeName docTypeName(ddbConfig.inputdoctypename);
@@ -176,6 +182,18 @@ ProtonConfigurer::configureDocumentDB(const ProtonConfigSnapshot &configSnapshot
 }
 
 void
+ProtonConfigurer::pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot)
+{
+    std::set<DocTypeName> docTypeNames;
+    const auto &bootstrapConfig = configSnapshot.getBootstrapConfig();
+    const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig();
+    for (const auto &ddbConfig : protonConfig.documentdb) {
+        docTypeNames.emplace(ddbConfig.inputdoctypename);
+    }
+    _diskLayout->init(docTypeNames);
+}
+
+void
 ProtonConfigurer::pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot)
 {
     // called by proton executor thread
@@ -193,6 +211,9 @@ ProtonConfigurer::pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot)
         if (found == newDocTypes.end()) {
             _owner.removeDocumentDB(dbitr->first);
             DocumentDBDirectoryHolder::waitUntilDestroyed(dbitr->second.second);
+            if (_diskLayout) {
+                _diskLayout->remove(dbitr->first);
+            }
             dbitr = _documentDBs.erase(dbitr);
         } else {
             ++dbitr;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
index ac0f6197fd7..c896f12bd4f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
@@ -16,6 +16,7 @@ class DocumentDBDirectoryHolder;
 class IDocumentDBConfigOwner;
 class IProtonConfigurerOwner;
 class BootstrapConfig;
+class IProtonDiskLayout;
 
 /*
  * Class to handle config changes to proton using config snapshots spanning
@@ -34,6 +35,7 @@ class ProtonConfigurer : public IProtonConfigurer
     mutable std::mutex _mutex;
     bool _allowReconfig;
     vespalib::SimpleComponentConfigProducer _componentConfig;
+    const std::unique_ptr<IProtonDiskLayout> &_diskLayout;
 
     void performReconfigure();
     bool skipConfig(const ProtonConfigSnapshot *configSnapshot, bool initialConfig);
@@ -43,10 +45,12 @@ class ProtonConfigurer : public IProtonConfigurer
                              const DocTypeName &docTypeName, document::BucketSpace bucketSpace,
                              const vespalib::string &configId, const InitializeThreads &initializeThreads);
     void pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot);
+    void pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot);
 
 public:
     ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor,
-                     IProtonConfigurerOwner &owner);
+                     IProtonConfigurerOwner &owner,
+                     const std::unique_ptr<IProtonDiskLayout> &diskLayout);
 
     ~ProtonConfigurer();
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp
new file mode 100644
index 00000000000..0e0d96a3211
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp
@@ -0,0 +1,32 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "proton_disk_layout.h"
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/fastos/file.h>
+#include <cassert>
+
+namespace proton {
+
+ProtonDiskLayout::ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec)
+    : _baseDir(baseDir),
+      _tlsSpec(tlsSpec)
+{
+    vespalib::mkdir(_baseDir + "/documents", true);
+}
+
+ProtonDiskLayout::~ProtonDiskLayout() = default;
+
+void
+ProtonDiskLayout::remove(const DocTypeName &docTypeName)
+{
+    (void) docTypeName;
+}
+
+void
+ProtonDiskLayout::init(const std::set<DocTypeName> &docTypeNames)
+{
+    (void) docTypeNames;
+}
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h
new file mode 100644
index 00000000000..b7aeafb6b54
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h
@@ -0,0 +1,27 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_proton_disk_layout.h"
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+/**
+ * Class with utility functions for handling the disk directory layout
+ * for proton instance.
+ */
+class ProtonDiskLayout : public IProtonDiskLayout
+{
+private:
+    const vespalib::string _baseDir;
+    const vespalib::string _tlsSpec;
+
+public:
+    ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec);
+    ~ProtonDiskLayout() override;
+    void remove(const DocTypeName &docTypeName) override;
+    void init(const std::set<DocTypeName> &docTypeNames) override;
+};
+
+} // namespace proton