Collapse searchcorespi into searchcore

9 files changed, 279 insertions, 1 deletions

diff --git a/searchcore/src/tests/index/disk_indexes/CMakeLists.txt b/searchcore/src/tests/index/disk_indexes/CMakeLists.txt
new file mode 100644
index 00000000000..81b6bb0e8a9
--- /dev/null
+++ b/searchcore/src/tests/index/disk_indexes/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcorespi_disk_indexes_test_app
+    SOURCES
+    disk_indexes_test.cpp
+    DEPENDS
+    searchcorespi
+    GTest::GTest
+)
+vespa_add_test(NAME searchcorespi_disk_indexes_test_app COMMAND searchcorespi_disk_indexes_test_app)
diff --git a/searchcore/src/tests/index/disk_indexes/disk_indexes_test.cpp b/searchcore/src/tests/index/disk_indexes/disk_indexes_test.cpp
new file mode 100644
index 00000000000..d22ad499316
--- /dev/null
+++ b/searchcore/src/tests/index/disk_indexes/disk_indexes_test.cpp
@@ -0,0 +1,196 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchcorespi/index/disk_indexes.h>
+#include <vespa/searchcorespi/index/index_disk_dir.h>
+#include <vespa/searchcorespi/index/indexdisklayout.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/size_literals.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <fstream>
+
+namespace {
+
+vespalib::string base_dir("base");
+
+constexpr uint32_t block_size = 4_Ki;
+
+}
+
+namespace searchcorespi::index {
+
+class DiskIndexesTest : public ::testing::Test,
+                              public DiskIndexes
+{
+    IndexDiskLayout _layout;
+protected:
+    DiskIndexesTest();
+    ~DiskIndexesTest();
+
+    static IndexDiskDir get_index_disk_dir(const vespalib::string& dir) {
+        return IndexDiskLayout::get_index_disk_dir(dir);
+    }
+
+    void assert_transient_size(uint64_t exp, IndexDiskDir index_disk_dir) {
+        EXPECT_EQ(exp, get_transient_size(_layout, index_disk_dir));
+    }
+};
+
+DiskIndexesTest::DiskIndexesTest()
+    : ::testing::Test(),
+      DiskIndexes(),
+      _layout(base_dir)
+{
+}
+
+DiskIndexesTest::~DiskIndexesTest() = default;
+
+TEST_F(DiskIndexesTest, simple_set_active_works)
+{
+    EXPECT_FALSE(isActive("index.flush.1"));
+    setActive("index.flush.1", 0);
+    EXPECT_TRUE(isActive("index.flush.1"));
+    notActive("index.flush.1");
+    EXPECT_FALSE(isActive("index.flush.1"));
+}
+
+TEST_F(DiskIndexesTest, nested_set_active_works)
+{
+    setActive("index.flush.1", 0);
+    setActive("index.flush.1", 0);
+    EXPECT_TRUE(isActive("index.flush.1"));
+    notActive("index.flush.1");
+    EXPECT_TRUE(isActive("index.flush.1"));
+    notActive("index.flush.1");
+    EXPECT_FALSE(isActive("index.flush.1"));
+}
+
+TEST_F(DiskIndexesTest, is_active_returns_false_for_bad_name)
+{
+    EXPECT_FALSE(isActive("foo/bar/baz"));
+    EXPECT_FALSE(isActive("index.flush.0"));
+}
+
+TEST_F(DiskIndexesTest, remove_works)
+{
+    EXPECT_TRUE(remove(IndexDiskDir()));
+    auto fusion1 = get_index_disk_dir("index.fusion.1");
+    EXPECT_TRUE(remove(fusion1));
+    add_not_active(fusion1);
+    EXPECT_TRUE(remove(fusion1));
+    setActive("index.fusion.1", 0);
+    EXPECT_FALSE(remove(fusion1));
+    notActive("index.fusion.1");
+    EXPECT_TRUE(remove(fusion1));
+}
+
+TEST_F(DiskIndexesTest, basic_get_transient_size_works)
+{
+    /*
+     * When starting to use a new fusion index, we have a transient
+     * period with two ISearchableIndexCollection instances:
+     * - old, containing index.fusion.1 and index.flush.2
+     * - new, containing index.fusion.2
+     */
+    setActive("index.fusion.1", 1000000);
+    setActive("index.flush.2", 500000);
+    setActive("index.fusion.2", 1200000);
+    auto fusion1 = get_index_disk_dir("index.fusion.1");
+    auto flush2 = get_index_disk_dir("index.flush.2");
+    auto fusion2 = get_index_disk_dir("index.fusion.2");
+    {
+        /*
+         * When using the old index collection, disk space used by
+         * index.fusion.2 is considered transient.
+         */
+        SCOPED_TRACE("index.fusion.1");
+        assert_transient_size(1200000, fusion1);
+    }
+    {
+        SCOPED_TRACE("index.flush.2");
+        assert_transient_size(0, flush2);
+    }
+    {
+        /*
+         * When using the new index collection, disk space used by
+         * index.fusion.1 and index.flush.2 is considered transient.
+         */
+        SCOPED_TRACE("index.fusion.2");
+        assert_transient_size(1500000, fusion2);
+    }
+    notActive("index.fusion.1");
+    notActive("index.flush.2");
+    {
+        /*
+         * old index collection removed.
+         */
+        SCOPED_TRACE("index.fusion.2 after remove of index.fusion.1 and index.flush.1");
+        assert_transient_size(0, fusion2);
+    }
+}
+
+TEST_F(DiskIndexesTest, get_transient_size_during_ongoing_fusion)
+{
+    /*
+     * During ongoing fusion, we have one ISearchableIndexCollection instance:
+     * - old, containing index.fusion.1 and index.flush.2
+     *
+     * Fusion output directory is index.fusion.2
+     */
+    setActive("index.fusion.1", 1000000);
+    setActive("index.flush.2", 500000);
+    auto fusion1 = get_index_disk_dir("index.fusion.1");
+    auto fusion2 = get_index_disk_dir("index.fusion.2");
+    add_not_active(fusion2); // start tracking disk space for fusion output
+    {
+        /*
+         * Fusion not yet started.
+         */
+        SCOPED_TRACE("dir missing");
+        assert_transient_size(0, fusion1);
+    }
+    auto dir = base_dir + "/index.fusion.2";
+    vespalib::mkdir(dir, true);
+    {
+        /*
+         * Fusion started, but no files written yet.
+         */
+        SCOPED_TRACE("empty dir");
+        assert_transient_size(0, fusion1);
+    }
+    constexpr uint32_t seek_pos = 999999;
+    {
+        std::string name = dir + "/foo";
+        std::ofstream ostr(name, std::ios::binary);
+        ostr.seekp(seek_pos);
+        ostr.write(" ", 1);
+        ostr.flush();
+        ostr.close();
+    }
+    {
+        /*
+         * Fusion started, one file written.
+         */
+        SCOPED_TRACE("single file");
+        assert_transient_size((seek_pos + block_size) / block_size * block_size, fusion1);
+    }
+    EXPECT_TRUE(remove(fusion2)); // stop tracking disk space for fusion output
+    {
+        /*
+         * Fusion aborted.
+         */
+        SCOPED_TRACE("removed");
+        assert_transient_size(0, fusion1);
+    }
+}
+
+}
+
+int
+main(int argc, char* argv[])
+{
+    vespalib::rmdir(base_dir, true);
+    ::testing::InitGoogleTest(&argc, argv);
+    auto result = RUN_ALL_TESTS();
+    vespalib::rmdir(base_dir, true);
+    return result;
+}
diff --git a/searchcore/src/tests/index/index_disk_layout/CMakeLists.txt b/searchcore/src/tests/index/index_disk_layout/CMakeLists.txt
new file mode 100644
index 00000000000..4e82cf1b9d2
--- /dev/null
+++ b/searchcore/src/tests/index/index_disk_layout/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcorespi_index_disk_layout_test_app
+    SOURCES
+    index_disk_layout_test.cpp
+    DEPENDS
+    searchcorespi
+    GTest::GTest
+)
+vespa_add_test(NAME searchcorespi_index_disk_layout_test_app COMMAND searchcorespi_index_disk_layout_test_app)
diff --git a/searchcore/src/tests/index/index_disk_layout/index_disk_layout_test.cpp b/searchcore/src/tests/index/index_disk_layout/index_disk_layout_test.cpp
new file mode 100644
index 00000000000..e35225b2745
--- /dev/null
+++ b/searchcore/src/tests/index/index_disk_layout/index_disk_layout_test.cpp
@@ -0,0 +1,60 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchcorespi/index/indexdisklayout.h>
+#include <vespa/searchcorespi/index/index_disk_dir.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+namespace searchcorespi::index {
+
+namespace {
+
+void expect_index_disk_dir(IndexDiskDir exp, const vespalib::string& dir)
+{
+    auto act = IndexDiskLayout::get_index_disk_dir(dir);
+    ASSERT_TRUE(act.valid());
+    ASSERT_EQ(exp, act);
+}
+
+void expect_bad_index_disk_dir(const vespalib::string& dir)
+{
+    auto act = IndexDiskLayout::get_index_disk_dir(dir);
+    ASSERT_FALSE(act.valid());
+}
+
+}
+
+TEST(IndexDiskLayoutTest, get_index_disk_dir_works)
+{
+    {
+        SCOPED_TRACE("index.fusion.1");
+        expect_index_disk_dir(IndexDiskDir(1, true), "index.fusion.1");
+    }
+    {
+        SCOPED_TRACE("index.flush.2");
+        expect_index_disk_dir(IndexDiskDir(2, false), "index.flush.2");
+    }
+    {
+        SCOPED_TRACE("index.flush.3");
+        expect_index_disk_dir(IndexDiskDir(3, false), "index.flush.3");
+    }
+    {
+        SCOPED_TRACE("foo/bar/index.flush.4");
+        expect_index_disk_dir(IndexDiskDir(4, false), "foo/bar/index.flush.4");
+    }
+    {
+        SCOPED_TRACE("index.flush.");
+        expect_bad_index_disk_dir("index.flush.");
+    }
+    {
+        SCOPED_TRACE("index.flush.0");
+        expect_bad_index_disk_dir("index.flush.0");
+    }
+    {
+        SCOPED_TRACE("asdf");
+        expect_bad_index_disk_dir("asdf");
+    }
+}
+
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchcore/src/tests/proton/flushengine/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/CMakeLists.txt
index b45c4fa411c..a47b3f2c93f 100644
--- a/searchcore/src/tests/proton/flushengine/CMakeLists.txt
+++ b/searchcore/src/tests/proton/flushengine/CMakeLists.txt
@@ -5,6 +5,7 @@ vespa_add_executable(searchcore_flushengine_test_app TEST
     DEPENDS
     searchcore_flushengine
     searchcore_pcommon
+    searchcore_test
 )
 vespa_add_test(
     NAME searchcore_flushengine_test_app
diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt
index 9f0c777a4d7..608f80e38aa 100644
--- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt
+++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt
@@ -3,8 +3,8 @@ vespa_add_executable(searchcore_flushengine_prepare_restart_flush_strategy_test_
     SOURCES
     prepare_restart_flush_strategy_test.cpp
     DEPENDS
-    searchcorespi
     searchcore_flushengine
+    searchcore_test
 )
 vespa_add_test(
     NAME searchcore_flushengine_prepare_restart_flush_strategy_test_app
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt
index 4a5dd2acb9d..aad11581e2f 100644
--- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt
@@ -4,5 +4,6 @@ vespa_add_executable(searchcore_documentdb_job_trackers_test_app TEST
     documentdb_job_trackers_test.cpp
     DEPENDS
     searchcore_proton_metrics
+    searchcore_test
 )
 vespa_add_test(NAME searchcore_documentdb_job_trackers_test_app COMMAND searchcore_documentdb_job_trackers_test_app)
diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt
index 6ab08602f67..4467bea4ab1 100644
--- a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt
+++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt
@@ -4,5 +4,6 @@ vespa_add_executable(searchcore_job_tracked_flush_test_app TEST
     job_tracked_flush_test.cpp
     DEPENDS
     searchcore_proton_metrics
+    searchcore_test
 )
 vespa_add_test(NAME searchcore_job_tracked_flush_test_app COMMAND searchcore_job_tracked_flush_test_app)
diff --git a/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt b/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt
index 80c5521c763..207883aab0e 100644
--- a/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt
+++ b/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt
@@ -5,5 +5,6 @@ vespa_add_executable(searchcore_memoryflush_test_app TEST
     DEPENDS
     searchcore_server
     searchcore_flushengine
+    searchcore_test
 )
 vespa_add_test(NAME searchcore_memoryflush_test_app COMMAND searchcore_memoryflush_test_app)