diff options
Diffstat (limited to 'searchcore')
116 files changed, 610 insertions, 419 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index fa5576a6612..3d72d70af48 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -54,9 +54,6 @@ vespa_define_module( src/apps/vespa-proton-cmd src/apps/vespa-transactionlog-inspect - APP_DEPENDS - persistence_persistence_conformancetest - TESTS src/tests/applyattrupdates src/tests/fdispatch/randomrow diff --git a/searchcore/OWNERS b/searchcore/OWNERS index 22a8fdfdfa6..81de3c9fc79 100644 --- a/searchcore/OWNERS +++ b/searchcore/OWNERS @@ -1,5 +1,5 @@ geirst -tegge -balder +toregge +baldersheim havardpe yngve diff --git a/searchcore/src/apps/tests/CMakeLists.txt b/searchcore/src/apps/tests/CMakeLists.txt index 307ae51f7f6..4bb704982d3 100644 --- a/searchcore/src/apps/tests/CMakeLists.txt +++ b/searchcore/src/apps/tests/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_persistenceconformance_test_app +vespa_add_executable(searchcore_persistenceconformance_test_app TEST SOURCES persistenceconformance_test.cpp DEPENDS @@ -21,5 +21,5 @@ vespa_add_executable(searchcore_persistenceconformance_test_app searchcore_fconfig searchcore_util vdstestlib + persistence_persistence_conformancetest ) -vespa_add_test(NAME searchcore_persistenceconformance_test_app COMMAND searchcore_persistenceconformance_test_app) diff --git a/searchcore/src/tests/applyattrupdates/CMakeLists.txt b/searchcore/src/tests/applyattrupdates/CMakeLists.txt index 2778d0f62dc..8e4b11bb71f 100644 --- a/searchcore/src/tests/applyattrupdates/CMakeLists.txt +++ b/searchcore/src/tests/applyattrupdates/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_applyattrupdates_test_app +vespa_add_executable(searchcore_applyattrupdates_test_app TEST SOURCES applyattrupdates.cpp DEPENDS diff --git a/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt b/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt index f3ad936ded2..204dc8fa00d 100644 --- a/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt +++ b/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_randomrow_test_app +vespa_add_executable(searchcore_randomrow_test_app TEST SOURCES randomrow_test.cpp DEPENDS diff --git a/searchcore/src/tests/fdispatch/search_path/CMakeLists.txt b/searchcore/src/tests/fdispatch/search_path/CMakeLists.txt index 86067faa4cc..cf2f4872c54 100644 --- a/searchcore/src/tests/fdispatch/search_path/CMakeLists.txt +++ b/searchcore/src/tests/fdispatch/search_path/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_search_path_test_app +vespa_add_executable(searchcore_search_path_test_app TEST SOURCES search_path_test.cpp DEPENDS diff --git a/searchcore/src/tests/grouping/CMakeLists.txt b/searchcore/src/tests/grouping/CMakeLists.txt index 245406187f6..1b7ae72f76d 100644 --- a/searchcore/src/tests/grouping/CMakeLists.txt +++ b/searchcore/src/tests/grouping/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_grouping_test_app +vespa_add_executable(searchcore_grouping_test_app TEST SOURCES grouping.cpp DEPENDS diff --git a/searchcore/src/tests/proton/attribute/CMakeLists.txt b/searchcore/src/tests/proton/attribute/CMakeLists.txt index 1439c2b2646..4082a8f5563 100644 --- a/searchcore/src/tests/proton/attribute/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_test_app +vespa_add_executable(searchcore_attribute_test_app TEST SOURCES attribute_test.cpp DEPENDS @@ -9,7 +9,7 @@ vespa_add_executable(searchcore_attribute_test_app searchcore_pcommon ) vespa_add_test(NAME searchcore_attribute_test_app COMMAND sh attribute_test.sh) -vespa_add_executable(searchcore_attributeflush_test_app +vespa_add_executable(searchcore_attributeflush_test_app TEST SOURCES attributeflush_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt index 7e8ab14a13b..d1a0e3a45e0 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_manager_test_app +vespa_add_executable(searchcore_attribute_manager_test_app TEST SOURCES attribute_manager_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt index 064759b88d1..e1de90c0284 100644 --- a/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_populator_test_app +vespa_add_executable(searchcore_attribute_populator_test_app TEST SOURCES attribute_populator_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt index 2dd66c2a3ec..db904aaf5b3 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_usage_filter_test_app +vespa_add_executable(searchcore_attribute_usage_filter_test_app TEST SOURCES attribute_usage_filter_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt index 322d22c8f0d..158dd78c0a7 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attributes_state_explorer_test_app +vespa_add_executable(searchcore_attributes_state_explorer_test_app TEST SOURCES attributes_state_explorer_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt index 4c6da0a3397..35a89676c5b 100644 --- a/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_field_populator_test_app +vespa_add_executable(searchcore_document_field_populator_test_app TEST SOURCES document_field_populator_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt index c39025ae39f..81afcc7a4af 100644 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_exclusive_attribute_read_accessor_test_app +vespa_add_executable(searchcore_exclusive_attribute_read_accessor_test_app TEST SOURCES exclusive_attribute_read_accessor_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt b/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt index f07ded6d89b..58c27f31726 100644 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt +++ b/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_bucketdb_test_app +vespa_add_executable(searchcore_bucketdb_test_app TEST SOURCES bucketdb_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/common/CMakeLists.txt b/searchcore/src/tests/proton/common/CMakeLists.txt index 833bac4c065..5f0fa227c88 100644 --- a/searchcore/src/tests/proton/common/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/CMakeLists.txt @@ -1,19 +1,19 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_schemautil_test_app +vespa_add_executable(searchcore_schemautil_test_app TEST SOURCES schemautil_test.cpp DEPENDS searchcore_pcommon ) vespa_add_test(NAME searchcore_schemautil_test_app COMMAND searchcore_schemautil_test_app) -vespa_add_executable(searchcore_selectpruner_test_app +vespa_add_executable(searchcore_selectpruner_test_app TEST SOURCES selectpruner_test.cpp DEPENDS searchcore_pcommon ) vespa_add_test(NAME searchcore_selectpruner_test_app COMMAND searchcore_selectpruner_test_app) -vespa_add_executable(searchcore_cachedselect_test_app +vespa_add_executable(searchcore_cachedselect_test_app TEST SOURCES cachedselect_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt b/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt index f5c4610fc1b..cd4533510ff 100644 --- a/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_type_inspector_test_app +vespa_add_executable(searchcore_document_type_inspector_test_app TEST SOURCES document_type_inspector_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt b/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt index 9b1ffd8aef2..d872cc5d89f 100644 --- a/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_state_reporter_utils_test_app +vespa_add_executable(searchcore_state_reporter_utils_test_app TEST SOURCES state_reporter_utils_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/config/CMakeLists.txt b/searchcore/src/tests/proton/config/CMakeLists.txt index 803556c2ae1..7bef6552154 100644 --- a/searchcore/src/tests/proton/config/CMakeLists.txt +++ b/searchcore/src/tests/proton/config/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_config_test_app +vespa_add_executable(searchcore_config_test_app TEST SOURCES config.cpp DEPENDS diff --git a/searchcore/src/tests/proton/docsummary/CMakeLists.txt b/searchcore/src/tests/proton/docsummary/CMakeLists.txt index dca65528840..7550a80aa3a 100644 --- a/searchcore/src/tests/proton/docsummary/CMakeLists.txt +++ b/searchcore/src/tests/proton/docsummary/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_docsummary_test_app +vespa_add_executable(searchcore_docsummary_test_app TEST SOURCES docsummary.cpp DEPENDS diff --git a/searchcore/src/tests/proton/document_iterator/CMakeLists.txt b/searchcore/src/tests/proton/document_iterator/CMakeLists.txt index 03d910ef02e..32a8e44efd5 100644 --- a/searchcore/src/tests/proton/document_iterator/CMakeLists.txt +++ b/searchcore/src/tests/proton/document_iterator/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_iterator_test_app +vespa_add_executable(searchcore_document_iterator_test_app TEST SOURCES document_iterator_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/CMakeLists.txt index 9270a4b0b7c..7cd23e47d00 100644 --- a/searchcore/src/tests/proton/documentdb/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentdb_test_app +vespa_add_executable(searchcore_documentdb_test_app TEST SOURCES documentdb_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt index 3c1f5c79a57..226d8965844 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_buckethandler_test_app +vespa_add_executable(searchcore_buckethandler_test_app TEST SOURCES buckethandler_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt index f107cddd103..6244fb0537a 100644 --- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_clusterstatehandler_test_app +vespa_add_executable(searchcore_clusterstatehandler_test_app TEST SOURCES clusterstatehandler_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt index 74f605d36d0..e435c27fbde 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_combiningfeedview_test_app +vespa_add_executable(searchcore_combiningfeedview_test_app TEST SOURCES combiningfeedview_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt index ee18f0f6938..1e8b241b72c 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_configurer_test_app +vespa_add_executable(searchcore_configurer_test_app TEST SOURCES configurer_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 1764d6f2996..6b6abe5bc09 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -135,6 +135,7 @@ struct Fixture _views(), _configurer() { + vespalib::rmdir(BASE_DIR, true); vespalib::mkdir(BASE_DIR); initViewSet(_views); _configurer.reset(new Configurer(_views._summaryMgr, @@ -146,7 +147,6 @@ struct Fixture 0)); } ~Fixture() { - vespalib::rmdir(BASE_DIR, true); } void initViewSet(ViewSet &views); }; @@ -293,11 +293,11 @@ struct FastAccessFixture _configurer(_view._feedView, IAttributeAdapterFactory::UP(new AttributeAdapterFactory), "test") { + vespalib::rmdir(BASE_DIR, true); vespalib::mkdir(BASE_DIR); } ~FastAccessFixture() { _writeService.sync(); - vespalib::rmdir(BASE_DIR, true); } }; @@ -608,4 +608,5 @@ TEST_F("require that we can reconfigure matchers", Fixture) TEST_MAIN() { TEST_RUN_ALL(); + vespalib::rmdir(BASE_DIR, true); } diff --git a/searchcore/src/tests/proton/documentdb/configvalidator/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/configvalidator/CMakeLists.txt index c7a3a6235cf..cea6144fd99 100644 --- a/searchcore/src/tests/proton/documentdb/configvalidator/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/configvalidator/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_configvalidator_test_app +vespa_add_executable(searchcore_configvalidator_test_app TEST SOURCES configvalidator_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt index 1a342660f7c..1789beee7da 100644 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_scan_iterator_test_app +vespa_add_executable(searchcore_document_scan_iterator_test_app TEST SOURCES document_scan_iterator_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt index d79b9ad92ae..121b37bbfcc 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_subdbs_test_app +vespa_add_executable(searchcore_document_subdbs_test_app TEST SOURCES document_subdbs_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt index 97a9ae1516b..a550ce6451f 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentbucketmover_test_app +vespa_add_executable(searchcore_documentbucketmover_test_app TEST SOURCES documentbucketmover_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt index b9105fc9c91..1ad3b7f21f6 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentdbconfig_test_app +vespa_add_executable(searchcore_documentdbconfig_test_app TEST SOURCES documentdbconfig_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt index e1dea56782d..8de0cc67b2a 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentdbconfigscout_test_app +vespa_add_executable(searchcore_documentdbconfigscout_test_app TEST SOURCES documentdbconfigscout_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt index 756c11f35b4..fe848e0588b 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_feedhandler_test_app +vespa_add_executable(searchcore_feedhandler_test_app TEST SOURCES feedhandler_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt index 0ea395a339a..a4cebe05541 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_feedview_test_app +vespa_add_executable(searchcore_feedview_test_app TEST SOURCES feedview_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/fileconfigmanager/CMakeLists.txt index 458607e66c5..f5c40a85192 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_fileconfigmanager_test_app +vespa_add_executable(searchcore_fileconfigmanager_test_app TEST SOURCES fileconfigmanager_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt index 3b81994e7da..e858efd45ea 100644 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_job_tracked_maintenance_job_test_app +vespa_add_executable(searchcore_job_tracked_maintenance_job_test_app TEST SOURCES job_tracked_maintenance_job_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt index 938e0dc7baf..6f993ac1593 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_lid_space_compaction_test_app +vespa_add_executable(searchcore_lid_space_compaction_test_app TEST SOURCES lid_space_compaction_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt index 4f26a79f0eb..388d527dec5 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_maintenancecontroller_test_app +vespa_add_executable(searchcore_maintenancecontroller_test_app TEST SOURCES maintenancecontroller_test.cpp DEPENDS @@ -18,7 +18,7 @@ vespa_add_executable(searchcore_maintenancecontroller_test_app searchcore_fconfig ) vespa_add_test(NAME searchcore_maintenancecontroller_test_app COMMAND searchcore_maintenancecontroller_test_app) -vespa_add_executable(searchcore_frozenbucketsmap_test_app +vespa_add_executable(searchcore_frozenbucketsmap_test_app TEST SOURCES frozenbucketsmap_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt index 294360b51ba..7c8b092811f 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_storeonlyfeedview_test_app +vespa_add_executable(searchcore_storeonlyfeedview_test_app TEST SOURCES storeonlyfeedview_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index fe9251e1193..266fb6279e9 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for storeonlyfeedview. - -#include <vespa/log/log.h> -LOG_SETUP("storeonlyfeedview_test"); #include <vespa/fastos/fastos.h> #include <vespa/document/base/documentid.h> @@ -20,6 +17,9 @@ LOG_SETUP("storeonlyfeedview_test"); #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("storeonlyfeedview_test"); + using document::BucketId; using document::DataType; using document::Document; diff --git a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt index fbaa86cafc8..56b07ffc181 100644 --- a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentmetastore_test_app +vespa_add_executable(searchcore_documentmetastore_test_app TEST SOURCES documentmetastore_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index e1e9f58fc14..7b09beb35b6 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -44,6 +44,15 @@ using proton::bucketdb::BucketState; namespace proton { +namespace +{ + +static constexpr uint32_t numBucketBits = UINT32_C(20); +static constexpr uint64_t timestampBias = UINT64_C(2000000000000); + +} + + class DummyTlsSyncer : public ITlsSyncer { public: @@ -552,23 +561,21 @@ TEST("requireThatWeCanStoreBucketIdAndTimestamp") { DocumentMetaStore dms(createBucketDB()); uint32_t numLids = 1000; - uint32_t bkBits = UINT32_C(20); - uint64_t tsbias = UINT64_C(2000000000000); dms.constructFreeList(); for (uint32_t lid = 1; lid <= numLids; ++lid) { GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); - bucketId.setUsedBits(bkBits); - uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(lid + tsbias)); + bucketId.setUsedBits(numBucketBits); + uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(lid + timestampBias)); EXPECT_EQUAL(lid, addLid); } for (uint32_t lid = 1; lid <= numLids; ++lid) { GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); - bucketId.setUsedBits(bkBits); + bucketId.setUsedBits(numBucketBits); EXPECT_TRUE(assertGid(gid, lid, dms, bucketId, - Timestamp(lid + tsbias))); + Timestamp(lid + timestampBias))); EXPECT_TRUE(assertLid(lid, gid, dms)); } } @@ -577,8 +584,6 @@ TEST("requireThatGidsCanBeSavedAndLoaded") { DocumentMetaStore dms1(createBucketDB()); uint32_t numLids = 1000; - uint32_t bkBits = UINT32_C(20); - uint64_t tsbias = UINT64_C(2000000000000); std::vector<uint32_t> removeLids; removeLids.push_back(10); removeLids.push_back(20); @@ -588,8 +593,8 @@ TEST("requireThatGidsCanBeSavedAndLoaded") for (uint32_t lid = 1; lid <= numLids; ++lid) { GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); - bucketId.setUsedBits(bkBits); - uint32_t addLid = addGid(dms1, gid, bucketId, Timestamp(lid + tsbias)); + bucketId.setUsedBits(numBucketBits); + uint32_t addLid = addGid(dms1, gid, bucketId, Timestamp(lid + timestampBias)); EXPECT_EQUAL(lid, addLid); } for (size_t i = 0; i < removeLids.size(); ++i) { @@ -612,10 +617,10 @@ TEST("requireThatGidsCanBeSavedAndLoaded") for (uint32_t lid = 1; lid <= numLids; ++lid) { GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); - bucketId.setUsedBits(bkBits); + bucketId.setUsedBits(numBucketBits); if (std::count(removeLids.begin(), removeLids.end(), lid) == 0) { EXPECT_TRUE(assertGid(gid, lid, dms2, bucketId, - Timestamp(lid + tsbias))); + Timestamp(lid + timestampBias))); EXPECT_TRUE(assertLid(lid, gid, dms2)); } else { LOG(info, "Lid %u was removed before saving", lid); @@ -629,7 +634,7 @@ TEST("requireThatGidsCanBeSavedAndLoaded") for (size_t i = 0; i < removeLids.size(); ++i) { LOG(info, "Re-use remove lid %u", removeLids[i]); GlobalId gid = createGid(removeLids[i]); - BucketId bucketId(bkBits, + BucketId bucketId(numBucketBits, gid.convertToBucketId().getRawId()); // re-use removeLid[i] uint32_t addLid = addGid(dms2, gid, bucketId, Timestamp(43u + i)); @@ -1870,6 +1875,69 @@ TEST("requireThatShrinkViaFlushTargetWorks") ft->getApproxMemoryGain().getAfter()); } + +namespace { + +void +addLid(DocumentMetaStore &dms, uint32_t lid) +{ + GlobalId gid = createGid(lid); + BucketId bucketId(gid.convertToBucketId()); + bucketId.setUsedBits(numBucketBits); + uint32_t addedLid = addGid(dms, gid, bucketId, Timestamp(lid + timestampBias)); + EXPECT_EQUAL(lid, addedLid); +} + +void +removeLid(DocumentMetaStore &dms, uint32_t lid) +{ + dms.remove(lid); + dms.removeComplete(lid); +} + + +void +assertCompact(DocumentMetaStore &dms, uint32_t docIdLimit, + uint32_t committedDocIdLimit, + uint32_t compactTarget, uint32_t numUsedLids) +{ + EXPECT_TRUE(assertLidSpace(docIdLimit, committedDocIdLimit, numUsedLids, false, false, dms)); + dms.compactLidSpace(compactTarget); + EXPECT_TRUE(assertLidSpace(docIdLimit, compactTarget, numUsedLids, true, false, dms)); + dms.holdUnblockShrinkLidSpace(); + EXPECT_TRUE(assertLidSpace(docIdLimit, compactTarget, numUsedLids, true, true, dms)); +} + + +void +assertShrink(DocumentMetaStore &dms, uint32_t shrinkTarget, + uint32_t numUsedLids) +{ + dms.shrinkLidSpace(); + TEST_DO(EXPECT_TRUE(assertLidSpace(shrinkTarget, shrinkTarget, numUsedLids, false, false, dms))); +} + +} + + +TEST("requireThatSecondShrinkWorksAfterCompactAndInactiveInsert") +{ + DocumentMetaStore dms(createBucketDB()); + dms.constructFreeList(); + TEST_DO(addLid(dms, 1)); + TEST_DO(addLid(dms, 2)); + TEST_DO(addLid(dms, 3)); + removeLid(dms, 2); + removeLid(dms, 3); + EXPECT_TRUE(assertLidSpace(4, 4, 1, false, false, dms)); + TEST_DO(assertCompact(dms, 4, 4, 2, 1)); + TEST_DO(addLid(dms, 2)); + TEST_DO(assertShrink(dms, 3, 2)); + removeLid(dms, 2); + TEST_DO(assertCompact(dms, 3, 3, 2, 1)); + TEST_DO(assertShrink(dms, 2, 1)); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt index 5e333c74b2f..8aab9158529 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_lidreusedelayer_test_app +vespa_add_executable(searchcore_lidreusedelayer_test_app TEST SOURCES lidreusedelayer_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt b/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt index b6d1258a386..0921195e56c 100644 --- a/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt +++ b/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_feed_and_search_test_app +vespa_add_executable(searchcore_feed_and_search_test_app TEST SOURCES feed_and_search.cpp DEPENDS diff --git a/searchcore/src/tests/proton/feedoperation/CMakeLists.txt b/searchcore/src/tests/proton/feedoperation/CMakeLists.txt index fc47a3bde85..613d4b9f603 100644 --- a/searchcore/src/tests/proton/feedoperation/CMakeLists.txt +++ b/searchcore/src/tests/proton/feedoperation/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_feedoperation_test_app +vespa_add_executable(searchcore_feedoperation_test_app TEST SOURCES feedoperation_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/feedtoken/CMakeLists.txt b/searchcore/src/tests/proton/feedtoken/CMakeLists.txt index 328d872f668..438c0b72b1c 100644 --- a/searchcore/src/tests/proton/feedtoken/CMakeLists.txt +++ b/searchcore/src/tests/proton/feedtoken/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_feedtoken_test_app +vespa_add_executable(searchcore_feedtoken_test_app TEST SOURCES feedtoken.cpp DEPENDS diff --git a/searchcore/src/tests/proton/flushengine/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/CMakeLists.txt index 4fc59180946..bbc97818f51 100644 --- a/searchcore/src/tests/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/tests/proton/flushengine/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_flushengine_test_app +vespa_add_executable(searchcore_flushengine_test_app TEST SOURCES flushengine.cpp DEPENDS diff --git a/searchcore/src/tests/proton/flushengine/flushengine.cpp b/searchcore/src/tests/proton/flushengine/flushengine.cpp index 59b86671a0d..add89d5fc58 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine.cpp @@ -15,6 +15,7 @@ LOG_SETUP("flushengine_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/test/insertion_operators.h> #include <memory> // -------------------------------------------------------------------------------- @@ -67,14 +68,63 @@ class SimpleTlsStatsFactory : public flushengine::ITlsStatsFactory } }; +class SimpleHandler; + +class WrappedFlushTask : public searchcorespi::FlushTask +{ + searchcorespi::FlushTask::UP _task; + SimpleHandler &_handler; + +public: + virtual void run() override; + WrappedFlushTask(searchcorespi::FlushTask::UP task, + SimpleHandler &handler) + : _task(std::move(task)), + _handler(handler) + { + } + + virtual search::SerialNum getFlushSerial() const override + { + return _task->getFlushSerial(); + } +}; + +class WrappedFlushTarget : public FlushTargetProxy +{ + SimpleHandler &_handler; +public: + WrappedFlushTarget(const IFlushTarget::SP &target, + SimpleHandler &handler) + : FlushTargetProxy(target), + _handler(handler) + { + } + + virtual Task::UP initFlush(SerialNum currentSerial) override + { + Task::UP task(_target->initFlush(currentSerial)); + if (task) { + return std::make_unique<WrappedFlushTask>(std::move(task), + _handler); + } + return std::move(task); + } +}; + typedef std::vector<IFlushTarget::SP> Targets; +using FlushDoneHistory = std::vector<search::SerialNum>; + class SimpleHandler : public test::DummyFlushHandler { public: Targets _targets; search::SerialNum _oldestSerial; search::SerialNum _currentSerial; + uint32_t _pendingDone; + vespalib::Lock _lock; vespalib::CountDownLatch _done; + FlushDoneHistory _flushDoneHistory; public: typedef std::shared_ptr<SimpleHandler> SP; @@ -85,7 +135,10 @@ public: _targets(targets), _oldestSerial(0), _currentSerial(currentSerial), - _done(targets.size()) + _pendingDone(0u), + _lock(), + _done(targets.size()), + _flushDoneHistory() { // empty } @@ -103,21 +156,54 @@ public: { LOG(info, "SimpleHandler(%s)::getFlushTargets()", getName().c_str()); - return _targets; + std::vector<IFlushTarget::SP> wrappedTargets; + for (const auto &target : _targets) { + wrappedTargets.push_back(std::make_shared<WrappedFlushTarget> + (target, *this)); + } + return std::move(wrappedTargets); + } + + // Called once by flush engine slave thread for each task done + void taskDone() + { + vespalib::LockGuard guard(_lock); + ++_pendingDone; } + // Called by flush engine master thread after flush handler is + // added to flush engine and when one or more flush tasks related + // to flush handler have completed. void flushDone(search::SerialNum oldestSerial) override { + vespalib::LockGuard guard(_lock); LOG(info, "SimpleHandler(%s)::flushDone(%" PRIu64 ")", getName().c_str(), oldestSerial); _oldestSerial = std::max(_oldestSerial, oldestSerial); - _done.countDown(); - } + _flushDoneHistory.push_back(oldestSerial); + while (_pendingDone > 0) { + --_pendingDone; + _done.countDown(); + } + } + FlushDoneHistory getFlushDoneHistory() + { + vespalib::LockGuard guard(_lock); + return _flushDoneHistory; + } }; +void WrappedFlushTask::run() +{ + _task->run(); + _handler.taskDone(); +} + class SimpleTask : public searchcorespi::FlushTask { + search::SerialNum &_flushedSerial; + search::SerialNum &_currentSerial; public: vespalib::Gate &_start; vespalib::Gate &_done; @@ -126,8 +212,11 @@ public: public: SimpleTask(vespalib::Gate &start, vespalib::Gate &done, - vespalib::Gate *proceed) - : _start(start), _done(done), _proceed(proceed) + vespalib::Gate *proceed, + search::SerialNum &flushedSerial, + search::SerialNum ¤tSerial) + : _flushedSerial(flushedSerial), _currentSerial(currentSerial), + _start(start), _done(done), _proceed(proceed) { // empty } @@ -137,6 +226,7 @@ public: if (_proceed != NULL) { _proceed->await(); } + _flushedSerial = _currentSerial; _done.countDown(); } @@ -150,6 +240,7 @@ public: class SimpleTarget : public test::DummyFlushTarget { public: search::SerialNum _flushedSerial; + search::SerialNum _currentSerial; vespalib::Gate _proceed; vespalib::Gate _initDone; vespalib::Gate _taskStart; @@ -162,6 +253,7 @@ public: SimpleTarget(Task::UP task, const std::string &name) : test::DummyFlushTarget(name), _flushedSerial(0), + _currentSerial(0), _proceed(), _initDone(), _taskStart(), @@ -177,7 +269,8 @@ public: _initDone(), _taskStart(), _taskDone(), - _task(new SimpleTask(_taskStart, _taskDone, &_proceed)) + _task(new SimpleTask(_taskStart, _taskDone, &_proceed, + _flushedSerial, _currentSerial)) { if (proceedImmediately) { _proceed.countDown(); @@ -190,8 +283,8 @@ public: virtual SerialNum getFlushedSerialNum() const override { - LOG(info, "SimpleTarget(%s)::getFlushedSerialNum()", - getName().c_str()); + LOG(info, "SimpleTarget(%s)::getFlushedSerialNum() = %" PRIu64, + getName().c_str(), _flushedSerial); return _flushedSerial; } @@ -200,6 +293,7 @@ public: { LOG(info, "SimpleTarget(%s)::initFlush(%" PRIu64 ")", getName().c_str(), currentSerial); + _currentSerial = currentSerial; _initDone.countDown(); return std::move(_task); } @@ -282,6 +376,10 @@ public: if (cached != NULL) { raw = cached->getFlushTarget().get(); } + WrappedFlushTarget *wrapped = dynamic_cast<WrappedFlushTarget *>(raw); + if (wrapped != nullptr) { + raw = wrapped->getFlushTarget().get(); + } for (uint32_t i = 0, len = _targets.size(); i < len; ++i) { if (raw == _targets[i].get()) { LOG(info, "Index of target %p is %d.", raw, i); @@ -340,7 +438,7 @@ struct Fixture Fixture(uint32_t numThreads, uint32_t idleIntervalMS) : tlsStatsFactory(std::make_shared<SimpleTlsStatsFactory>()), strategy(std::make_shared<SimpleStrategy>()), - engine(tlsStatsFactory, strategy, numThreads, idleIntervalMS, false) + engine(tlsStatsFactory, strategy, numThreads, idleIntervalMS) { } }; @@ -398,7 +496,9 @@ TEST_F("require that oldest serial is found", Fixture(1, IINTERVAL)) f.engine.start(); EXPECT_TRUE(handler->_done.await(LONG_TIMEOUT)); - EXPECT_EQUAL(20ul, handler->_oldestSerial); + EXPECT_EQUAL(25ul, handler->_oldestSerial); + FlushDoneHistory handlerFlushDoneHistory(handler->getFlushDoneHistory()); + EXPECT_EQUAL(FlushDoneHistory({ 10, 20, 25 }), handlerFlushDoneHistory); } TEST_F("require that oldest serial is found in group", Fixture(2, IINTERVAL)) @@ -423,9 +523,21 @@ TEST_F("require that oldest serial is found in group", Fixture(2, IINTERVAL)) f.engine.start(); EXPECT_TRUE(fooH->_done.await(LONG_TIMEOUT)); - EXPECT_EQUAL(20ul, fooH->_oldestSerial); + EXPECT_EQUAL(25ul, fooH->_oldestSerial); + // [ 10, 25 ], [ 10, 25, 25 ] and [ 10, 20, 25 ] are legal histories + FlushDoneHistory fooHFlushDoneHistory(fooH->getFlushDoneHistory()); + if (fooHFlushDoneHistory != FlushDoneHistory({ 10, 25 }) && + fooHFlushDoneHistory != FlushDoneHistory({ 10, 25, 25 })) { + EXPECT_EQUAL(FlushDoneHistory({ 10, 20, 25 }), fooHFlushDoneHistory); + } EXPECT_TRUE(barH->_done.await(LONG_TIMEOUT)); - EXPECT_EQUAL(15ul, barH->_oldestSerial); + EXPECT_EQUAL(20ul, barH->_oldestSerial); + // [ 5, 20 ], [ 5, 20, 20 ] and [ 5, 15, 20 ] are legal histories + FlushDoneHistory barHFlushDoneHistory(barH->getFlushDoneHistory()); + if (barHFlushDoneHistory != FlushDoneHistory({ 5, 20 }) && + barHFlushDoneHistory != FlushDoneHistory({ 5, 20, 20 })) { + EXPECT_EQUAL(FlushDoneHistory({ 5, 15, 20 }), barHFlushDoneHistory); + } } TEST_F("require that target can refuse flush", Fixture(2, IINTERVAL)) diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt index a4bff892ffa..d1a2c909a37 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_flushengine_prepare_restart_flush_strategy_test_app +vespa_add_executable(searchcore_flushengine_prepare_restart_flush_strategy_test_app TEST SOURCES prepare_restart_flush_strategy_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index ac3dbb8fed2..b3fd9a050a8 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -72,7 +72,7 @@ public: targetType, flushedSerial, approxDiskBytes); - _result.push_back(std::make_shared<FlushContext>(handler, target, 0, 0)); + _result.push_back(std::make_shared<FlushContext>(handler, target, 0)); return *this; } ContextsBuilder &add(const vespalib::string &handlerName, diff --git a/searchcore/src/tests/proton/index/CMakeLists.txt b/searchcore/src/tests/proton/index/CMakeLists.txt index 46cbf4117ac..be3fff431a9 100644 --- a/searchcore/src/tests/proton/index/CMakeLists.txt +++ b/searchcore/src/tests/proton/index/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_indexmanager_test_app +vespa_add_executable(searchcore_indexmanager_test_app TEST SOURCES indexmanager_test.cpp DEPENDS @@ -9,7 +9,7 @@ vespa_add_executable(searchcore_indexmanager_test_app searchcore_pcommon searchcore_util ) -vespa_add_executable(searchcore_fusionrunner_test_app +vespa_add_executable(searchcore_fusionrunner_test_app TEST SOURCES fusionrunner_test.cpp DEPENDS @@ -18,13 +18,13 @@ vespa_add_executable(searchcore_fusionrunner_test_app searchcore_pcommon searchcore_util ) -vespa_add_executable(searchcore_diskindexcleaner_test_app +vespa_add_executable(searchcore_diskindexcleaner_test_app TEST SOURCES diskindexcleaner_test.cpp DEPENDS searchcore_index ) -vespa_add_executable(searchcore_indexcollection_test_app +vespa_add_executable(searchcore_indexcollection_test_app TEST SOURCES indexcollection_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt b/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt index 88db11dbdba..d588baf7211 100644 --- a/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt +++ b/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_index_writer_test_app +vespa_add_executable(searchcore_index_writer_test_app TEST SOURCES index_writer_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/initializer/CMakeLists.txt b/searchcore/src/tests/proton/initializer/CMakeLists.txt index 0f20324cae3..165e12fb808 100644 --- a/searchcore/src/tests/proton/initializer/CMakeLists.txt +++ b/searchcore/src/tests/proton/initializer/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_task_runner_test_app +vespa_add_executable(searchcore_task_runner_test_app TEST SOURCES task_runner_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/matchengine/CMakeLists.txt b/searchcore/src/tests/proton/matchengine/CMakeLists.txt index 32c0b47ae4b..820fd8a1bcb 100644 --- a/searchcore/src/tests/proton/matchengine/CMakeLists.txt +++ b/searchcore/src/tests/proton/matchengine/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_matchengine_test_app +vespa_add_executable(searchcore_matchengine_test_app TEST SOURCES matchengine.cpp DEPENDS diff --git a/searchcore/src/tests/proton/matching/CMakeLists.txt b/searchcore/src/tests/proton/matching/CMakeLists.txt index 8007ff0344d..ecd9c19eec0 100644 --- a/searchcore/src/tests/proton/matching/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_matching_test_app +vespa_add_executable(searchcore_matching_test_app TEST SOURCES matching_test.cpp DEPENDS @@ -14,7 +14,7 @@ vespa_add_executable(searchcore_matching_test_app searchcore_util ) vespa_add_test(NAME searchcore_matching_test_app COMMAND searchcore_matching_test_app) -vespa_add_executable(searchcore_sessionmanager_test_app +vespa_add_executable(searchcore_sessionmanager_test_app TEST SOURCES sessionmanager_test.cpp DEPENDS @@ -22,14 +22,14 @@ vespa_add_executable(searchcore_sessionmanager_test_app searchcore_grouping ) vespa_add_test(NAME searchcore_sessionmanager_test_app COMMAND searchcore_sessionmanager_test_app) -vespa_add_executable(searchcore_matching_stats_test_app +vespa_add_executable(searchcore_matching_stats_test_app TEST SOURCES matching_stats_test.cpp DEPENDS searchcore_matching ) vespa_add_test(NAME searchcore_matching_stats_test_app COMMAND searchcore_matching_stats_test_app) -vespa_add_executable(searchcore_query_test_app +vespa_add_executable(searchcore_query_test_app TEST SOURCES query_test.cpp DEPENDS @@ -37,21 +37,21 @@ vespa_add_executable(searchcore_query_test_app searchcore_matching ) vespa_add_test(NAME searchcore_query_test_app COMMAND searchcore_query_test_app) -vespa_add_executable(searchcore_termdataextractor_test_app +vespa_add_executable(searchcore_termdataextractor_test_app TEST SOURCES termdataextractor_test.cpp DEPENDS searchcore_matching ) vespa_add_test(NAME searchcore_termdataextractor_test_app COMMAND searchcore_termdataextractor_test_app) -vespa_add_executable(searchcore_resolveviewvisitor_test_app +vespa_add_executable(searchcore_resolveviewvisitor_test_app TEST SOURCES resolveviewvisitor_test.cpp DEPENDS searchcore_matching ) vespa_add_test(NAME searchcore_resolveviewvisitor_test_app COMMAND searchcore_resolveviewvisitor_test_app) -vespa_add_executable(searchcore_querynodes_test_app +vespa_add_executable(searchcore_querynodes_test_app TEST SOURCES querynodes_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/matching/docid_range_scheduler/CMakeLists.txt b/searchcore/src/tests/proton/matching/docid_range_scheduler/CMakeLists.txt index 3892ac41b92..6cba6019fa5 100644 --- a/searchcore/src/tests/proton/matching/docid_range_scheduler/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/docid_range_scheduler/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_docid_range_scheduler_test_app +vespa_add_executable(searchcore_docid_range_scheduler_test_app TEST SOURCES docid_range_scheduler_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt b/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt index 513e002f064..e3b9d94b659 100644 --- a/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_match_loop_communicator_test_app +vespa_add_executable(searchcore_match_loop_communicator_test_app TEST SOURCES match_loop_communicator_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/matching/match_phase_limiter/CMakeLists.txt b/searchcore/src/tests/proton/matching/match_phase_limiter/CMakeLists.txt index 78c16d1435d..54edcf2f61c 100644 --- a/searchcore/src/tests/proton/matching/match_phase_limiter/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/match_phase_limiter/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_match_phase_limiter_test_app +vespa_add_executable(searchcore_match_phase_limiter_test_app TEST SOURCES match_phase_limiter_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/matching/partial_result/CMakeLists.txt b/searchcore/src/tests/proton/matching/partial_result/CMakeLists.txt index 39c1679fc27..8246a35f8fe 100644 --- a/searchcore/src/tests/proton/matching/partial_result/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/partial_result/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_partial_result_test_app +vespa_add_executable(searchcore_partial_result_test_app TEST SOURCES partial_result_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp index 078a6985fc4..00c1ebeab16 100644 --- a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp +++ b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for sessionmanager. -#include <vespa/log/log.h> -LOG_SETUP("sessionmanager_test"); #include <vespa/fastos/fastos.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> @@ -13,6 +11,9 @@ LOG_SETUP("sessionmanager_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/log/log.h> +LOG_SETUP("sessionmanager_test"); + using vespalib::string; using namespace proton; using namespace proton::matching; diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt index bf77c583468..42c2fc29ada 100644 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt +++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentdb_job_trackers_test_app +vespa_add_executable(searchcore_documentdb_job_trackers_test_app TEST SOURCES documentdb_job_trackers_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt index 478a7201228..ae4a6df9373 100644 --- a/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt +++ b/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_job_load_sampler_test_app +vespa_add_executable(searchcore_job_load_sampler_test_app TEST SOURCES job_load_sampler_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt index f4544740f8e..1a1644097be 100644 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt +++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_job_tracked_flush_test_app +vespa_add_executable(searchcore_job_tracked_flush_test_app TEST SOURCES job_tracked_flush_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt b/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt index e50e584e578..1c75b9af82b 100644 --- a/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt +++ b/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_metrics_engine_test_app +vespa_add_executable(searchcore_metrics_engine_test_app TEST SOURCES metrics_engine_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/persistenceengine/CMakeLists.txt b/searchcore/src/tests/proton/persistenceengine/CMakeLists.txt index b5e42ac5075..6317d48bae6 100644 --- a/searchcore/src/tests/proton/persistenceengine/CMakeLists.txt +++ b/searchcore/src/tests/proton/persistenceengine/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_persistenceengine_test_app +vespa_add_executable(searchcore_persistenceengine_test_app TEST SOURCES persistenceengine_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt index 5e17a1d6606..07abb46b3e1 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_reprocessing_initializer_test_app +vespa_add_executable(searchcore_attribute_reprocessing_initializer_test_app TEST SOURCES attribute_reprocessing_initializer_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/CMakeLists.txt b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/CMakeLists.txt index 170e381c99c..50f6d2e55d9 100644 --- a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/CMakeLists.txt +++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_reprocessing_handler_test_app +vespa_add_executable(searchcore_document_reprocessing_handler_test_app TEST SOURCES document_reprocessing_handler_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/CMakeLists.txt b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/CMakeLists.txt index f5eff73f9f0..6e2e0689440 100644 --- a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/CMakeLists.txt +++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_reprocessing_runner_test_app +vespa_add_executable(searchcore_reprocessing_runner_test_app TEST SOURCES reprocessing_runner_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/server/CMakeLists.txt b/searchcore/src/tests/proton/server/CMakeLists.txt index 3ae89e7393d..7b689e33de7 100644 --- a/searchcore/src/tests/proton/server/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_metrics_test_app +vespa_add_executable(searchcore_attribute_metrics_test_app TEST SOURCES attribute_metrics_test.cpp DEPENDS @@ -7,7 +7,7 @@ vespa_add_executable(searchcore_attribute_metrics_test_app searchcore_proton_metrics ) vespa_add_test(NAME searchcore_attribute_metrics_test_app COMMAND searchcore_attribute_metrics_test_app) -vespa_add_executable(searchcore_documentretriever_test_app +vespa_add_executable(searchcore_documentretriever_test_app TEST SOURCES documentretriever_test.cpp DEPENDS @@ -21,7 +21,7 @@ vespa_add_executable(searchcore_documentretriever_test_app searchcore_persistenceengine ) vespa_add_test(NAME searchcore_documentretriever_test_app COMMAND searchcore_documentretriever_test_app) -vespa_add_executable(searchcore_feeddebugger_test_app +vespa_add_executable(searchcore_feeddebugger_test_app TEST SOURCES feeddebugger_test.cpp DEPENDS @@ -29,7 +29,7 @@ vespa_add_executable(searchcore_feeddebugger_test_app searchcore_pcommon ) vespa_add_test(NAME searchcore_feeddebugger_test_app COMMAND searchcore_feeddebugger_test_app) -vespa_add_executable(searchcore_feedstates_test_app +vespa_add_executable(searchcore_feedstates_test_app TEST SOURCES feedstates_test.cpp DEPENDS @@ -42,7 +42,7 @@ vespa_add_executable(searchcore_feedstates_test_app searchcore_fconfig ) vespa_add_test(NAME searchcore_feedstates_test_app COMMAND searchcore_feedstates_test_app) -vespa_add_executable(searchcore_memoryconfigstore_test_app +vespa_add_executable(searchcore_memoryconfigstore_test_app TEST SOURCES memoryconfigstore_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt b/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt index 9c8048a0a69..ffcc7a82d2e 100644 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_data_directory_upgrader_test_app +vespa_add_executable(searchcore_data_directory_upgrader_test_app TEST SOURCES data_directory_upgrader_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt b/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt index 1d9b0234d76..48f64db5d82 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_disk_mem_usage_filter_test_app +vespa_add_executable(searchcore_disk_mem_usage_filter_test_app TEST SOURCES disk_mem_usage_filter_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt b/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt index 2fee205f636..b4dab3d9f54 100644 --- a/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_health_adapter_test_app +vespa_add_executable(searchcore_health_adapter_test_app TEST SOURCES health_adapter_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt b/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt index 51ea36dc077..aad23ab7b0a 100644 --- a/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_memoryflush_test_app +vespa_add_executable(searchcore_memoryflush_test_app TEST SOURCES memoryflush_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp index 2f4083228f9..6c235766de6 100644 --- a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp +++ b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp @@ -105,7 +105,7 @@ public: return *this; } ContextBuilder &add(const IFlushTarget::SP &target, SerialNum lastSerial = 0) { - FlushContext::SP ctx(new FlushContext(_handler, target, 0, lastSerial)); + FlushContext::SP ctx(new FlushContext(_handler, target, lastSerial)); return add(ctx); } const FlushContext::List &list() const { return _list; } @@ -282,22 +282,22 @@ requireThatWeCanOrderByTlsSize() (handler1, createTargetT("t2", TimeStamp(now.val() - 10 * TimeStamp::SEC), 1900), - 2000, 2000)). + 2000)). add(std::make_shared<FlushContext> (handler2, createTargetT("t1", TimeStamp(now.val() - 5 * TimeStamp::SEC), 1000), - 2000, 2000)). + 2000)). add(std::make_shared<FlushContext> (handler1, createTargetT("t4", TimeStamp(), 1000), - 2000, 2000)). + 2000)). add(std::make_shared<FlushContext> (handler2, createTargetT("t3", TimeStamp(now.val() - 15 * TimeStamp::SEC), 1900), - 2000, 2000)); + 2000)); { // sum of tls sizes above limit, trigger sort order based on tls size MemoryFlush flush({1000, 3 * gibi, 1.0, 1000, 1.0, 2000, TimeStamp(2 * TimeStamp::SEC)}, start); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t1").add("t2").add("t3"), diff --git a/searchcore/src/tests/proton/server/visibility_handler/CMakeLists.txt b/searchcore/src/tests/proton/server/visibility_handler/CMakeLists.txt index f86504c84dc..247bda1f5f8 100644 --- a/searchcore/src/tests/proton/server/visibility_handler/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/visibility_handler/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_visibility_handler_test_app +vespa_add_executable(searchcore_visibility_handler_test_app TEST SOURCES visibility_handler_test.cpp DEPENDS diff --git a/searchcore/src/tests/proton/statusreport/CMakeLists.txt b/searchcore/src/tests/proton/statusreport/CMakeLists.txt index fa11b343d0d..98462c157fc 100644 --- a/searchcore/src/tests/proton/statusreport/CMakeLists.txt +++ b/searchcore/src/tests/proton/statusreport/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_statusreport_test_app +vespa_add_executable(searchcore_statusreport_test_app TEST SOURCES statusreport.cpp DEPENDS diff --git a/searchcore/src/tests/proton/summaryengine/CMakeLists.txt b/searchcore/src/tests/proton/summaryengine/CMakeLists.txt index af3b09db1de..d3699428a9d 100644 --- a/searchcore/src/tests/proton/summaryengine/CMakeLists.txt +++ b/searchcore/src/tests/proton/summaryengine/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_summaryengine_test_app +vespa_add_executable(searchcore_summaryengine_test_app TEST SOURCES summaryengine.cpp DEPENDS diff --git a/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt b/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt index 2d74c323c1a..a3a03ea8849 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt +++ b/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_verify_ranksetup_test_app +vespa_add_executable(searchcore_verify_ranksetup_test_app TEST SOURCES verify_ranksetup_test.cpp DEPENDS diff --git a/searchcore/src/vespa/searchcore/common/OWNERS b/searchcore/src/vespa/searchcore/common/OWNERS index 1037590124e..9dc0c2d970d 100644 --- a/searchcore/src/vespa/searchcore/common/OWNERS +++ b/searchcore/src/vespa/searchcore/common/OWNERS @@ -1 +1 @@ -balder +baldersheim diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index b55f29ff810..89e15952a42 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -57,26 +57,26 @@ flush.idleinterval double default=10.0 restart flush.strategy enum {SIMPLE, MEMORY} default=MEMORY restart ## Total number of bytes allowed before forcing flush. -flush.memory.maxmemory long default=4294967296 restart +flush.memory.maxmemory long default=4294967296 ## Maximum total disk bloat factor before forcing flush. -flush.memory.diskbloatfactor double default=0.2 restart +flush.memory.diskbloatfactor double default=0.2 ## Max disk usage (in bytes) for all transaction logs before forcing flush. -flush.memory.maxtlssize long default=21474836480 restart +flush.memory.maxtlssize long default=21474836480 ## Number of bytes allowed per component before forcing memory prioritization. -flush.memory.each.maxmemory long default=1073741824 restart +flush.memory.each.maxmemory long default=1073741824 ## Maximum disk bloat factor per component before forcing flush. -flush.memory.each.diskbloatfactor double default=0.2 restart +flush.memory.each.diskbloatfactor double default=0.2 ## Age of unflushed content before forcing age prioritization. ## Unit is seconds with 1 day being the default. -flush.memory.maxage.time double default=86400.0 restart +flush.memory.maxage.time double default=86400.0 ## Max diff in serial number allowed before that takes precedence. -flush.memory.maxage.serial long default=1000000 restart +flush.memory.maxage.serial long default=1000000 ## The cost of doing replay when replaying the transaction log. ## @@ -86,7 +86,7 @@ flush.memory.maxage.serial long default=1000000 restart ## The prepare for restart flush strategy will choose a set of components to flush ## such that the cost of flushing these + the cost of replaying the transaction log ## is as low as possible. -flush.preparerestart.replaycost double default=4.0 restart +flush.preparerestart.replaycost double default=4.0 ## The cost of doing writes when flushing components to disk. ## @@ -96,7 +96,7 @@ flush.preparerestart.replaycost double default=4.0 restart ## The prepare for restart flush strategy will choose a set of components to flush ## such that the cost of flushing these + the cost of replaying the transaction log ## is as low as possible. -flush.preparerestart.writecost double default=1.0 restart +flush.preparerestart.writecost double default=1.0 ## Control io options during write both under dump and fusion. indexing.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart diff --git a/searchcore/src/vespa/searchcore/fdispatch/OWNERS b/searchcore/src/vespa/searchcore/fdispatch/OWNERS index 3e9fc8ab356..f4d47806ed9 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/OWNERS +++ b/searchcore/src/vespa/searchcore/fdispatch/OWNERS @@ -1,2 +1,2 @@ -balder -tegge +baldersheim +toregge diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp index 193c4365189..6e06c91a902 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp @@ -31,18 +31,12 @@ using document::CompressionConfig; char FastS_VersionTag[] = V_TAG; -int FastS_verbose = 0; -/** @todo Use a config file for these variables */ -int FastS_nsearches; -double FastS_searchtime; - - namespace fdispatch { FastS_FNETAdapter::FastS_FNETAdapter(FastS_AppContext *appCtx) : _appCtx(appCtx), - _nodeManager(NULL), + _nodeManager(), _timeKeeper(NULL), _transport(NULL), _last_now(0.0), @@ -92,57 +86,57 @@ FastS_FNETAdapter::fini() Fdispatch::~Fdispatch(void) { - if (_transportServer != NULL) { + if (_transportServer) { _transportServer->shutDown(); // sync shutdown } _FNET_adapter.fini(); - if (_nodeManager != NULL) + if (_nodeManager) { _nodeManager->ShutdownConfig(); - if (_transport != NULL && _transportStarted) + } + if (_transport && _transportStarted) { _transport->ShutDown(true); // sync shutdown - if (_rpc != NULL) + } + if (_rpc) { _rpc->ShutDown(); // sync shutdown + } LOG(debug, "Will close threadpool"); _mypool->Close(); LOG(debug, "Has closed threadpool"); - delete _transportServer; - delete _engineAdapter; - delete _nodeManager; - if (_transport != NULL) { - delete _transport; - } - delete _rpc; - delete _mypool; + _transportServer.reset(); + _engineAdapter.reset(); + _nodeManager.reset(); + _transport.reset(); + _rpc.reset(); + _mypool.reset(); } FNET_Transport * Fdispatch::GetFNETTransport() { - return _transport; + return _transport.get(); } FNET_Scheduler * Fdispatch::GetFNETScheduler() { - return (_transport == NULL) ? - NULL : _transport->GetScheduler(); + return ( ! _transport) ? NULL : _transport->GetScheduler(); } FastS_NodeManager * Fdispatch::GetNodeManager() { - return _nodeManager; + return _nodeManager.get(); } FastS_DataSetCollection * Fdispatch::GetDataSetCollection() { - if (_nodeManager == NULL) + if ( ! _nodeManager) return NULL; return _nodeManager->GetDataSetCollection(); } @@ -151,14 +145,14 @@ Fdispatch::GetDataSetCollection() FastOS_ThreadPool * Fdispatch::GetThreadPool() { - return _mypool; + return _mypool.get(); } bool Fdispatch::Failed(void) { - return ( (_transportServer != NULL && _transportServer->isFailed())); + return ( (_transportServer && _transportServer->isFailed())) || _needRestart; } @@ -204,7 +198,7 @@ Fdispatch::CheckTempFail(void) if (failflag == _tempFail) return ret; - if (_transportServer != NULL) { + if (_transportServer) { if (failflag) { _transportServer->setListen(false); LOG(error, "Disabling fnet server interface"); @@ -223,16 +217,18 @@ Fdispatch::CheckTempFail(void) * Set up stuff as specified in the fdispatch-rc-file. */ Fdispatch::Fdispatch(const config::ConfigUri &configUri) - : _mypool(NULL), - _engineAdapter(NULL), - _transportServer(NULL), + : _mypool(), + _engineAdapter(), + _transportServer(), _componentConfig(), - _nodeManager(NULL), - _transport(NULL), + _nodeManager(), + _transport(), _FNET_adapter(this), - _rpc(NULL), + _rpc(), _config(), _configUri(configUri), + _fdispatchrcFetcher(configUri.getContext()), + _rndGen(), _partition(0), _tempFail(false), _FNETLiveCounterDanger(false), @@ -243,32 +239,51 @@ Fdispatch::Fdispatch(const config::ConfigUri &configUri) _FNETLiveCounterDangerStart(), _timeouts(0u), _checkLimit(0u), - _healthPort(0) + _healthPort(0), + _needRestart(false) { int64_t cfgGen = -1; _config = config::ConfigGetter<FdispatchrcConfig>:: - getConfig(cfgGen, - _configUri.getConfigId(), - _configUri.getContext()); + getConfig(cfgGen, _configUri.getConfigId(), _configUri.getContext()); LOG(config, "fdispatch version %s (RPC-port: %d, transport at %d)", FastS_VersionTag, _config->frtport, _config->ptport); _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("fdispatch", cfgGen, "config only obtained at startup")); + _fdispatchrcFetcher.subscribe<FdispatchrcConfig>(configUri.getConfigId(), this); + _fdispatchrcFetcher.start(); } +namespace { -void -Fdispatch::CheckCacheMaxEntries(unsigned int queryCacheMaxEntries, - unsigned int queryAttrCacheMaxEntries) +bool needRestart(const FdispatchrcConfig & curr, const FdispatchrcConfig & next) { - if (queryAttrCacheMaxEntries == 0) - return; + if (curr.frtport != next.frtport) { + LOG(error, "FRT port has changed from %d to %d.", curr.frtport, next.frtport); + return true; + } + if (curr.ptport != next.ptport) { + LOG(error, "PT port has changed from %d to %d.", curr.ptport, next.ptport); + return true; + } + if (curr.healthport != next.healthport) { + LOG(error, "Health port has changed from %d to %d.", curr.healthport, next.healthport); + return true; + } + return false; +} + +} - if ((queryAttrCacheMaxEntries <= queryCacheMaxEntries) || - (queryCacheMaxEntries == 0)) { - FastS_abort("Please edit fdispatchrc such that " - "queryattrcachequeries > querycachequeries."); +void Fdispatch::configure(std::unique_ptr<FdispatchrcConfig> cfg) +{ + if (cfg && _config) { + if ( needRestart(*_config, *cfg) ) { + const int sleepMS = (1.0 + 30 * _rndGen.nextDouble()) * 1000; + LOG(error, "Will restart by abort in %d ms.", sleepMS); + std::this_thread::sleep_for(std::chrono::milliseconds(sleepMS)); + _needRestart.store(true); + } } } @@ -298,16 +313,13 @@ Fdispatch::Init(void) _timeouts = 0; _checkLimit = 60; - FS4PersistentPacketStreamer::Instance.SetCompressionLimit( - _config->packetcompresslimit); - FS4PersistentPacketStreamer::Instance.SetCompressionLevel( - _config->packetcompresslevel); - FS4PersistentPacketStreamer::Instance.SetCompressionType( - convert(_config->packetcompresstype)); + FS4PersistentPacketStreamer::Instance.SetCompressionLimit(_config->packetcompresslimit); + FS4PersistentPacketStreamer::Instance.SetCompressionLevel(_config->packetcompresslevel); + FS4PersistentPacketStreamer::Instance.SetCompressionType(convert(_config->packetcompresstype)); LOG(debug, "Creating FNET transport"); - _transport = new FNET_Transport(_config->transportthreads); + _transport = std::make_unique<FNET_Transport>(_config->transportthreads); // grab node slowness limit defaults @@ -317,22 +329,20 @@ Fdispatch::Init(void) FastS_DataSetDesc::SetDefaultSlowDocsumLimitBias(_config->defaultslowdocsumlimitbias); maxthreads = _config->maxthreads; - _mypool = new FastOS_ThreadPool(256 * 1024, maxthreads); + _mypool = std::make_unique<FastOS_ThreadPool>(256 * 1024, maxthreads); // Max interval betw read from socket. FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent] = _config->maxsocksilent; - if (_transport != NULL) + if (_transport) { _transport->SetIOCTimeOut((uint32_t) (FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent] * 1000.0)); + } char timestr[40]; - FastS_TimeOut::WriteTime(timestr, sizeof(timestr), - FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent]); - LOG(debug, - "VERBOSE: Max time between successful read from a socket: %s", - timestr); + FastS_TimeOut::WriteTime(timestr, sizeof(timestr), FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent]); + LOG(debug, "VERBOSE: Max time between successful read from a socket: %s", timestr); FastS_QueryCacheUtil::_systemMaxHits = std::numeric_limits<int>::max(); LOG(debug, "VERBOSE: maxhits: %d", FastS_QueryCacheUtil::_systemMaxHits); @@ -341,9 +351,7 @@ Fdispatch::Init(void) const uint32_t linesize = 1; if (FastS_QueryCacheUtil::_systemMaxHits < linesize && FastS_QueryCacheUtil::_maxOffset < linesize - FastS_QueryCacheUtil::_systemMaxHits) { - LOG(warning, - "maxoffset must be >= %d! (overriding config value)", - linesize - FastS_QueryCacheUtil::_systemMaxHits); + LOG(warning, "maxoffset must be >= %d! (overriding config value)", linesize - FastS_QueryCacheUtil::_systemMaxHits); FastS_QueryCacheUtil::_maxOffset = linesize - FastS_QueryCacheUtil::_systemMaxHits; } LOG(debug, "VERBOSE: maxoffset: %d", FastS_QueryCacheUtil::_maxOffset); @@ -354,25 +362,21 @@ Fdispatch::Init(void) LOG(debug, "Using port number %d", ptportnum); - _nodeManager = new FastS_NodeManager(_componentConfig, this, _partition); - + _nodeManager = std::make_unique<FastS_NodeManager>(_componentConfig, this, _partition); GetFNETTransport()->SetTCPNoDelay(_config->transportnodelay); GetFNETTransport()->SetDirectWrite(_config->transportdirectwrite); assert (ptportnum != 0); - _engineAdapter = new fdispatch::EngineAdapter(this, _mypool); - _transportServer = new search::engine::TransportServer - (*_engineAdapter, *_engineAdapter, *_engineAdapter, ptportnum, search::engine::TransportServer::DEBUG_ALL); + _engineAdapter = std::make_unique<fdispatch::EngineAdapter>(this, _mypool.get()); + _transportServer = std::make_unique<TransportServer>(*_engineAdapter, *_engineAdapter, *_engineAdapter, ptportnum, search::engine::TransportServer::DEBUG_ALL); _transportServer->setTCPNoDelay(_config->transportnodelay); _transportServer->setDirectWrite(_config->transportdirectwrite); if (!_transportServer->start()) { - delete _transportServer; - _transportServer = NULL; - delete _engineAdapter; - _engineAdapter = NULL; + _transportServer.reset(); + _engineAdapter.reset(); LOG(error, "CRITICAL: Failed to init upwards FNET transport on port %d", ptportnum); return false; } @@ -380,21 +384,19 @@ Fdispatch::Init(void) _nodeManager->SubscribePartMap(_configUri); if (_config->frtport != 0) { - _rpc = new FastS_fdispatch_RPC(this); - FastS_assert(_rpc != NULL); + _rpc = std::make_unique<FastS_fdispatch_RPC>(this); if (!_rpc->Init(_config->frtport, _configUri.getConfigId())) { LOG(error, "RPC init failed"); - delete _rpc; - _rpc = NULL; + _rpc.reset(); } } else { - _rpc = NULL; + _rpc.reset(); } // Kick off fdispatch administrative threads. - if (_transport != NULL) { + if (_transport) { _FNET_adapter.init(); - bool rc = _transport->Start(_mypool); + bool rc = _transport->Start(_mypool.get()); if (rc) { LOG(debug, "Started FNET transport"); _transportStarted = true; @@ -403,7 +405,7 @@ Fdispatch::Init(void) } } FastOS_Thread::Sleep(1000); - if (_rpc != NULL) { + if (_rpc) { _rpc->Start(); } _healthPort = _config->healthport; diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h index b4ff8d9f37e..f4fa80a77ab 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h @@ -10,6 +10,7 @@ #include <vespa/searchcore/config/config-fdispatchrc.h> #include <vespa/config/subscription/configuri.h> #include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/util/random.h> class FastS_NodeManager; class FastS_fdispatch_RPC; @@ -53,33 +54,40 @@ public: /** * Note: There is only one instance of this. */ -class Fdispatch : public FastS_AppContext +class Fdispatch : public FastS_AppContext, + public config::IFetcherCallback<vespa::config::search::core::FdispatchrcConfig> { private: + typedef search::engine::TransportServer TransportServer; + typedef vespa::config::search::core::FdispatchrcConfig FdispatchrcConfig; Fdispatch(const Fdispatch &); Fdispatch& operator=(const Fdispatch &); - FastOS_ThreadPool *_mypool; - EngineAdapter *_engineAdapter; - search::engine::TransportServer *_transportServer; + std::unique_ptr<FastOS_ThreadPool> _mypool; + std::unique_ptr<EngineAdapter> _engineAdapter; + std::unique_ptr<TransportServer> _transportServer; vespalib::SimpleComponentConfigProducer _componentConfig; - FastS_NodeManager *_nodeManager; - FNET_Transport *_transport; - FastS_FNETAdapter _FNET_adapter; - FastS_fdispatch_RPC *_rpc; - std::unique_ptr<vespa::config::search::core::FdispatchrcConfig> _config; - config::ConfigUri _configUri; + std::unique_ptr<FastS_NodeManager> _nodeManager; + std::unique_ptr<FNET_Transport> _transport; + FastS_FNETAdapter _FNET_adapter; + std::unique_ptr<FastS_fdispatch_RPC> _rpc; + std::unique_ptr<FdispatchrcConfig> _config; + config::ConfigUri _configUri; + config::ConfigFetcher _fdispatchrcFetcher; + vespalib::RandomGen _rndGen; unsigned int _partition; - bool _tempFail; - bool _FNETLiveCounterDanger; - bool _FNETLiveCounterWarned; - bool _FNETLiveCounterFailed; - bool _transportStarted; + bool _tempFail; + bool _FNETLiveCounterDanger; + bool _FNETLiveCounterWarned; + bool _FNETLiveCounterFailed; + bool _transportStarted; unsigned int _lastFNETLiveCounter; - FastOS_Time _FNETLiveCounterDangerStart; + FastOS_Time _FNETLiveCounterDangerStart; unsigned int _timeouts; unsigned int _checkLimit; - int _healthPort; + int _healthPort; + std::atomic<bool> _needRestart; + void configure(std::unique_ptr<FdispatchrcConfig> cfg); public: // Implements FastS_AppContext virtual FNET_Transport *GetFNETTransport(); @@ -95,10 +103,6 @@ public: int getHealthPort() const { return _healthPort; } vespalib::SimpleComponentConfigProducer &getComponentConfig() { return _componentConfig; } - void - CheckCacheMaxEntries(unsigned int queryCacheMaxEntries, - unsigned int queryAttrCacheMaxEntries); - Fdispatch(const config::ConfigUri &configUri); ~Fdispatch(void); }; diff --git a/searchcore/src/vespa/searchcore/grouping/OWNERS b/searchcore/src/vespa/searchcore/grouping/OWNERS index 9bbc3a73836..a59a5ddcdbf 100644 --- a/searchcore/src/vespa/searchcore/grouping/OWNERS +++ b/searchcore/src/vespa/searchcore/grouping/OWNERS @@ -1,2 +1,2 @@ -balder +baldersheim havardpe diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS b/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS index 7c03446a5d4..614d678c95e 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS @@ -1,2 +1,2 @@ geirst -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 7b66a4b0105..ea26ef20788 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -178,9 +178,8 @@ DocumentMetaStore::insert(DocId lid, metaData.getBucketId().stripUnused(), metaData.getTimestamp(), _subDbType); - if (state.isActive()) { - _lidAlloc.markAsActive(lid); - } + _lidAlloc.updateActiveLids(lid, state.isActive()); + _lidAlloc.commitActiveLids(); updateCommittedDocIdLimit(); return true; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 2e22b14eccc..244fb940f13 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -90,16 +90,6 @@ LidAllocator::ensureSpace(DocId lid, } void -LidAllocator::markAsActive(DocId lid) -{ - if (_activeLids.get(lid) == 0) { - ++_numActiveLids; - } - _activeLids.update(lid, 1); - _activeLids.commit(); -} - -void LidAllocator::unregisterLid(DocId lid) { assert(!_pendingHoldLids.testBit(lid)); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index 0922f9d4edd..836d4c12cbb 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -45,7 +45,6 @@ public: uint32_t newSize, uint32_t newCapacity); void registerLid(DocId lid) { _usedLids.setBit(lid); } - void markAsActive(DocId lid); void unregisterLid(DocId lid); size_t getUsedLidsSize() const; void trimHoldLists(generation_t firstUsed); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/OWNERS b/searchcore/src/vespa/searchcore/proton/flushengine/OWNERS index 1037590124e..9dc0c2d970d 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/flushengine/OWNERS @@ -1 +1 @@ -balder +baldersheim diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp index 7f436a8d594..1624bc3b51c 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp @@ -10,13 +10,11 @@ namespace proton { FlushContext::FlushContext( const IFlushHandler::SP &handler, const IFlushTarget::SP &target, - search::SerialNum oldestFlushable, search::SerialNum lastSerial) : _name(createName(*handler, *target)), _handler(handler), _target(target), _task(), - _oldestFlushable(oldestFlushable), _lastSerial(lastSerial) { // empty diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h index 9f2b557b3a9..837a99f153e 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h @@ -19,7 +19,6 @@ private: IFlushHandler::SP _handler; IFlushTarget::SP _target; searchcorespi::FlushTask::UP _task; - search::SerialNum _oldestFlushable; search::SerialNum _lastSerial; public: @@ -43,7 +42,6 @@ public: */ FlushContext(const IFlushHandler::SP &handler, const IFlushTarget::SP &target, - search::SerialNum oldestFlushable, search::SerialNum lastSerial); /** @@ -83,13 +81,6 @@ public: const IFlushTarget::SP & getTarget() const { return _target; } /** - * Returns the oldest flushable serial number. - * - * @return The oldest flushable serial number - */ - search::SerialNum getOldestFlushable() const { return _oldestFlushable; } - - /** * Returns the last serial number. * * @return The last serial number diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index b1a2e23fca0..9d5cf4c006f 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -22,16 +22,13 @@ namespace { search::SerialNum findOldestFlushedSerial(const IFlushTarget::List &lst, - const IFlushHandler &handler, - const IFlushTarget *self) + const IFlushHandler &handler) { search::SerialNum ret(handler.getCurrentSerialNumber()); for (const IFlushTarget::SP & target : lst) { - if (self != target.get()) { - ret = std::min(ret, target->getFlushedSerialNum()); - } + ret = std::min(ret, target->getFlushedSerialNum()); } - LOG(debug, "Oldest flushed serial for '%s' will be %" PRIu64 " after flush.", handler.getName().c_str(), ret); + LOG(debug, "Oldest flushed serial for '%s' is %" PRIu64 ".", handler.getName().c_str(), ret); return ret; } @@ -55,11 +52,10 @@ FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory, IFlushStrategy::SP strategy, uint32_t numThreads, - uint32_t idleIntervalMS, bool enableAutoPrune) + uint32_t idleIntervalMS) : _closed(false), _maxConcurrent(numThreads), _idleIntervalMS(idleIntervalMS), - _enableAutoPrune(enableAutoPrune), _taskId(0), _threadPool(128 * 1024), _strategy(strategy), @@ -70,7 +66,8 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> _flushing(), _strategyLock(), _strategyMonitor(), - _tlsStatsFactory(tlsStatsFactory) + _tlsStatsFactory(tlsStatsFactory), + _pendingPrune() { // empty } @@ -93,8 +90,8 @@ FlushEngine::start() FlushEngine & FlushEngine::close() { - MonitorGuard strategyGuard(_strategyMonitor); { + MonitorGuard strategyGuard(_strategyMonitor); MonitorGuard guard(_monitor); _closed = true; guard.broadcast(); @@ -128,10 +125,10 @@ bool FlushEngine::wait(size_t minimumWaitTimeIfReady) { MonitorGuard guard(_monitor); - if ( (minimumWaitTimeIfReady > 0) && canFlushMore(guard)) { + if ( (minimumWaitTimeIfReady > 0) && canFlushMore(guard) && _pendingPrune.empty()) { guard.wait(minimumWaitTimeIfReady); } - while ( ! canFlushMore(guard) ) { + while ( ! canFlushMore(guard) && _pendingPrune.empty()) { guard.wait(1000); // broadcast when flush done } return !_closed; @@ -146,6 +143,9 @@ FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg) vespalib::string prevFlushName; while (wait(shouldIdle ? _idleIntervalMS : 0)) { shouldIdle = false; + if (prune()) { + continue; // Prune attempted on one or more handlers + } prevFlushName = flushNextTarget(prevFlushName); if ( ! prevFlushName.empty()) { // Sleep at least 10 ms after a successful flush in order to avoid busy loop in case @@ -154,23 +154,26 @@ FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg) } else { shouldIdle = true; } - if (_enableAutoPrune) { - prune(); - } LOG(debug, "Making another wait(idle=%s, timeMS=%d) last was '%s'", shouldIdle ? "true" : "false", shouldIdle ? _idleIntervalMS : 0, prevFlushName.c_str()); } } -void FlushEngine::prune() +bool +FlushEngine::prune() { - if (_flushing.empty()) { + std::set<IFlushHandler::SP> toPrune; + { MonitorGuard guard(_monitor); - for (const auto & it : _handlers) { - IFlushHandler & handler(*it.second); - IFlushTarget::List lst = handler.getFlushTargets(); - handler.flushDone(findOldestFlushedSerial(lst, handler, NULL)); + if (_pendingPrune.empty()) { + return false; } + _pendingPrune.swap(toPrune); } + for (const auto &handler : toPrune) { + IFlushTarget::List lst = handler->getFlushTargets(); + handler->flushDone(findOldestFlushedSerial(lst, *handler)); + } + return true; } bool FlushEngine::isFlushing(const MonitorGuard & guard, const vespalib::string & name) const @@ -201,7 +204,6 @@ FlushEngine::getTargetList(bool includeFlushingTargets) const if (!isFlushing(guard, FlushContext::createName(handler, *target)) || includeFlushingTargets) { ret.push_back(FlushContext::SP(new FlushContext(it.second, IFlushTarget::SP(new CachedFlushTarget(target)), - findOldestFlushedSerial(lst, handler, target.get()), serial))); } else { LOG(debug, "Target '%s' with flushedSerialNum = %ld already has a flush going. Local last serial = %ld.", @@ -263,7 +265,7 @@ FlushEngine::flushAll(const FlushContext::List &lst) ctx->getName().c_str(), ctx->getTarget()->getFlushedSerialNum() + 1, ctx->getHandler()->getCurrentSerialNumber()); - _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx, ctx->getOldestFlushable()))); + _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx))); } else { LOG(debug, "Target '%s' failed to initiate flush of transactions %" PRIu64 " through %" PRIu64 ".", ctx->getName().c_str(), @@ -303,7 +305,7 @@ FlushEngine::flushNextTarget(const vespalib::string & name) name.c_str(), lst.first.size()); FastOS_Thread::Sleep(1000); } - _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx, ctx->getOldestFlushable()))); + _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx))); return ctx->getName(); } @@ -340,6 +342,8 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId) LOG(debug, "FlushEngine::flushDone(taskId='%d') took '%f' secs", taskId, duration.sec()); MonitorGuard guard(_monitor); _flushing.erase(taskId); + assert(ctx.getHandler()); + _pendingPrune.insert(ctx.getHandler()); guard.broadcast(); } @@ -348,7 +352,12 @@ FlushEngine::putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler::SP &flushHandler) { MonitorGuard guard(_monitor); - return _handlers.putHandler(docTypeName, flushHandler); + IFlushHandler::SP result(_handlers.putHandler(docTypeName, flushHandler)); + if (result) { + _pendingPrune.erase(result); + } + _pendingPrune.insert(flushHandler); + return std::move(result); } IFlushHandler::SP @@ -362,7 +371,9 @@ IFlushHandler::SP FlushEngine::removeFlushHandler(const DocTypeName &docTypeName) { MonitorGuard guard(_monitor); - return _handlers.removeHandler(docTypeName); + IFlushHandler::SP result(_handlers.removeHandler(docTypeName)); + _pendingPrune.erase(result); + return std::move(result); } FlushEngine::FlushMetaSet @@ -395,6 +406,7 @@ FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP void FlushEngine::setStrategy(IFlushStrategy::SP strategy) { + vespalib::LockGuard strategyLock(_strategyLock); MonitorGuard strategyGuard(_strategyMonitor); if (_closed) { return; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 215a8ac8de5..d8d77c11b8f 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -49,7 +49,6 @@ private: bool _closed; const uint32_t _maxConcurrent; const uint32_t _idleIntervalMS; - const bool _enableAutoPrune; uint32_t _taskId; FastOS_ThreadPool _threadPool; IFlushStrategy::SP _strategy; @@ -61,13 +60,14 @@ private: vespalib::Lock _strategyLock; // serialize setStrategy calls vespalib::Monitor _strategyMonitor; std::shared_ptr<flushengine::ITlsStatsFactory> _tlsStatsFactory; + std::set<IFlushHandler::SP> _pendingPrune; FlushContext::List getTargetList(bool includeFlushingTargets) const; std::pair<FlushContext::List,bool> getSortedTargetList(vespalib::MonitorGuard &strategyGuard) const; FlushContext::SP initNextFlush(const FlushContext::List &lst); vespalib::string flushNextTarget(const vespalib::string & name); void flushAll(const FlushContext::List &lst); - void prune(); + bool prune(); uint32_t initFlush(const FlushContext &ctx); uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target); void flushDone(const FlushContext &ctx, uint32_t taskId); @@ -93,15 +93,10 @@ public: * @param strategy The flushing strategy to use. * @param numThreads The number of worker threads to use. * @param idleInterval The interval between when flushes are checked whne there are no one progressing. - * @param enableAutoPrune Indicate if pruning shall be done even if there - are no flushing happening. Turn off for some tests. - Needed for pruning to be correct if one flush is started - while another is in progress. In that case the pruning - will be too conservative. */ FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory, - IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS, bool enableAutoPrune); + IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS); /** * Destructor. Waits for all pending tasks to complete. diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp index fc692979753..8af59d3079f 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp @@ -9,12 +9,10 @@ namespace proton { FlushTask::FlushTask(uint32_t taskId, FlushEngine &engine, - const FlushContext::SP &ctx, - search::SerialNum serial) + const FlushContext::SP &ctx) : _taskId(taskId), _engine(engine), - _context(ctx), - _serial(serial) + _context(ctx) { LOG_ASSERT(_context.get() != NULL); } @@ -34,7 +32,6 @@ FlushTask::run() } task->run(); task.reset(); - _context->getHandler()->flushDone(_serial); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h index fd27538ca4b..c10fb740410 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h @@ -25,13 +25,10 @@ public: * @param taskId The identifier used by IFlushStrategy. * @param engine The running flush engine. * @param ctx The context of the flush to perform. - * @param serial The oldest unflushed serial available in the handler once - * this task has been run. */ FlushTask(uint32_t taskId, FlushEngine &engine, - const FlushContext::SP &ctx, - search::SerialNum serial); + const FlushContext::SP &ctx); /** * Destructor. Notifies the engine that the flush is done to prevent the diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h index 6912cbfde13..2d3eddf7af9 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h @@ -76,9 +76,10 @@ public: * up to the given serial number can be pruned from the domain of this * handler. This method is called by an arbitrary worker thread. * - * @param oldestSerial The oldest transaction that is still in use. + * @param flushedSerial Serial number flushed for all flush + * targets belonging to this handler. */ - virtual void flushDone(SerialNum oldestSerial) = 0; + virtual void flushDone(SerialNum flushedSerial) = 0; /* * This method is called to sync tls to stable media, up to and diff --git a/searchcore/src/vespa/searchcore/proton/index/OWNERS b/searchcore/src/vespa/searchcore/proton/index/OWNERS index 64735d11d93..1708c0d4695 100644 --- a/searchcore/src/vespa/searchcore/proton/index/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/index/OWNERS @@ -1 +1 @@ -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp index 606312fcae2..06a20e8da78 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp @@ -12,14 +12,16 @@ TransLogServerMetrics::DomainMetrics::DomainMetrics(metrics::MetricSet *parent, const vespalib::string &documentType) : metrics::MetricSet("transactionlog", {{"documenttype", documentType}}, "Transaction log metrics for a document type", parent), - entries("entries", "", "The current number of entries in the transaction log", this) + entries("entries", "", "The current number of entries in the transaction log", this), + diskUsage("disk_usage", "", "The disk usage (in bytes) of the transaction log", this) { } void TransLogServerMetrics::DomainMetrics::update(const DomainInfo &stats) { - entries.set(stats.count); + entries.set(stats.numEntries); + diskUsage.set(stats.byteSize); } void diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h index 0d52f6cf0d1..4a7241bbef7 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h @@ -16,6 +16,7 @@ public: struct DomainMetrics : public metrics::MetricSet { metrics::LongValueMetric entries; + metrics::LongValueMetric diskUsage; typedef std::unique_ptr<DomainMetrics> UP; DomainMetrics(metrics::MetricSet *parent, const vespalib::string &documentType); diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS b/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS index 7c03446a5d4..614d678c95e 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS @@ -1,2 +1,2 @@ geirst -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/proton/server/OWNERS b/searchcore/src/vespa/searchcore/proton/server/OWNERS index 7c03446a5d4..614d678c95e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/server/OWNERS @@ -1,2 +1,2 @@ geirst -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/proton/server/configstore.h b/searchcore/src/vespa/searchcore/proton/server/configstore.h index ee6352d61d5..38d0dcfe7bd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/configstore.h +++ b/searchcore/src/vespa/searchcore/proton/server/configstore.h @@ -33,6 +33,12 @@ struct ConfigStore : FeedConfigStore { SerialNum serialNum) = 0; virtual void removeInvalid() = 0; + /** + * Perform prune after everything up to and including serialNum has been + * flushed to stable storage. + * + * @param serialNum The serial number flushed to stable storage. + */ virtual void prune(SerialNum serialNum) = 0; virtual SerialNum getBestSerialNum() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index eaa166afe9c..fb5e426933a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -648,13 +648,13 @@ DocumentDB::onTransactionLogReplayDone() void -DocumentDB::onPerformPrune(SerialNum oldestSerial) +DocumentDB::onPerformPrune(SerialNum flushedSerial) { if (!getAllowPrune()) { assert(_state.getClosed()); return; } - _config_store->prune(oldestSerial); + _config_store->prune(flushedSerial); } @@ -760,9 +760,9 @@ DocumentDB::getFlushTargets() } void -DocumentDB::flushDone(SerialNum oldestSerial) +DocumentDB::flushDone(SerialNum flushedSerial) { - _feedHandler.flushDone(oldestSerial); + _feedHandler.flushDone(flushedSerial); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 0c49c2c0170..ebfede59497 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -201,7 +201,7 @@ private: * Implements FeedHandler::IOwner */ virtual void onTransactionLogReplayDone() __attribute__((noinline)); - virtual void onPerformPrune(SerialNum oldestSerial); + virtual void onPerformPrune(SerialNum flushedSerial); virtual bool isFeedBlockedByRejectedConfig(); /** @@ -391,7 +391,7 @@ public: getDocsums(const search::engine::DocsumRequest & request); IFlushTarget::List getFlushTargets(); - void flushDone(SerialNum oldestSerial); + void flushDone(SerialNum flushedSerial); virtual SerialNum getCurrentSerialNumber() const diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 61e2a83bd37..6b7eadb2cb8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -432,32 +432,32 @@ FeedHandler::performEof() void -FeedHandler::performFlushDone(SerialNum oldestSerial) +FeedHandler::performFlushDone(SerialNum flushedSerial) { assert(_writeService.master().isCurrentThread()); - // XXX: oldestSerial can go backwards when attribute vectors are + // XXX: flushedSerial can go backwards when attribute vectors are // resurrected. This can be avoided if resurrected attribute vectors // pretends to have been flushed at resurrect time. - if (oldestSerial <= _prunedSerialNum) { + if (flushedSerial <= _prunedSerialNum) { return; // Cannot unprune. } if (!_owner.getAllowPrune()) { - _prunedSerialNum = oldestSerial; + _prunedSerialNum = flushedSerial; _delayedPrune = true; return; } _delayedPrune = false; - performPrune(oldestSerial); + performPrune(flushedSerial); } void -FeedHandler::performPrune(SerialNum oldestSerial) +FeedHandler::performPrune(SerialNum flushedSerial) { try { - tlsPrune(oldestSerial); // throws on error - LOG(debug, "Pruned TLS to token %" PRIu64 ".", oldestSerial); - _owner.onPerformPrune(oldestSerial); + tlsPrune(flushedSerial); // throws on error + LOG(debug, "Pruned TLS to token %" PRIu64 ".", flushedSerial); + _owner.onPerformPrune(flushedSerial); } catch (const vespalib::IllegalStateException & e) { LOG(warning, "FeedHandler::performPrune failed due to '%s'.", e.what()); } @@ -606,7 +606,7 @@ FeedHandler::replayTransactionLog(SerialNum flushedIndexMgrSerial, void -FeedHandler::flushDone(SerialNum oldestSerial) +FeedHandler::flushDone(SerialNum flushedSerial) { // Called by flush worker thread after performing a flush task _writeService.master().execute( @@ -614,7 +614,7 @@ FeedHandler::flushDone(SerialNum oldestSerial) makeClosure( this, &FeedHandler::performFlushDone, - oldestSerial))); + flushedSerial))); } void FeedHandler::changeToNormalFeedState(void) { diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 536013e0d02..a1bf80181db 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -62,7 +62,7 @@ public: virtual void performWipeHistory() = 0; virtual void onTransactionLogReplayDone() = 0; virtual void enterRedoReprocessState() = 0; - virtual void onPerformPrune(SerialNum oldestSerial) = 0; + virtual void onPerformPrune(SerialNum flushedSerial) = 0; virtual bool isFeedBlockedByRejectedConfig() = 0; virtual bool getAllowPrune() const = 0; }; @@ -153,10 +153,10 @@ private: * Used when flushing is done */ void - performFlushDone(SerialNum oldestSerial); + performFlushDone(SerialNum flushedSerial); void - performPrune(SerialNum oldestSerial); + performPrune(SerialNum flushedSerial); public: void @@ -240,10 +240,10 @@ public: /** * Called when a flush is done and allows pruning of the transaction log. * - * @param oldestSerial The oldest serial number that is still in use. + * @param flushedSerial serial number flushed for all relevant flush targets. */ void - flushDone(SerialNum oldestSerial); + flushDone(SerialNum flushedSerial); /** * Used to flip between normal and recovery feed states. diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp index 275b8006be2..6cd4df5c571 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/log/log.h> -LOG_SETUP(".feedstate"); #include <vespa/fastos/fastos.h> - -#include "feedstate.h" #include <vespa/document/bucket/bucketid.h> +#include "feedstate.h" +#include <vespa/log/log.h> +LOG_SETUP(".feedstate"); using document::BucketId; using vespalib::IllegalStateException; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index d20fe9f093a..6ded3dd7879 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.feedstates"); #include <vespa/fastos/fastos.h> - #include "feedstates.h" #include "feedconfigstore.h" #include "ireplaypackethandler.h" @@ -11,6 +7,8 @@ LOG_SETUP(".proton.server.feedstates"); #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.feedstates"); using search::transactionlog::Packet; using search::transactionlog::RPC; diff --git a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp index 7736161ddbc..5599b245655 100644 --- a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp @@ -36,9 +36,9 @@ FlushHandlerProxy::getCurrentSerialNumber(void) const void -FlushHandlerProxy::flushDone(SerialNum oldestSerial) +FlushHandlerProxy::flushDone(SerialNum flushedSerial) { - _documentDB->flushDone(oldestSerial); + _documentDB->flushDone(flushedSerial); } diff --git a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h index 7e4cd1f3176..9f27c29c2bb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h @@ -28,7 +28,7 @@ public: getCurrentSerialNumber(void) const; virtual void - flushDone(SerialNum oldestSerial); + flushDone(SerialNum flushedSerial); virtual void syncTls(SerialNum syncTo); diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp index 2f26c1a2601..bbf803adfe4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp @@ -83,13 +83,7 @@ MemoryFlush::Config::Config(uint64_t maxGlobalMemory_in, MemoryFlush::MemoryFlush(const Config &config, fastos::TimeStamp startTime) : _lock(), - _globalMaxMemory(config.maxGlobalMemory), - _maxGlobalTlsSize(config.maxGlobalTlsSize), - _globalDiskBloatFactor(config.globalDiskBloatFactor), - _maxMemoryGain(config.maxMemoryGain), - _diskBloatFactor(config.diskBloatFactor), - _maxSerialGain(config.maxSerialGain), - _maxTimeGain(config.maxTimeGain), + _config(config), _startTime(startTime) { } @@ -101,6 +95,18 @@ MemoryFlush::MemoryFlush() // empty } +MemoryFlush::Config MemoryFlush::getConfig() const +{ + vespalib::LockGuard guard(_lock); + return _config; +} + +void MemoryFlush::setConfig(const Config &config) +{ + vespalib::LockGuard guard(_lock); + _config = config; +} + FlushContext::List MemoryFlush::getFlushTargets(const FlushContext::List &targetList, const flushengine::TlsStatsMap & @@ -110,13 +116,16 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, uint64_t totalMemory(0); IFlushTarget::DiskGain totalDisk; uint64_t totalTlsSize(0); + const Config config(getConfig()); vespalib::hash_set<const void *> visitedHandlers; fastos::TimeStamp now(fastos::ClockSystem::now()); LOG(debug, "getFlushTargets(): globalMaxMemory(%" PRIu64 "), globalDiskBloatFactor(%f), " "maxMemoryGain(%" PRIu64 "), diskBloatFactor(%f), maxSerialGain(%" PRIu64 "), maxTimeGain(%f), startTime(%f)", - _globalMaxMemory, _globalDiskBloatFactor, _maxMemoryGain, _diskBloatFactor, - _maxSerialGain, _maxTimeGain.sec(), _startTime.sec()); + config.maxGlobalMemory, config.globalDiskBloatFactor, + config.maxMemoryGain, config.diskBloatFactor, + config.maxSerialGain, config.maxTimeGain.sec(), + _startTime.sec()); for (size_t i(0), m(targetList.size()); i < m; i++) { const IFlushTarget & target(*targetList[i]->getTarget()); const IFlushHandler & handler(*targetList[i]->getHandler()); @@ -133,26 +142,26 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, tlsStatsMap.getTlsStats(handler.getName()); if (visitedHandlers.insert(&handler).second) { totalTlsSize += tlsStats.getNumBytes(); - if ((totalTlsSize > _maxGlobalTlsSize) && (order < TLSSIZE)) { + if ((totalTlsSize > config.maxGlobalTlsSize) && (order < TLSSIZE)) { order = TLSSIZE; } } - if (((totalMemory >= _globalMaxMemory) || - (mgain >= _maxMemoryGain)) && (order < MEMORY)) { + if (((totalMemory >= config.maxGlobalMemory) || + (mgain >= config.maxMemoryGain)) && (order < MEMORY)) { order = MEMORY; } else if (((totalDisk.gain() > - _globalDiskBloatFactor * std::max(100000000l, + config.globalDiskBloatFactor * std::max(100000000l, std::max(totalDisk.getBefore(), totalDisk.getAfter()))) - || dgain.gain() > _diskBloatFactor * + || dgain.gain() > config.diskBloatFactor * std::max(10000000l, std::max(dgain.getBefore(), dgain.getAfter()))) && (order < DISKBLOAT) ) { order = DISKBLOAT; - } else if ((serialDiff >= _maxSerialGain) && (order < MAXSERIAL)) { + } else if ((serialDiff >= config.maxSerialGain) && (order < MAXSERIAL)) { order = MAXSERIAL; - } else if ((timeDiff >= _maxTimeGain) && (order < MAXAGE)) { + } else if ((timeDiff >= config.maxTimeGain) && (order < MAXAGE)) { order = MAXAGE; } LOG(debug, diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.h b/searchcore/src/vespa/searchcore/proton/server/memoryflush.h index cb0c84b6364..165ba4de70e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.h +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.h @@ -15,12 +15,21 @@ class MemoryFlush : public boost::noncopyable, public: struct Config { + /// Global maxMemory uint64_t maxGlobalMemory; + /// Maximum global tls size. uint64_t maxGlobalTlsSize; + /// Maximum global disk bloat factor. When this limit is reached + /// flush is forced. double globalDiskBloatFactor; + /// Maximum memory saved. When this limit is reached flush is forced. int64_t maxMemoryGain; + /// Maximum disk bloat factor. When this limit is reached + /// flush is forced. double diskBloatFactor; + /// Maximum count of what a target can have outstanding in the TLS. int64_t maxSerialGain; + /// Maximum age of unflushed data. fastos::TimeStamp maxTimeGain; Config(); Config(uint64_t maxGlobalMemory_in, @@ -35,21 +44,7 @@ public: private: /// Needed as flushDone is called in different context from the rest vespalib::Lock _lock; - /// Global maxMemory - uint64_t _globalMaxMemory; - /// Maximum global tls size. - uint64_t _maxGlobalTlsSize; - /// Maximum global disk bloat factor. When this limit is reached - /// flush is forced. - double _globalDiskBloatFactor; - /// Maximum memory saved. When this limit is reached flush is forced. - int64_t _maxMemoryGain; - /// Maximum disk bloat factor. When this limit is reached flush is forced. - double _diskBloatFactor; - /// Maximum count of what a target can have outstanding in the TLS. - int64_t _maxSerialGain; - /// Maximum age of unflushed data. - fastos::TimeStamp _maxTimeGain; + Config _config; /// The time when the strategy was started. fastos::TimeStamp _startTime; @@ -73,6 +68,8 @@ private: const flushengine::TlsStatsMap &_tlsStatsMap; }; + Config getConfig() const; + public: MemoryFlush(); @@ -84,6 +81,8 @@ public: getFlushTargets(const FlushContext::List &targetList, const flushengine::TlsStatsMap & tlsStatsMap) const override; + + void setConfig(const Config &config); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 60c54bf1ef3..2750515661f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -89,10 +89,41 @@ diskMemUsageSamplerConfig(const ProtonConfig &proton) proton.writefilter.sampleinterval); } +static constexpr size_t TOTAL_HARD_MEMORY_LIMIT=16*1024*1024*1024ul; +static constexpr size_t EACH_HARD_MEMORY_LIMIT=12*1024*1024*1024ul; + +MemoryFlush::Config +memoryFlushConfig(const ProtonConfig::Flush &flush) +{ + size_t totalMaxMemory = flush.memory.maxmemory; + if (totalMaxMemory > TOTAL_HARD_MEMORY_LIMIT) { + LOG(warning, "flush.memory.maxmemory=%ld can not" + " be set above the hard limit of %ld so we cap it", + flush.memory.maxmemory, + TOTAL_HARD_MEMORY_LIMIT); + totalMaxMemory = TOTAL_HARD_MEMORY_LIMIT; + } + size_t eachMaxMemory = flush.memory.each.maxmemory; + if (eachMaxMemory > EACH_HARD_MEMORY_LIMIT) { + LOG(warning, "flush.memory.each.maxmemory=%ld can not" + " be set above the hard limit of %ld so we cap it", + flush.memory.maxmemory, + EACH_HARD_MEMORY_LIMIT); + eachMaxMemory = EACH_HARD_MEMORY_LIMIT; + } + return MemoryFlush::Config(totalMaxMemory, + flush.memory.maxtlssize, + flush.memory.diskbloatfactor, + eachMaxMemory, + flush.memory.each.diskbloatfactor, + flush.memory.maxage.serial, + static_cast<long> + (flush.memory.maxage.time) * + fastos::TimeStamp::NANO); +} + } -static const size_t TOTAL_HARD_MEMORY_LIMIT=16*1024*1024*1024ul; -static const size_t EACH_HARD_MEMORY_LIMIT=12*1024*1024*1024ul; static const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const Proton &proton_, @@ -262,34 +293,9 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) IFlushStrategy::SP strategy; const ProtonConfig::Flush & flush(protonConfig.flush); switch (flush.strategy) { - case ProtonConfig::Flush::MEMORY: { - size_t totalMaxMemory = flush.memory.maxmemory; - if (totalMaxMemory > TOTAL_HARD_MEMORY_LIMIT) { - LOG(warning, "flush.memory.maxmemory=%ld can not" - " be set above the hard limit of %ld so we cap it", - flush.memory.maxmemory, - TOTAL_HARD_MEMORY_LIMIT); - totalMaxMemory = TOTAL_HARD_MEMORY_LIMIT; - } - size_t eachMaxMemory = flush.memory.each.maxmemory; - if (eachMaxMemory > EACH_HARD_MEMORY_LIMIT) { - LOG(warning, "flush.memory.each.maxmemory=%ld can not" - " be set above the hard limit of %ld so we cap it", - flush.memory.maxmemory, - EACH_HARD_MEMORY_LIMIT); - eachMaxMemory = EACH_HARD_MEMORY_LIMIT; - } + case ProtonConfig::Flush::MEMORY: strategy = std::make_shared<MemoryFlush>( - MemoryFlush::Config(totalMaxMemory, - flush.memory.maxtlssize, - flush.memory.diskbloatfactor, - eachMaxMemory, - flush.memory.each.diskbloatfactor, - flush.memory.maxage.serial, - static_cast<long> - (flush.memory.maxage.time) * - fastos::TimeStamp::NANO)); - } + memoryFlushConfig(flush)); break; case ProtonConfig::Flush::SIMPLE: default: @@ -299,8 +305,9 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::mkdir(protonConfig.basedir + "/documents", true); vespalib::chdir(protonConfig.basedir); _tls->start(); + _strategy = strategy; _flushEngine.reset(new FlushEngine(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()), - strategy, flush.maxconcurrent, flush.idleinterval*1000, true)); + strategy, flush.maxconcurrent, flush.idleinterval*1000)); _fs4Server.reset(new TransportServer(*_matchEngine, *_summaryEngine, *this, protonConfig.ptport, TransportServer::DEBUG_ALL)); _fs4Server->setTCPNoDelay(true); _metricsEngine->addExternalMetrics(_fs4Server->getMetrics()); @@ -538,6 +545,12 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot, configSnapshot->getGeneration())); _diskMemUsageSampler-> setConfig(diskMemUsageSamplerConfig(protonConfig)); + std::shared_ptr<MemoryFlush> memoryFlushStrategy = + std::dynamic_pointer_cast<MemoryFlush>(_strategy); + if (memoryFlushStrategy) { + memoryFlushStrategy->setConfig(memoryFlushConfig(protonConfig.flush)); + _flushEngine->kick(); + } } void diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 023a6173984..be8e602fa2a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -106,6 +106,7 @@ private: MatchEngine::UP _matchEngine; SummaryEngine::UP _summaryEngine; DocsumBySlime::UP _docsumBySlime; + IFlushStrategy::SP _strategy; FlushEngine::UP _flushEngine; RPCHooks::UP _rpcHooks; HealthAdapter _healthAdapter; diff --git a/searchcore/src/vespa/searchcore/util/OWNERS b/searchcore/src/vespa/searchcore/util/OWNERS index 1037590124e..9dc0c2d970d 100644 --- a/searchcore/src/vespa/searchcore/util/OWNERS +++ b/searchcore/src/vespa/searchcore/util/OWNERS @@ -1 +1 @@ -balder +baldersheim |