diff options
11 files changed, 78 insertions, 136 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index e73e2b3171f..6b0a14c7005 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -20,7 +20,7 @@ set(AUTORUN_UNIT_TESTS FALSE CACHE BOOL "If TRUE, tests will be run immediately set(WARN_OPTS "-Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wcomment -Wformat -Wparentheses -Wreturn-type -Wswitch -Wtrigraphs -Wunused -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings") # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 ${WARN_OPTS} -fPIC -D_GLIBCXX_USE_CXX11_ABI=0 -DBOOST_DISABLE_ASSERTS -march=westmere -mtune=intel") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Og ${WARN_OPTS} -fPIC -D_GLIBCXX_USE_CXX11_ABI=0 -DBOOST_DISABLE_ASSERTS -march=westmere -mtune=intel") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -Wnon-virtual-dtor -fvisibility-inlines-hidden -fdiagnostics-color=auto") diff --git a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp index 6e843545b58..45dd12834fe 100644 --- a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp +++ b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp @@ -1,13 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("memoryflush_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/flushengine/flushcontext.h> -#include <vespa/searchcore/proton/flushengine/iflushhandler.h> #include <vespa/searchcore/proton/flushengine/tls_stats_map.h> #include <vespa/searchcore/proton/test/dummy_flush_target.h> -#include <vespa/searchcorespi/flush/iflushtarget.h> #include <vespa/searchcore/proton/server/memoryflush.h> using fastos::TimeStamp; @@ -153,7 +149,8 @@ createTargetF(const vespalib::string &name, bool urgentFlush) bool assertOrder(const StringList &exp, const FlushContext::List &act) { - if (!EXPECT_EQUAL(exp.size(), act.size())) return false; + if (!EXPECT_EQUAL(exp.size(), act.size())) + return false; for (size_t i = 0; i < exp.size(); ++i) { if (!EXPECT_EQUAL(exp[i], act[i]->getTarget()->getName())) return false; } @@ -169,12 +166,12 @@ requireThatWeCanOrderByMemoryGain() .add(createTargetM("t4", MemoryGain(20, 0))) .add(createTargetM("t3", MemoryGain(15, 0))); { // target t4 has memoryGain >= maxMemoryGain - MemoryFlush flush({1000, 20 * gibi, 1.0, 20, 1.0, 1000, TimeStamp(TimeStamp::MINUTE)}); + MemoryFlush flush({1000, 20 * gibi, 1.0, 20, 1.0, TimeStamp(TimeStamp::MINUTE)}); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } { // trigger totalMemoryGain >= globalMaxMemory - MemoryFlush flush({50, 20 * gibi, 1.0, 1000, 1.0, 1000, TimeStamp(TimeStamp::MINUTE)}); + MemoryFlush flush({50, 20 * gibi, 1.0, 1000, 1.0, TimeStamp(TimeStamp::MINUTE)}); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } @@ -193,13 +190,13 @@ requireThatWeCanOrderByDiskGainWithLargeValues() .add(createTargetD("t3", DiskGain(before, 50 * milli))); // gain 50M { // target t4 has diskGain > bloatValue // t4 gain: 55M / 100M = 0.55 -> bloat factor 0.54 to trigger - MemoryFlush flush({1000, 20 * gibi, 10.0, 1000, 0.54, 1000, TimeStamp(TimeStamp::MINUTE)}); + MemoryFlush flush({1000, 20 * gibi, 10.0, 1000, 0.54, TimeStamp(TimeStamp::MINUTE)}); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } { // trigger totalDiskGain > totalBloatValue // total gain: 160M / 4 * 100M = 0.4 -> bloat factor 0.39 to trigger - MemoryFlush flush({1000, 20 * gibi, 0.39, 1000, 10.0, 1000, TimeStamp(TimeStamp::MINUTE)}); + MemoryFlush flush({1000, 20 * gibi, 0.39, 1000, 10.0, TimeStamp(TimeStamp::MINUTE)}); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } @@ -214,32 +211,16 @@ requireThatWeCanOrderByDiskGainWithSmallValues() .add(createTargetD("t4", DiskGain(100, 45))) // gain 55 .add(createTargetD("t3", DiskGain(100, 50))); // gain 50 // total disk bloat value calculation uses min 100M disk size - // target bloat value calculation uses min 10M disk size + // target bloat value calculation uses min 100M disk size { // target t4 has diskGain > bloatValue - // t4 gain: 55 / 10M = 0.0000055 -> bloat factor 0.0000054 to trigger - MemoryFlush flush({1000, 20 * gibi, 10.0, 1000, 0.0000054, 1000, TimeStamp(TimeStamp::MINUTE)}); + // t4 gain: 55 / 100M = 0.0000055 -> bloat factor 0.0000054 to trigger + MemoryFlush flush({1000, 20 * gibi, 10.0, 1000, 0.00000054, TimeStamp(TimeStamp::MINUTE)}); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } { // trigger totalDiskGain > totalBloatValue // total gain: 160 / 100M = 0.0000016 -> bloat factor 0.0000015 to trigger - MemoryFlush flush({1000, 20 * gibi, 0.0000015, 1000, 10.0, 1000, TimeStamp(TimeStamp::MINUTE)}); - EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"), - flush.getFlushTargets(cb.list(), cb.tlsStats()))); - } -} - -void -requireThatWeCanOrderBySerialNum() -{ - SerialNum lastSerial = 99; - ContextBuilder cb; - cb.add(createTargetS("t2", 89), lastSerial) - .add(createTargetS("t1", 94), lastSerial) - .add(createTargetS("t4", 98), lastSerial + 19) - .add(createTargetS("t3", 84), lastSerial); - { // target t4 has serialDiff >= maxSerialGain - MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 20, TimeStamp(TimeStamp::MINUTE)}); + MemoryFlush flush({1000, 20 * gibi, 0.0000015, 1000, 10.0, TimeStamp(TimeStamp::MINUTE)}); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } @@ -257,12 +238,12 @@ requireThatWeCanOrderByAge() .add(createTargetT("t3", TimeStamp(now.val() - 15 * TimeStamp::SEC))); { // all targets have timeDiff >= maxTimeGain - MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 1000, TimeStamp(2 * TimeStamp::SEC)}, start); + MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, TimeStamp(2 * TimeStamp::SEC)}, start); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t3").add("t2").add("t1"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } { // no targets have timeDiff >= maxTimeGain - MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 1000, TimeStamp(30 * TimeStamp::SEC)}, start); + MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, TimeStamp(30 * TimeStamp::SEC)}, start); EXPECT_TRUE(assertOrder(StringList(), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } } @@ -298,12 +279,12 @@ requireThatWeCanOrderByTlsSize() 1900), 2000)); { // sum of tls sizes above limit, trigger sort order based on tls size - MemoryFlush flush({1000, 3 * gibi, 1.0, 1000, 1.0, 2000, TimeStamp(2 * TimeStamp::SEC)}, start); + MemoryFlush flush({1000, 3 * gibi, 1.0, 1000, 1.0, TimeStamp(2 * TimeStamp::SEC)}, start); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t1").add("t2").add("t3"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } { // sum of tls sizes below limit - MemoryFlush flush({1000, 30 * gibi, 1.0, 1000, 1.0, 2000, TimeStamp(30 * TimeStamp::SEC)}, start); + MemoryFlush flush({1000, 30 * gibi, 1.0, 1000, 1.0, TimeStamp(30 * TimeStamp::SEC)}, start); EXPECT_TRUE(assertOrder(StringList(), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } } @@ -319,7 +300,7 @@ requireThatWeHandleLargeSerialNumbersWhenOrderingByTlsSize() builder.add(createTargetT("t1", TimeStamp(), uint32_max + 5), lastSerial); builder.add(createTargetT("t2", TimeStamp(), uint32_max - 5), lastSerial); uint64_t maxMemoryGain = 10; - MemoryFlush flush({maxMemoryGain, 1000, 0, maxMemoryGain, 0, 0, TimeStamp()}, TimeStamp()); + MemoryFlush flush({maxMemoryGain, 1000, 0, maxMemoryGain, 0, TimeStamp()}, TimeStamp()); EXPECT_TRUE(assertOrder(StringList().add("t2").add("t1"), flush.getFlushTargets(builder.list(), builder.tlsStats()))); } @@ -332,32 +313,26 @@ requireThatOrderTypeIsPreserved() TimeStamp ts2(now.val() - 20 * TimeStamp::SEC); TimeStamp ts3(now.val() - 10 * TimeStamp::SEC); TimeStamp maxTimeGain(15 * TimeStamp::SEC); - { // MAXAGE VS MAXSERIAL + + { // MAXAGE VS DISKBLOAT ContextBuilder cb; cb.add(createTargetT("t2", ts2, 5), 14) - .add(createTargetS("t1", 4, ts3), 14); - MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 10, maxTimeGain}, ts1); - EXPECT_TRUE(assertOrder(StringList().add("t1").add("t2"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); - } - { // MAXSERIAL VS DISKBLOAT - ContextBuilder cb; - cb.add(createTargetS("t2", 4)) .add(createTargetD("t1", DiskGain(100 * milli, 80 * milli), 5)); - MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 0.19, 10, TimeStamp(30 * TimeStamp::SEC)}); + MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 0.19, TimeStamp(30 * TimeStamp::SEC)}); EXPECT_TRUE(assertOrder(StringList().add("t1").add("t2"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } { // DISKBLOAT VS MEMORY ContextBuilder cb; cb.add(createTargetD("t2", DiskGain(100 * milli, 80 * milli))) .add(createTargetM("t1", MemoryGain(100, 80))); - MemoryFlush flush({1000, 20 * gibi, 1.0, 20, 0.19, 1000, TimeStamp(30 * TimeStamp::SEC)}); + MemoryFlush flush({1000, 20 * gibi, 1.0, 20, 0.19, TimeStamp(30 * TimeStamp::SEC)}); EXPECT_TRUE(assertOrder(StringList().add("t1").add("t2"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } { // urgent flush ContextBuilder cb; cb.add(createTargetF("t2", false)) .add(createTargetF("t1", true)); - MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, 1000, TimeStamp(30 * TimeStamp::SEC)}); + MemoryFlush flush({1000, 20 * gibi, 1.0, 1000, 1.0, TimeStamp(30 * TimeStamp::SEC)}); EXPECT_TRUE(assertOrder(StringList().add("t1").add("t2"), flush.getFlushTargets(cb.list(), cb.tlsStats()))); } } @@ -367,7 +342,6 @@ TEST_MAIN() TEST_DO(requireThatWeCanOrderByMemoryGain()); TEST_DO(requireThatWeCanOrderByDiskGainWithLargeValues()); TEST_DO(requireThatWeCanOrderByDiskGainWithSmallValues()); - TEST_DO(requireThatWeCanOrderBySerialNum()); TEST_DO(requireThatWeCanOrderByAge()); TEST_DO(requireThatWeCanOrderByTlsSize()); TEST_DO(requireThatWeHandleLargeSerialNumbersWhenOrderingByTlsSize()); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.cpp index 57d1042df3d..bb237d1ff7e 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "flush_all_strategy.h" -#include <algorithm> using search::SerialNum; namespace proton { -namespace -{ +namespace { class CompareTarget { diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.h index 4c0076c738c..c584dd00cfa 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_all_strategy.h @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchcore/proton/flushengine/iflushstrategy.h> -#include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/stllike/string.h> -#include <map> +#include "iflushstrategy.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h index 0795876a04e..ab6166a7082 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchcore/proton/flushengine/iflushhandler.h> +#include "iflushhandler.h" #include <vespa/vespalib/util/executor.h> namespace proton { @@ -14,11 +14,11 @@ using searchcorespi::IFlushTarget; */ class FlushContext { private: - vespalib::string _name; - IFlushHandler::SP _handler; - IFlushTarget::SP _target; - searchcorespi::FlushTask::UP _task; - search::SerialNum _lastSerial; + vespalib::string _name; + IFlushHandler::SP _handler; + IFlushTarget::SP _target; + searchcorespi::FlushTask::UP _task; + search::SerialNum _lastSerial; public: typedef std::shared_ptr<FlushContext> SP; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h index 9f7f3e37188..41774c085c1 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/searchcorespi/flush/iflushtarget.h> -#include <string> #include <vector> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h index c905f4486a1..c4c49b5de13 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchcore/proton/flushengine/iflushhandler.h> -#include <vespa/searchcore/proton/flushengine/flushcontext.h> +#include "iflushhandler.h" +#include "flushcontext.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp index 96b9fd21ddd..a8917c38245 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "memory_flush_config_updater.h" #include <vespa/log/log.h> LOG_SETUP(".proton.server.memory_flush_config_updater"); -#include "memory_flush_config_updater.h" namespace proton { @@ -115,7 +114,6 @@ MemoryFlushConfigUpdater::convertConfig(const ProtonConfig::Flush::Memory &confi config.diskbloatfactor, eachMaxMemory, config.each.diskbloatfactor, - config.maxage.serial, static_cast<long> (config.maxage.time) * fastos::TimeStamp::NANO); diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp index 98226d0d495..4cf24514127 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.memoryflush"); -#include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/searchcore/proton/flushengine/tls_stats_map.h> #include "memoryflush.h" -#include <algorithm> +#include <vespa/searchcore/proton/flushengine/tls_stats_map.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_set.h> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.memoryflush"); using search::SerialNum; using proton::flushengine::TlsStats; @@ -36,8 +36,7 @@ estimateNeededTlsSizeForFlushTarget(const TlsStats &tlsStats, if (flushedSerialNum < tlsStats.getFirstSerial()) { return tlsStats.getNumBytes(); } - int64_t numEntries = tlsStats.getLastSerial() - - tlsStats.getFirstSerial() + 1; + int64_t numEntries = tlsStats.getLastSerial() - tlsStats.getFirstSerial() + 1; if (numEntries <= 0) { return 0u; } @@ -57,10 +56,8 @@ MemoryFlush::Config::Config() globalDiskBloatFactor(0.2), maxMemoryGain(1000*1024*1024ul), diskBloatFactor(0.2), - maxSerialGain(1*1000*1000ul), maxTimeGain(fastos::TimeStamp::MINUTE*60*24) -{ -} +{ } MemoryFlush::Config::Config(uint64_t maxGlobalMemory_in, @@ -68,32 +65,27 @@ MemoryFlush::Config::Config(uint64_t maxGlobalMemory_in, double globalDiskBloatFactor_in, uint64_t maxMemoryGain_in, double diskBloatFactor_in, - uint64_t maxSerialGain_in, fastos::TimeStamp maxTimeGain_in) : maxGlobalMemory(maxGlobalMemory_in), maxGlobalTlsSize(maxGlobalTlsSize_in), globalDiskBloatFactor(globalDiskBloatFactor_in), maxMemoryGain(maxMemoryGain_in), diskBloatFactor(diskBloatFactor_in), - maxSerialGain(maxSerialGain_in), maxTimeGain(maxTimeGain_in) -{ -} +{ } -MemoryFlush::MemoryFlush(const Config &config, - fastos::TimeStamp startTime) +MemoryFlush::MemoryFlush(const Config &config, fastos::TimeStamp startTime) : _lock(), _config(config), _startTime(startTime) -{ -} +{ } MemoryFlush::MemoryFlush() : MemoryFlush(Config(), fastos::ClockSystem::now()) -{ - // empty -} +{ } + +MemoryFlush::~MemoryFlush() { } MemoryFlush::Config MemoryFlush::getConfig() const { @@ -116,13 +108,23 @@ getOrderName(MemoryFlush::OrderType &orderType) case MemoryFlush::OrderType::MEMORY: return "MEMORY"; case MemoryFlush::OrderType::TLSSIZE: return "TLSSIZE"; case MemoryFlush::OrderType::DISKBLOAT: return "DISKBLOAT"; - case MemoryFlush::OrderType::MAXSERIAL: return "MAXSERIAL"; case MemoryFlush::OrderType::MAXAGE: return "MAXAGE"; case MemoryFlush::OrderType::DEFAULT: return "DEFAULT"; } return "DEFAULT"; } +size_t computeGain(const IFlushTarget::DiskGain & gain) { + return std::max(100000000l, std::max(gain.getBefore(), gain.getAfter())); +} +bool isDiskBloatToHigh(const IFlushTarget::DiskGain & totalDisk, + const MemoryFlush::Config & config, + const IFlushTarget::DiskGain & dgain) +{ + return (totalDisk.gain() > config.globalDiskBloatFactor * computeGain(totalDisk)) + || (dgain.gain() > config.diskBloatFactor * computeGain(dgain)); +} + } FlushContext::List @@ -139,10 +141,10 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, fastos::TimeStamp now(fastos::ClockSystem::now()); LOG(debug, "getFlushTargets(): globalMaxMemory(%" PRIu64 "), maxGlobalTlsSize(%" PRIu64 "), globalDiskBloatFactor(%f), " - "maxMemoryGain(%" PRIu64 "), diskBloatFactor(%f), maxSerialGain(%" PRIu64 "), maxTimeGain(%f), startTime(%f)", + "maxMemoryGain(%" PRIu64 "), diskBloatFactor(%f), maxTimeGain(%f), startTime(%f)", config.maxGlobalMemory, config.maxGlobalTlsSize, config.globalDiskBloatFactor, config.maxMemoryGain, config.diskBloatFactor, - config.maxSerialGain, config.maxTimeGain.sec(), + config.maxTimeGain.sec(), _startTime.sec()); for (size_t i(0), m(targetList.size()); i < m; i++) { const IFlushTarget & target(*targetList[i]->getTarget()); @@ -156,29 +158,17 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, fastos::TimeStamp lastFlushTime = target.getLastFlushTime(); fastos::TimeStamp timeDiff(now - (lastFlushTime.val() > 0 ? lastFlushTime : _startTime)); totalMemory += mgain; - const flushengine::TlsStats &tlsStats = - tlsStatsMap.getTlsStats(handler.getName()); + const flushengine::TlsStats &tlsStats = tlsStatsMap.getTlsStats(handler.getName()); if (visitedHandlers.insert(&handler).second) { totalTlsSize += tlsStats.getNumBytes(); if ((totalTlsSize > config.maxGlobalTlsSize) && (order < TLSSIZE)) { order = TLSSIZE; } } - if (((totalMemory >= config.maxGlobalMemory) || - (mgain >= config.maxMemoryGain)) && (order < MEMORY)) { + if (((totalMemory >= config.maxGlobalMemory) || (mgain >= config.maxMemoryGain)) && (order < MEMORY)) { order = MEMORY; - } else if (((totalDisk.gain() > - config.globalDiskBloatFactor * std::max(100000000l, - std::max(totalDisk.getBefore(), - totalDisk.getAfter()))) - || dgain.gain() > config.diskBloatFactor * - std::max(10000000l, - std::max(dgain.getBefore(), dgain.getAfter()))) - && (order < DISKBLOAT) - ) { + } else if (isDiskBloatToHigh(totalDisk, config, dgain) && (order < DISKBLOAT)) { order = DISKBLOAT; - } else if ((serialDiff >= config.maxSerialGain) && (order < MAXSERIAL)) { - order = MAXSERIAL; } else if ((timeDiff >= config.maxTimeGain) && (order < MAXAGE)) { order = MAXAGE; } @@ -194,8 +184,7 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, totalDisk.gain(), dgain.gain(), tlsStats.getNumBytes(), - estimateNeededTlsSizeForFlushTarget(tlsStats, - target.getFlushedSerialNum()), + estimateNeededTlsSizeForFlushTarget(tlsStats, target.getFlushedSerialNum()), target.getFlushedSerialNum(), localLastSerial, serialDiff, @@ -214,12 +203,12 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, return FlushContext::List(); } if (LOG_WOULD_LOG(debug)) { - std::ostringstream oss; + vespalib::asciistream oss; for (size_t i = 0; i < fv.size(); ++i) { if (i > 0) { oss << ","; } - oss << fv[i]->getName().c_str(); + oss << fv[i]->getName(); } LOG(debug, "getFlushTargets(): %zu sorted targets: [%s]", fv.size(), oss.str().c_str()); } @@ -239,34 +228,25 @@ MemoryFlush::CompareTarget::operator()(const FlushContext::SP &lfc, switch (_order) { case MEMORY: - return (lhs.getApproxMemoryGain().gain() > - rhs.getApproxMemoryGain().gain()); + return (lhs.getApproxMemoryGain().gain() > rhs.getApproxMemoryGain().gain()); case TLSSIZE: { - const flushengine::TlsStats &lhsTlsStats = - _tlsStatsMap.getTlsStats(lfc->getHandler()->getName()); - const flushengine::TlsStats &rhsTlsStats = - _tlsStatsMap.getTlsStats(rfc->getHandler()->getName()); + const flushengine::TlsStats &lhsTlsStats = _tlsStatsMap.getTlsStats(lfc->getHandler()->getName()); + const flushengine::TlsStats &rhsTlsStats = _tlsStatsMap.getTlsStats(rfc->getHandler()->getName()); SerialNum lhsFlushedSerialNum(lhs.getFlushedSerialNum()); SerialNum rhsFlushedSerialNum(rhs.getFlushedSerialNum()); - uint64_t lhsNeededTlsSize = - estimateNeededTlsSizeForFlushTarget(lhsTlsStats, - lhsFlushedSerialNum); - uint64_t rhsNeededTlsSize = - estimateNeededTlsSizeForFlushTarget(rhsTlsStats, - rhsFlushedSerialNum); + uint64_t lhsNeededTlsSize = estimateNeededTlsSizeForFlushTarget(lhsTlsStats, lhsFlushedSerialNum); + uint64_t rhsNeededTlsSize = estimateNeededTlsSizeForFlushTarget(rhsTlsStats, rhsFlushedSerialNum); if (lhsNeededTlsSize != rhsNeededTlsSize) { return (lhsNeededTlsSize > rhsNeededTlsSize); } return (lhs.getLastFlushTime() < rhs.getLastFlushTime()); } case DISKBLOAT: - return (lhs.getApproxDiskGain().gain() > - rhs.getApproxDiskGain().gain()); + return (lhs.getApproxDiskGain().gain() > rhs.getApproxDiskGain().gain()); case MAXAGE: return (lhs.getLastFlushTime() < rhs.getLastFlushTime()); default: - return (getSerialDiff(lfc->getLastSerial(), lhs) > - getSerialDiff(rfc->getLastSerial(), rhs)); + return (getSerialDiff(lfc->getLastSerial(), lhs) > getSerialDiff(rfc->getLastSerial(), rhs)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.h b/searchcore/src/vespa/searchcore/proton/server/memoryflush.h index 71af7bf6375..2b11be6bcca 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.h +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.h @@ -3,8 +3,6 @@ #include <vespa/searchcore/proton/flushengine/iflushstrategy.h> #include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/stllike/string.h> -#include <map> namespace proton { @@ -25,8 +23,7 @@ public: /// Maximum disk bloat factor. When this limit is reached /// flush is forced. double diskBloatFactor; - /// Maximum count of what a target can have outstanding in the TLS. - int64_t maxSerialGain; + /// Maximum age of unflushed data. fastos::TimeStamp maxTimeGain; Config(); @@ -35,11 +32,10 @@ public: double globalDiskBloatFactor_in, uint64_t maxMemoryGain_in, double diskBloatFactor_in, - uint64_t maxSerialGain_in, fastos::TimeStamp maxTimeGain_in); }; - enum OrderType { DEFAULT, MAXAGE, MAXSERIAL, DISKBLOAT, TLSSIZE, MEMORY }; + enum OrderType { DEFAULT, MAXAGE, DISKBLOAT, TLSSIZE, MEMORY }; private: /// Needed as flushDone is called in different context from the rest @@ -55,8 +51,7 @@ private: const flushengine::TlsStatsMap &tlsStatsMap) : _order(order), _tlsStatsMap(tlsStatsMap) - { - } + { } bool operator ()(const FlushContext::SP &lfc, @@ -74,6 +69,7 @@ public: MemoryFlush(const Config &config, fastos::TimeStamp startTime = fastos::TimeStamp(fastos::ClockSystem::now())); + ~MemoryFlush(); // Implements IFlushStrategy virtual FlushContext::List diff --git a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h index e2ced502d9e..b8da96ccce8 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h +++ b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h @@ -4,6 +4,7 @@ #include "flushstats.h" #include "flushtask.h" #include <vespa/fastos/timestamp.h> +#include <vector> namespace searchcorespi { |