diff options
Diffstat (limited to 'searchlib')
3 files changed, 49 insertions, 0 deletions
diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index dde6b14121d..36a697eaa12 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -226,4 +226,24 @@ TEST_F("Compaction is not executed when free lists are used", EXPECT_EQUAL(1001u, afterSpace.dead()); } +TEST_F("Compaction is peformed when compaction strategy is changed to enable compaction", + Fixture(compactAddressSpaceAttributeConfig(false))) +{ + populate_and_hammer(f, true); + AddressSpace after1 = f.getMultiValueAddressSpaceUsage("after1"); + // 100 * 1000 dead arrays due to new values for docids + // 1 reserved array accounted as dead + EXPECT_EQUAL(100001u, after1.dead()); + f._v->update_config(compactAddressSpaceAttributeConfig(true)); + auto old_dead = after1.dead(); + AddressSpace after2 = f.getMultiValueAddressSpaceUsage("after2"); + while (after2.dead() < old_dead) { + old_dead = after2.dead(); + f._v->commit(); // new commit might trigger further compaction + after2 = f.getMultiValueAddressSpaceUsage("after2"); + } + // DEAD_ARRAYS_SLACK in multi value mapping is is 64k + EXPECT_GREATER(65536u, after2.dead()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 2168bbe4276..d2574bd32a2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -21,6 +21,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/searchlib/util/logutil.h> #include <vespa/searchcommon/attribute/attribute_utils.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attributevector"); @@ -769,6 +770,31 @@ AttributeVector::logEnumStoreEvent(const char *reason, const char *stage) EV_STATE(eventName.c_str(), jstr.toString().data()); } +void +AttributeVector::drain_hold(uint64_t hold_limit) +{ + incGeneration(); + for (int retry = 0; ; ++retry) { + removeAllOldGenerations(); + updateStat(true); + if (_status.getOnHold() <= hold_limit) { + return; + } + std::this_thread::sleep_for(retry < 20 ? 20ms : 100ms); + } +} + +void +AttributeVector::update_config(const Config& cfg) +{ + commit(true); + drain_hold(1024 * 1024); // Wait until 1MiB or less on hold + _config.setGrowStrategy(cfg.getGrowStrategy()); + _config.setCompactionStrategy(cfg.getCompactionStrategy()); + commit(); // might trigger compaction if compaction strategy changed + drain_hold(1024 * 1024); // Wait until 1MiB or less on hold +} + template bool AttributeVector::append<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t, bool); template bool AttributeVector::update<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &); template bool AttributeVector::remove<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 4fc6589850e..f308ee8d024 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -393,6 +393,7 @@ public: /** Return the fixed length of the attribute. If 0 then you must inquire each document. */ size_t getFixedWidth() const override { return _config.basicType().fixedSize(); } const Config &getConfig() const { return _config; } + void update_config(const Config& cfg); BasicType getInternalBasicType() const { return _config.basicType(); } CollectionType getInternalCollectionType() const { return _config.collectionType(); } const BaseName & getBaseFileName() const { return _baseFileName; } @@ -667,6 +668,8 @@ public: static bool isEnumerated(const vespalib::GenericHeader &header); virtual vespalib::MemoryUsage getChangeVectorMemoryUsage() const; + + void drain_hold(uint64_t hold_limit); }; } |