diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-10-21 10:37:13 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-10-21 10:48:47 +0000 |
commit | 54f5454cdb112e458aea3b5296199b7531f487d2 (patch) | |
tree | cf27d043d2c865e231c90a08784229881d54e9f8 /searchlib | |
parent | 9abe019606f2367b05e4e13d796de65dddf7c449 (diff) |
If the uncommitted changes pass 128k for an attribute force a commit.
Diffstat (limited to 'searchlib')
6 files changed, 55 insertions, 14 deletions
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 97813a2a65e..6ee97595b79 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -275,6 +275,7 @@ private: void testReaderDuringLastUpdate(); void testPendingCompaction(); + void testConditionalCommit(); public: AttributeTest(); @@ -402,13 +403,18 @@ void AttributeTest::populate(StringAttribute & v, unsigned seed) v.commit(); } -template <> -void AttributeTest::populateSimple(IntegerAttribute & v, uint32_t docIdLow, uint32_t docIdHigh) +void populateSimpleUncommitted(IntegerAttribute & v, uint32_t docIdLow, uint32_t docIdHigh) { - for(uint32_t docId(docIdLow); docId < docIdHigh; ++docId) { + for (uint32_t docId(docIdLow); docId < docIdHigh; ++docId) { v.clearDoc(docId); EXPECT_TRUE( v.update(docId, docId + 1) ); } +} + +template <> +void AttributeTest::populateSimple(IntegerAttribute & v, uint32_t docIdLow, uint32_t docIdHigh) +{ + populateSimpleUncommitted(v, docIdLow, docIdHigh); v.commit(); } @@ -2233,6 +2239,34 @@ AttributeTest::testPendingCompaction() populateSimple(iv, 1, 2); // should not trigger new compaction } +void +AttributeTest::testConditionalCommit() { + Config cfg(BasicType::INT32, CollectionType::SINGLE); + cfg.setFastSearch(true); + cfg.setMaxUnCommittedMemory(70000); + AttributePtr v = createAttribute("sfsint32_cc", cfg); + addClearedDocs(v, 1000); + auto &iv = static_cast<IntegerAttribute &>(*v.get()); + EXPECT_EQUAL(0x8000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQUAL(0u, iv.getChangeVectorMemoryUsage().usedBytes()); + AttributeGuard guard1(v); + populateSimpleUncommitted(iv, 1, 3); + EXPECT_EQUAL(0x8000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQUAL(128u, iv.getChangeVectorMemoryUsage().usedBytes()); + populateSimpleUncommitted(iv, 1, 1000); + EXPECT_EQUAL(0x10000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQUAL(64064u, iv.getChangeVectorMemoryUsage().usedBytes()); + EXPECT_FALSE(v->commitIfChangeVectorTooLarge()); + EXPECT_EQUAL(0x10000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQUAL(64064u, iv.getChangeVectorMemoryUsage().usedBytes()); + populateSimpleUncommitted(iv, 1, 200); + EXPECT_EQUAL(0x20000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQUAL(76800u, iv.getChangeVectorMemoryUsage().usedBytes()); + EXPECT_TRUE(v->commitIfChangeVectorTooLarge()); + EXPECT_EQUAL(0x2000u, iv.getChangeVectorMemoryUsage().allocatedBytes()); + EXPECT_EQUAL(0u, iv.getChangeVectorMemoryUsage().usedBytes()); +} + void testNamePrefix() { Config cfg(BasicType::INT32, CollectionType::SINGLE); AttributeVector::SP vFlat = createAttribute("sfsint32_pc", cfg); @@ -2312,6 +2346,7 @@ int AttributeTest::Main() TEST_DO(requireThatAddressSpaceUsageIsReported()); testReaderDuringLastUpdate(); TEST_DO(testPendingCompaction()); + TEST_DO(testConditionalCommit()); TEST_DO(testNamePrefix()); test_multi_value_mapping_has_free_lists_enabled(); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 15aec41934a..3bc1e5ec25f 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -756,6 +756,15 @@ AttributeVector::getChangeVectorMemoryUsage() const return vespalib::MemoryUsage(0, 0, 0, 0); } +bool +AttributeVector::commitIfChangeVectorTooLarge() { + bool needCommit = getChangeVectorMemoryUsage().usedBytes() > getConfig().getMaxUnCommittedMemory(); + if (needCommit) { + commit(false); + } + return needCommit; +} + void AttributeVector::logEnumStoreEvent(const char *reason, const char *stage) { diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 61c86d53dcf..bb88b168474 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -359,8 +359,7 @@ protected: template <typename BaseType, typename ChangeData> static BaseType - applyArithmetic(const BaseType &value, - const ChangeTemplate<ChangeData> & arithmetic) + applyArithmetic(const BaseType &value, const ChangeTemplate<ChangeData> & arithmetic) { typedef typename ChangeData::DataType LargeType; if (attribute::isUndefined(value)) { @@ -371,12 +370,10 @@ protected: return value - static_cast<LargeType>(arithmetic._arithOperand); } else if (arithmetic._type == ChangeBase::MUL) { LargeType r; - return round((static_cast<double>(value) * - arithmetic._arithOperand), r); + return round((static_cast<double>(value) * arithmetic._arithOperand), r); } else if (arithmetic._type == ChangeBase::DIV) { LargeType r; - return round(static_cast<double>(value) / - arithmetic._arithOperand, r); + return round(static_cast<double>(value) / arithmetic._arithOperand, r); } return value; } @@ -669,6 +666,7 @@ public: static bool isEnumerated(const vespalib::GenericHeader &header); virtual vespalib::MemoryUsage getChangeVectorMemoryUsage() const; + bool commitIfChangeVectorTooLarge(); void drain_hold(uint64_t hold_limit); }; diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index d8abdfae92a..12ac77febb9 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/searchcommon/common/undefinedvalues.h> -#include <vespa/vespalib/stllike/allocator.h> #include <vector> namespace vespalib { class MemoryUsage; } @@ -130,7 +129,7 @@ NumericChangeData<double>::operator<(const NumericChangeData<double> &rhs) const template <typename T> class ChangeVectorT { private: - using Vector = std::vector<T, vespalib::allocator_large<T>>; + using Vector = std::vector<T>; public: using const_iterator = typename Vector::const_iterator; ChangeVectorT(); @@ -152,7 +151,7 @@ public: const Vector &_v; }; class DocIdInsertOrder { - using AdjacentDocIds = std::vector<uint64_t, vespalib::allocator_large<uint64_t>>; + using AdjacentDocIds = std::vector<uint64_t>; public: class const_iterator { public: diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 761543f4bad..f6e8266f858 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -109,6 +109,7 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) retval.setFastAccess(cfg.fastaccess); retval.setMutable(cfg.ismutable); retval.setPaged(cfg.paged); + retval.setMaxUnCommittedMemory(cfg.maxuncommittedmemory); predicateParams.setArity(cfg.arity); predicateParams.setBounds(cfg.lowerbound, cfg.upperbound); predicateParams.setDensePostingListThreshold(cfg.densepostinglistthreshold); diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h index 234666d7845..c4a2035b200 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h @@ -30,13 +30,12 @@ public: bool applyWeight(DocId doc, const FieldValue & fv, const ArithmeticValueUpdate & wAdjust) override; bool applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust) override; uint32_t clearDoc(DocId doc) override; + vespalib::MemoryUsage getChangeVectorMemoryUsage() const override; protected: IntegerAttribute(const vespalib::string & name, const Config & c); using Change = ChangeTemplate<NumericChangeData<largeint_t>>; using ChangeVector = ChangeVectorT<Change>; ChangeVector _changes; - - vespalib::MemoryUsage getChangeVectorMemoryUsage() const override; private: const char * getString(DocId doc, char * s, size_t sz) const override; uint32_t get(DocId doc, vespalib::string * v, uint32_t sz) const override; |