diff options
Diffstat (limited to 'searchcore')
117 files changed, 1023 insertions, 632 deletions
diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index f40502e14d1..e6b7b34903b 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -71,7 +71,7 @@ shafile(const string &baseDir, bool openres = f.OpenReadOnly(fullFile.c_str()); if (!openres) { LOG(error, "Could not open %s for sha256 checksum", fullFile.c_str()); - abort(); + LOG_ABORT("should not be reached"); } int64_t flen = f.GetSize(); int64_t remainder = flen; diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 17f1faffbba..8ffe4807427 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config/helper/configgetter.h> - #include <vespa/searchcore/proton/server/replaypacketdispatcher.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/searchlib/transactionlog/translogserver.h> diff --git a/searchcore/src/tests/applyattrupdates/applyattrupdates.cpp b/searchcore/src/tests/applyattrupdates/applyattrupdates.cpp index 4c4027da6d6..48c02215732 100644 --- a/searchcore/src/tests/applyattrupdates/applyattrupdates.cpp +++ b/searchcore/src/tests/applyattrupdates/applyattrupdates.cpp @@ -1,4 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/document/base/testdocrepo.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> @@ -21,7 +22,6 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> - LOG_SETUP("applyattrupdates_test"); using namespace document; diff --git a/searchcore/src/tests/grouping/grouping.cpp b/searchcore/src/tests/grouping/grouping.cpp index b159be5288c..7e21fffe9fd 100644 --- a/searchcore/src/tests/grouping/grouping.cpp +++ b/searchcore/src/tests/grouping/grouping.cpp @@ -12,6 +12,9 @@ #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <iostream> +#include <vespa/log/log.h> +LOG_SETUP("grouping_test"); + using namespace search::attribute; using namespace search::aggregation; using namespace search::expression; diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp index 3ef2461c682..24c38ad118b 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp @@ -1,4 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchcore/proton/test/attribute_utils.h> @@ -10,6 +11,9 @@ #include <vespa/config-attributes.h> #include <vespa/config-summarymap.h> +#include <vespa/log/log.h> +LOG_SETUP("attibute_aspect_delayer_test"); + using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespa::config::search::IndexschemaConfig; @@ -36,15 +40,24 @@ namespace proton namespace { -AttributesConfig::Attribute make_sv_cfg(AttributesConfig::Attribute::Datatype dataType) +AttributesConfig::Attribute make_sv_cfg(const vespalib::string &name, AttributesConfig::Attribute::Datatype dataType) { AttributesConfig::Attribute attr; - attr.name = "a"; + attr.name = name; attr.datatype = dataType; attr.collectiontype = AttributesConfig::Attribute::Collectiontype::SINGLE; return attr; } +AttributesConfig::Attribute make_sv_cfg(AttributesConfig::Attribute::Datatype dataType) +{ + return make_sv_cfg("a", dataType); +} + +AttributesConfig::Attribute make_int32_sv_cfg(const vespalib::string &name) { + return make_sv_cfg(name, AttributesConfig::Attribute::Datatype::INT32); +} + AttributesConfig::Attribute make_int32_sv_cfg() { return make_sv_cfg(AttributesConfig::Attribute::Datatype::INT32); } @@ -104,6 +117,14 @@ SummarymapConfig::Override make_geopos_override(const vespalib::string &name) return override; } +SummarymapConfig::Override make_attribute_combiner_override(const vespalib::string &name) +{ + SummarymapConfig::Override override; + override.field = name; + override.command = "attributecombiner"; + return override; +} + SummarymapConfig smCfg(std::vector<SummarymapConfig::Override> overrides) { SummarymapConfigBuilder result; @@ -306,6 +327,29 @@ TEST_F("require that fast access flag change is not delayed, true->false edge, s TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } +TEST_F("require that adding attribute aspect to struct field is not delayed if field type is changed", Fixture) +{ + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")})); + TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg("array.a")})); + TEST_DO(f.assertSummarymapConfig({make_attribute_combiner_override("array")})); +} + +TEST_F("require that adding attribute aspect to struct field is delayed if field type is unchanged", Fixture) +{ + f.addFields({"array.a"}); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")})); + TEST_DO(f.assertAttributeConfig({})); + TEST_DO(f.assertSummarymapConfig({})); +} + +TEST_F("require that removing attribute aspect from struct field is not delayed", Fixture) +{ + f.addFields({"array.a"}); + f.setup(attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")}), attrCfg({}), smCfg({})); + TEST_DO(f.assertAttributeConfig({})); + TEST_DO(f.assertSummarymapConfig({})); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp index 7958d05d825..4957d3aead2 100644 --- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -10,6 +10,9 @@ #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_initializer_test"); + using search::attribute::Config; using search::attribute::BasicType; using search::attribute::CollectionType; diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 9bf4d43fff7..ce09a9a3742 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -1,7 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attribute_manager_test"); - #include <vespa/config-attributes.h> #include <vespa/fastos/file.h> #include <vespa/searchcommon/attribute/attributecontent.h> @@ -42,6 +39,9 @@ LOG_SETUP("attribute_manager_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_manager_test"); + namespace vespa { namespace config { namespace search {}}} using std::string; @@ -84,7 +84,7 @@ class MyAttributeFunctor : public proton::IAttributeFunctor std::vector<vespalib::string> _names; public: - virtual void + void operator()(const search::AttributeVector &attributeVector) override { _names.push_back(attributeVector.getName()); } @@ -166,13 +166,12 @@ BaseFixture::BaseFixture() _hwInfo() { } -BaseFixture::~BaseFixture() {} +BaseFixture::~BaseFixture() = default; struct AttributeManagerFixture { proton::AttributeManager::SP _msp; proton::AttributeManager &_m; - AttributeWriter _aw; ImportedAttributesRepoBuilder _builder; AttributeManagerFixture(BaseFixture &bf); ~AttributeManagerFixture(); @@ -191,10 +190,9 @@ AttributeManagerFixture::AttributeManagerFixture(BaseFixture &bf) : _msp(std::make_shared<proton::AttributeManager>(test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext, bf._attributeFieldWriter, bf._hwInfo)), _m(*_msp), - _aw(_msp), _builder() {} -AttributeManagerFixture::~AttributeManagerFixture() {} +AttributeManagerFixture::~AttributeManagerFixture() = default; struct Fixture : public BaseFixture, public AttributeManagerFixture { @@ -318,16 +316,21 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture) fillAttribute(a1, 1, 2, 200); EXPECT_EQUAL(4u, a1->getNumDocs()); AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded - EXPECT_EQUAL(5u, a2->getNumDocs()); - EXPECT_EQUAL(4u, a1->getNumDocs()); - amf._aw.onReplayDone(5u); - EXPECT_EQUAL(5u, a2->getNumDocs()); - EXPECT_EQUAL(5u, a1->getNumDocs()); - fillAttribute(a2, 1, 4, 200); - EXPECT_EQUAL(6u, a2->getNumDocs()); + { + AttributeWriter aw(amf._msp); + + EXPECT_EQUAL(5u, a2->getNumDocs()); + EXPECT_EQUAL(4u, a1->getNumDocs()); + aw.onReplayDone(5u); + EXPECT_EQUAL(5u, a2->getNumDocs()); + EXPECT_EQUAL(5u, a1->getNumDocs()); + fillAttribute(a2, 1, 4, 200); + EXPECT_EQUAL(6u, a2->getNumDocs()); + } AttributeVector::SP a3 = amf.addAttribute("a3"); // not-loaded + AttributeWriter aw(amf._msp); EXPECT_EQUAL(1u, a3->getNumDocs()); - amf._aw.onReplayDone(6); + aw.onReplayDone(6); EXPECT_EQUAL(6u, a3->getNumDocs()); fillAttribute(a3, 1, 7, 6, 200); EXPECT_EQUAL(7u, a3->getNumDocs()); @@ -352,10 +355,11 @@ TEST_F("require that attributes are flushed and loaded", BaseFixture) EXPECT_EQUAL(6u, a1->getNumDocs()); EXPECT_EQUAL(6u, a2->getNumDocs()); AttributeVector::SP a3 = amf.addAttribute("a3"); // loaded + AttributeWriter aw(amf._msp); EXPECT_EQUAL(6u, a1->getNumDocs()); EXPECT_EQUAL(6u, a2->getNumDocs()); EXPECT_EQUAL(7u, a3->getNumDocs()); - amf._aw.onReplayDone(7); + aw.onReplayDone(7); EXPECT_EQUAL(7u, a1->getNumDocs()); EXPECT_EQUAL(7u, a2->getNumDocs()); EXPECT_EQUAL(7u, a3->getNumDocs()); @@ -559,6 +563,7 @@ TEST_F("require that lid space can be compacted", Fixture) AttributeVector::SP a2 = f.addAttribute("a2"); AttributeVector::SP ex(new Int32Attribute("ex")); f._m.addExtraAttribute(ex); + AttributeWriter aw(f._msp); const int64_t attrValue = 33; fillAttribute(a1, 20, attrValue, 100); fillAttribute(a2, 20, attrValue, 100); @@ -571,7 +576,7 @@ TEST_F("require that lid space can be compacted", Fixture) EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit()); EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit()); - f._aw.compactLidSpace(10, 101); + aw.compactLidSpace(10, 101); EXPECT_EQUAL(21u, a1->getNumDocs()); EXPECT_EQUAL(21u, a2->getNumDocs()); @@ -587,6 +592,7 @@ TEST_F("require that lid space compaction op can be ignored", Fixture) AttributeVector::SP a2 = f.addAttribute("a2"); AttributeVector::SP ex(new Int32Attribute("ex")); f._m.addExtraAttribute(ex); + AttributeWriter aw(f._msp); const int64_t attrValue = 33; fillAttribute(a1, 20, attrValue, 200); fillAttribute(a2, 20, attrValue, 100); @@ -599,7 +605,7 @@ TEST_F("require that lid space compaction op can be ignored", Fixture) EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit()); EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit()); - f._aw.compactLidSpace(10, 101); + aw.compactLidSpace(10, 101); EXPECT_EQUAL(21u, a1->getNumDocs()); EXPECT_EQUAL(21u, a2->getNumDocs()); diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index c41b6bdcb97..0e6bd2c74fe 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -445,7 +445,7 @@ TEST_F("require that attribute writer handles update", Fixture) schema.addAttributeField(Schema::AttributeField("a2", schema::DataType::INT32, CollectionType::SINGLE)); DocBuilder idb(schema); const document::DocumentType &dt(idb.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("doc::1")); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 5))); upd.addUpdate(FieldUpdate(upd.getType().getField("a2")) @@ -489,7 +489,7 @@ TEST_F("require that attribute writer handles predicate update", Fixture) EXPECT_EQUAL(2u, a1->getNumDocs()); const document::DocumentType &dt(idb.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("doc::1")); PredicateFieldValue new_value(builder.feature("foo").value("bar").build()); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); @@ -678,7 +678,7 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) EXPECT_TRUE(tensor->equals(*tensor2)); const document::DocumentType &dt(builder.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*builder.getDocumentTypeRepo(), dt, DocumentId("doc::1")); auto new_tensor = createTensor({ {{{"x", "8"}, {"y", "9"}}, 11} }, {"x", "y"}); TensorFieldValue new_value; diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index d73cd805af1..8a5f58bd7e0 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -256,13 +256,12 @@ BaseFixture::BaseFixture(const HwInfo &hwInfo) _attributeFieldWriter(), _hwInfo(hwInfo) {} -BaseFixture::~BaseFixture() {} +BaseFixture::~BaseFixture() = default; struct AttributeManagerFixture { AttributeManager::SP _msp; AttributeManager &_m; - AttributeWriter _aw; AttributeManagerFixture(BaseFixture &bf); ~AttributeManagerFixture(); AttributeVector::SP addAttribute(const vespalib::string &name) { @@ -278,10 +277,9 @@ struct AttributeManagerFixture AttributeManagerFixture::AttributeManagerFixture(BaseFixture &bf) : _msp(std::make_shared<AttributeManager>(test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext, bf._attributeFieldWriter, bf._hwInfo)), - _m(*_msp), - _aw(_msp) + _m(*_msp) {} -AttributeManagerFixture::~AttributeManagerFixture() {} +AttributeManagerFixture::~AttributeManagerFixture() = default; struct Fixture : public BaseFixture, public AttributeManagerFixture { @@ -530,6 +528,7 @@ Test::requireThatShrinkWorks() Fixture f; AttributeManager &am = f._m; AttributeVector::SP av = f.addAttribute("a10"); + AttributeWriter aw(f._msp); av->addDocs(1000 - av->getNumDocs()); av->commit(50, 50); @@ -546,13 +545,13 @@ Test::requireThatShrinkWorks() EXPECT_FALSE(av->canShrinkLidSpace()); EXPECT_EQUAL(1000u, av->getNumDocs()); EXPECT_EQUAL(100u, av->getCommittedDocIdLimit()); - f._aw.heartBeat(51); + aw.heartBeat(51); EXPECT_TRUE(av->wantShrinkLidSpace()); EXPECT_FALSE(av->canShrinkLidSpace()); EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(), ft->getApproxMemoryGain().getAfter()); g.reset(); - f._aw.heartBeat(52); + aw.heartBeat(52); EXPECT_TRUE(av->wantShrinkLidSpace()); EXPECT_TRUE(av->canShrinkLidSpace()); EXPECT_TRUE(ft->getApproxMemoryGain().getBefore() > diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp index 6d69d9b225b..c5ae0f97875 100644 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp @@ -37,7 +37,7 @@ TEST_F("require that attribute write thread is blocked while guard is held", Fix { ReadGuard::UP guard = f.accessor.takeGuard(); Gate gate; - f.writer.execute("myattr", [&gate]() { gate.countDown(); }); + f.writer.execute(f.writer.getExecutorId(f.attribute->getNamePrefix()), [&gate]() { gate.countDown(); }); bool reachedZero = gate.await(100); EXPECT_FALSE(reachedZero); EXPECT_EQUAL(1u, gate.getCount()); diff --git a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp b/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp index 2dbff7d40dd..167865b5c68 100644 --- a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp +++ b/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp @@ -1,78 +1,131 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/common/document_type_inspector.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/configbuilder.h> -#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/base/field.h> +#include <vespa/document/datatype/datatypes.h> using namespace document; using namespace proton; -using document::config_builder::DocumenttypesConfigBuilderHelper; -using document::config_builder::Struct; -const int32_t doc_type_id = 787121340; -const vespalib::string type_name = "test"; -const vespalib::string header_name = type_name + ".header"; -const vespalib::string body_name = type_name + ".body"; - -std::unique_ptr<const DocumentTypeRepo> -makeOldDocTypeRepo() +template <class Type> +void +addFields(Type &type, bool fieldF3IsString, bool hasFieldF4, bool hasFieldF5) { - DocumenttypesConfigBuilderHelper builder; - builder.document(doc_type_id, type_name, - Struct(header_name), Struct(body_name). - addField("f1", DataType::T_STRING). - addField("f2", DataType::T_STRING). - addField("f3", DataType::T_STRING). - addField("f4", DataType::T_STRING)); - return std::unique_ptr<const DocumentTypeRepo>(new DocumentTypeRepo(builder.config())); + type.addField(Field("f1", 1, *DataType::STRING, true)); + type.addField(Field("f2", 2, *DataType::STRING, true)); + type.addField(Field("f3", 3, fieldF3IsString ? *DataType::STRING : *DataType::INT, true)); + if (hasFieldF4) { + type.addField(Field("f4", 4, *DataType::STRING, true)); + } + if (hasFieldF5) { + type.addField(Field("f5", 5, *DataType::STRING, true)); + } } -std::unique_ptr<const DocumentTypeRepo> -makeNewDocTypeRepo() +struct DocumentTypeFixture +{ + DocumentType _documentType; + StructDataType _structFieldType; + ArrayDataType _structArrayFieldType; + MapDataType _structMapFieldType; + MapDataType _mapFieldType; + + DocumentTypeFixture(bool fieldF3IsString, bool hasFieldF4, bool hasFieldF5, bool hasStruct, bool mapKeyIsByte); + ~DocumentTypeFixture(); +}; + +DocumentTypeFixture::DocumentTypeFixture(bool fieldF3IsString, bool hasFieldF4, bool hasFieldF5, bool hasStruct, bool mapKeyIsByte) + : _documentType("test"), + _structFieldType("struct"), + _structArrayFieldType(_structFieldType), + _structMapFieldType(mapKeyIsByte ? *DataType::BYTE : *DataType::STRING, _structFieldType), + _mapFieldType(mapKeyIsByte ? *DataType::BYTE : *DataType::STRING, *DataType::STRING) { - DocumenttypesConfigBuilderHelper builder; - builder.document(doc_type_id, type_name, - Struct(header_name), Struct(body_name). - addField("f1", DataType::T_STRING). - addField("f2", DataType::T_STRING). - addField("f3", DataType::T_INT). - addField("f5", DataType::T_STRING)); - return std::unique_ptr<const DocumentTypeRepo>(new DocumentTypeRepo(builder.config())); + addFields(_documentType, fieldF3IsString, hasFieldF4, hasFieldF5); + if (hasStruct) { + addFields(_structFieldType, fieldF3IsString, hasFieldF4, hasFieldF5); + _documentType.addField(Field("sarray", 11, _structArrayFieldType, true)); + _documentType.addField(Field("smap", 12, _structMapFieldType, true)); + _documentType.addField(Field("map", 13, _mapFieldType, true)); + } } +DocumentTypeFixture::~DocumentTypeFixture() = default; + struct Fixture { - std::unique_ptr<const DocumentTypeRepo> _oldRepo; - std::unique_ptr<const DocumentTypeRepo> _newRepo; + DocumentTypeFixture _oldDocType; + DocumentTypeFixture _newDocType; DocumentTypeInspector _inspector; - Fixture() - : _oldRepo(makeOldDocTypeRepo()), - _newRepo(makeNewDocTypeRepo()), - _inspector(*_oldRepo->getDocumentType("test"), *_newRepo->getDocumentType("test")) + Fixture(bool hasStruct = true, bool mapKeyIsByte = false) + : _oldDocType(true, true, false, hasStruct, mapKeyIsByte), + _newDocType(false, false, true, true, false), + _inspector(_oldDocType._documentType, _newDocType._documentType) { } }; TEST_F("require that unchanged fields are known", Fixture) { - EXPECT_TRUE(f._inspector.hasUnchangedField("f1")); - EXPECT_TRUE(f._inspector.hasUnchangedField("f2")); + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_TRUE(inspector.hasUnchangedField("f1")); + EXPECT_TRUE(inspector.hasUnchangedField("f2")); + EXPECT_TRUE(inspector.hasUnchangedField("sarray.f1")); + EXPECT_TRUE(inspector.hasUnchangedField("sarray.f2")); + EXPECT_TRUE(inspector.hasUnchangedField("smap.key")); + EXPECT_TRUE(inspector.hasUnchangedField("smap.value.f1")); + EXPECT_TRUE(inspector.hasUnchangedField("smap.value.f2")); + EXPECT_TRUE(inspector.hasUnchangedField("map.key")); + EXPECT_TRUE(inspector.hasUnchangedField("map.value")); } TEST_F("require that changed fields are detected", Fixture) { - EXPECT_FALSE(f._inspector.hasUnchangedField("f3")); + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("f3")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f3")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f3")); } TEST_F("require that partially missing fields are detected", Fixture) { - EXPECT_FALSE(f._inspector.hasUnchangedField("f4")); - EXPECT_FALSE(f._inspector.hasUnchangedField("f5")); + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("f4")); + EXPECT_FALSE(inspector.hasUnchangedField("f5")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f4")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f5")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f4")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f5")); } TEST_F("require that non-existing fields are NOT known", Fixture) { - EXPECT_FALSE(f._inspector.hasUnchangedField("not")); + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("not")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.not")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.not")); +} + +TEST_F("require that map key type change is detected", Fixture(true, true)) +{ + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("smap.key")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f1")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f2")); + EXPECT_FALSE(inspector.hasUnchangedField("map.key")); + EXPECT_FALSE(inspector.hasUnchangedField("map.value")); +} + +TEST_F("require that struct addition is detected", Fixture(false, false)) +{ + const IDocumentTypeInspector &inspector = f._inspector; + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f1")); + EXPECT_FALSE(inspector.hasUnchangedField("sarray.f2")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.key")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f1")); + EXPECT_FALSE(inspector.hasUnchangedField("smap.value.f2")); + EXPECT_FALSE(inspector.hasUnchangedField("map.key")); + EXPECT_FALSE(inspector.hasUnchangedField("map.value")); } TEST_MAIN() diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 7eae1c9d12d..db6116073e6 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -213,7 +213,9 @@ public: _tuneFileDocumentDB, _hwInfo); _configMgr.forwardConfig(b); _configMgr.nextGeneration(0); - if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { abort(); } + if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { + LOG_ABORT("should not be reached"); + } _ddb.reset(new DocumentDB("tmpdb", _configMgr.getConfig(), "tcp/localhost:9013", _queryLimiter, _clock, DocTypeName(docTypeName), makeBucketSpace(), *b->getProtonConfigSP(), *this, _summaryExecutor, _summaryExecutor, @@ -829,20 +831,16 @@ Test::requireThatAttributesAreUsed() "bi:[]}", *rep, 1, false)); TEST_DO(assertTensor(Tensor::UP(), "bj", *rep, 1, rclass)); - proton::IAttributeManager::SP attributeManager = - dc._ddb->getReadySubDB()->getAttributeManager(); - search::ISequencedTaskExecutor &attributeFieldWriter = - attributeManager->getAttributeFieldWriter(); - search::AttributeVector *bjAttr = - attributeManager->getWritableAttribute("bj"); - search::tensor::TensorAttribute *bjTensorAttr = - dynamic_cast<search::tensor::TensorAttribute *>(bjAttr); - - attributeFieldWriter. - execute("bj", - [&]() { bjTensorAttr->setTensor(3, - *createTensor({ {{{"x", "a"},{"y", "b"}}, 4} }, { "x"})); - bjTensorAttr->commit(); }); + proton::IAttributeManager::SP attributeManager = dc._ddb->getReadySubDB()->getAttributeManager(); + search::ISequencedTaskExecutor &attributeFieldWriter = attributeManager->getAttributeFieldWriter(); + search::AttributeVector *bjAttr = attributeManager->getWritableAttribute("bj"); + auto bjTensorAttr = dynamic_cast<search::tensor::TensorAttribute *>(bjAttr); + + attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(bjAttr->getNamePrefix()), + [&]() { + bjTensorAttr->setTensor(3, *createTensor({ {{{"x", "a"},{"y", "b"}}, 4} }, { "x"})); + bjTensorAttr->commit(); + }); attributeFieldWriter.sync(); DocsumReply::UP rep2 = dc._ddb->getDocsums(req); @@ -961,8 +959,7 @@ Test::requireThatUrisAreUsed() Document::UP exp = bc._bld.startDocument("doc::0"). startIndexField("urisingle"). startSubField("all"). - addUrlTokenizedString( - "http://www.example.com:81/fluke?ab=2#4"). + addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). @@ -986,8 +983,7 @@ Test::requireThatUrisAreUsed() startIndexField("uriarray"). startElement(1). startSubField("all"). - addUrlTokenizedString( - "http://www.example.com:82/fluke?ab=2#8"). + addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). @@ -1010,8 +1006,7 @@ Test::requireThatUrisAreUsed() endElement(). startElement(1). startSubField("all"). - addUrlTokenizedString( - "http://www.flickr.com:82/fluke?ab=2#9"). + addUrlTokenizedString("http://www.flickr.com:82/fluke?ab=2#9"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). @@ -1036,8 +1031,7 @@ Test::requireThatUrisAreUsed() startIndexField("uriwset"). startElement(4). startSubField("all"). - addUrlTokenizedString( - "http://www.example.com:83/fluke?ab=2#12"). + addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). @@ -1060,8 +1054,7 @@ Test::requireThatUrisAreUsed() endElement(). startElement(7). startSubField("all"). - addUrlTokenizedString( - "http://www.flickr.com:85/fluke?ab=2#13"). + addUrlTokenizedString("http://www.flickr.com:85/fluke?ab=2#13"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp index 7f8ce99beb9..73d071be96f 100644 --- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp +++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp @@ -33,7 +33,7 @@ #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/document/predicate/predicate.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> #include <vespa/searchsummary/docsummary/linguisticsannotation.h> #include <vespa/searchsummary/docsummary/searchdatatype.h> diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index a7512b41f30..3add3b727b9 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -17,6 +17,9 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/log/log.h> +LOG_SETUP("document_iterator_test"); + using document::BucketId; using document::DataType; using document::Document; diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp b/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp index b5cf1733ba9..dc4e88ae005 100644 --- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp @@ -6,6 +6,9 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/log/log.h> +LOG_SETUP("cluster_state_handler_test"); + using namespace proton; using document::BucketId; using storage::lib::Distribution; diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp index f06bb124eb8..958be6a4686 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("combiningfeedview_test"); #include <vespa/document/test/make_bucket_space.h> -#include <vespa/searchcore/proton/feedoperation/moveoperation.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/server/combiningfeedview.h> #include <vespa/searchcore/proton/test/test.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("combiningfeedview_test"); + using document::DocumentTypeRepo; using document::DocumentUpdate; using document::test::makeBucketSpace; @@ -22,8 +23,8 @@ typedef std::vector<IFeedView::SP> FeedViewVector; struct MyStreamHandler : public NewConfigOperation::IStreamHandler { - virtual void serializeConfig(SerialNum, vespalib::nbostream &) override {} - virtual void deserializeConfig(SerialNum, vespalib::nbostream &) override {} + void serializeConfig(SerialNum, vespalib::nbostream &) override {} + void deserializeConfig(SerialNum, vespalib::nbostream &) override {} }; diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index bab6aff821f..760f7457d33 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -14,6 +14,7 @@ #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> #include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h> #include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/document_subdb_explorer.h> #include <vespa/searchcore/proton/server/emptysearchview.h> diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 11e9aa8c3ec..af6a9d38385 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -18,6 +18,9 @@ #include <vespa/document/test/make_bucket_space.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("document_bucket_mover_test"); + using namespace proton; using document::BucketId; using document::Document; diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index e774728b41e..a1bce7174bf 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -316,7 +316,7 @@ struct UpdateContext { DocumentUpdate::SP update; BucketId bucketId; UpdateContext(const vespalib::string &docId, DocBuilder &builder) : - update(new DocumentUpdate(builder.getDocumentType(), DocumentId(docId))), + update(new DocumentUpdate(*builder.getDocumentTypeRepo(), builder.getDocumentType(), DocumentId(docId))), bucketId(BucketFactory::getBucketId(update->getId())) { } diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index a9fa79f513c..d21e5abe30b 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -11,6 +11,7 @@ #include <vespa/searchcore/proton/server/isummaryadapter.h> #include <vespa/searchcore/proton/server/matchview.h> #include <vespa/searchcore/proton/server/searchable_feed_view.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/test/document_meta_store_context_observer.h> #include <vespa/searchcore/proton/test/dummy_document_store.h> #include <vespa/searchcore/proton/test/dummy_summary_manager.h> @@ -478,11 +479,11 @@ struct DocumentContext DocumentContext::DocumentContext(const vespalib::string &docId, uint64_t timestamp, DocBuilder &builder) : doc(builder.startDocument(docId).startSummaryField("s1").addStr(docId).endField().endDocument().release()), - upd(new DocumentUpdate(builder.getDocumentType(), doc->getId())), + upd(new DocumentUpdate(*builder.getDocumentTypeRepo(), builder.getDocumentType(), doc->getId())), bid(BucketFactory::getNumBucketBits(), doc->getId().getGlobalId().convertToBucketId().getRawId()), ts(timestamp) {} -DocumentContext::~DocumentContext() {} +DocumentContext::~DocumentContext() = default; struct FeedTokenContext { diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index c7790f9c507..4a3ced6891c 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -153,14 +153,14 @@ struct MyDocumentRetriever : public DocumentRetrieverBaseForTest virtual const document::DocumentTypeRepo & getDocumentTypeRepo() const override { - abort(); + LOG_ABORT("should not be reached"); } virtual void getBucketMetaData(const storage::spi::Bucket &, DocumentMetaData::Vector &) const override { - abort(); + LOG_ABORT("should not be reached"); } virtual DocumentMetaData getDocumentMetaData(const DocumentId &) const override diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index e8247a52199..d07c29ead69 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -9,6 +9,8 @@ #include <vespa/searchcore/proton/server/putdonecontext.h> #include <vespa/searchcore/proton/server/removedonecontext.h> #include <vespa/searchcore/proton/server/storeonlyfeedview.h> +#include <vespa/searchcore/proton/feedoperation/moveoperation.h> +#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> #include <vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchcore/proton/test/thread_utils.h> diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index d56340be2b2..df411ea0346 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -113,7 +113,7 @@ public: virtual const RawDocumentMetaData &getRawMetaData(DocId) const override { - abort(); + LOG_ABORT("should not be reached"); } virtual bool getFreeListActive() const override diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index 681d3543ee1..e87e9209a17 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -100,14 +100,12 @@ makeDocTypeRepo() { DocumenttypesConfigBuilderHelper builder; builder.document(doc_type_id, type_name, - Struct(header_name), Struct(body_name). - addField("string", DataType::T_STRING). - addField("struct", Struct("pair"). - addField("x", DataType::T_STRING). - addField("y", DataType::T_STRING)). - addField("map", Map(DataType::T_STRING, - DataType::T_STRING))); - return std::unique_ptr<const DocumentTypeRepo>(new DocumentTypeRepo(builder.config())); + Struct(header_name), + Struct(body_name) + .addField("string", DataType::T_STRING) + .addField("struct", Struct("pair").addField("x", DataType::T_STRING).addField("y", DataType::T_STRING)) + .addField("map", Map(DataType::T_STRING, DataType::T_STRING))); + return std::make_unique<const DocumentTypeRepo>(builder.config()); } @@ -124,7 +122,7 @@ public: } auto makeUpdate() { - auto upd(std::make_shared<DocumentUpdate>(_docType, docId)); + auto upd(std::make_shared<DocumentUpdate>(*_repo, _docType, docId)); upd->addUpdate(FieldUpdate(upd->getType().getField("string")). addUpdate(AssignValueUpdate(StringFieldValue("newval")))); return upd; @@ -138,6 +136,7 @@ public: TEST("require that toString() on derived classes are meaningful") { + DocumentTypeRepo repo; BucketId bucket_id1(42); BucketId bucket_id2(43); BucketId bucket_id3(44); @@ -148,7 +147,7 @@ TEST("require that toString() on derived classes are meaningful") MyStreamHandler stream_handler; DocumentIdT doc_id_limit = 15; DocumentId doc_id("doc:foo:bar"); - DocumentUpdate::SP update(new DocumentUpdate(*DataType::DOCUMENT, doc_id)); + DocumentUpdate::SP update(new DocumentUpdate(repo, *DataType::DOCUMENT, doc_id)); EXPECT_EQUAL("DeleteBucket(BucketId(0x0000000000000000), serialNum=0)", DeleteBucketOperation().toString()); diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp index 8b58a9e271c..85960de2021 100644 --- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp @@ -240,6 +240,65 @@ TEST("requireThatPartitionsAreAddedCorrectly") { EXPECT_EQUAL(1.0, all1.getPartition(1).wait_time_max()); } +TEST("requireThatSoftDoomIsSetAndAdded") { + MatchingStats stats; + MatchingStats stats2; + EXPECT_EQUAL(0ul, stats.softDoomed()); + EXPECT_EQUAL(0.5, stats.softDoomFactor()); + stats.softDoomFactor(0.7); + stats.softDoomed(3); + EXPECT_EQUAL(3ul, stats.softDoomed()); + EXPECT_EQUAL(0.7, stats.softDoomFactor()); + stats2.add(stats); + EXPECT_EQUAL(3ul, stats2.softDoomed()); + EXPECT_EQUAL(0.5, stats2.softDoomFactor()); // Not affected by add +} + +TEST("requireThatSoftDoomFacorIsComputedCorrectlyForDownAdjustment") { + MatchingStats stats; + EXPECT_EQUAL(0ul, stats.softDoomed()); + EXPECT_EQUAL(0.5, stats.softDoomFactor()); + stats.softDoomed(1); + stats.updatesoftDoomFactor(1.0, 0.5, 2.0); + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.47, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.5, 2.0); + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.44, stats.softDoomFactor()); + stats.updatesoftDoomFactor(0.0009, 0.5, 2.0); // hard limits less than 1ms should be ignored + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.44, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.0009, 2.0); // soft limits less than 1ms should be ignored + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.44, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.5, 10.0); // Prevent changes above 10% + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.396, stats.softDoomFactor()); +} + +TEST("requireThatSoftDoomFacorIsComputedCorrectlyForUpAdjustment") { + MatchingStats stats; + EXPECT_EQUAL(0ul, stats.softDoomed()); + EXPECT_EQUAL(0.5, stats.softDoomFactor()); + stats.softDoomed(1); + stats.updatesoftDoomFactor(1.0, 0.9, 0.1); + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.508, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.9, 0.1); + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.516, stats.softDoomFactor()); + stats.updatesoftDoomFactor(0.0009, 0.9, 0.1); // hard limits less than 1ms should be ignored + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.516, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.0009, 0.1); // soft limits less than 1ms should be ignored + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.516, stats.softDoomFactor()); + stats.softDoomFactor(0.1); + stats.updatesoftDoomFactor(1.0, 0.9, 0.001); // Prevent changes above 5% + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.105, stats.softDoomFactor()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 76e3b3e4af9..0d474fc57cf 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> - #include <vespa/document/base/globalid.h> #include <initializer_list> #include <vespa/searchcommon/attribute/iattributecontext.h> @@ -33,6 +32,7 @@ #include <vespa/searchcore/proton/matching/match_params.h> #include <vespa/searchcore/proton/matching/match_tools.h> #include <vespa/searchcore/proton/matching/match_context.h> + #include <vespa/log/log.h> LOG_SETUP("matching_test"); diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 61823a17f09..82aab72068d 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -63,6 +63,12 @@ using search::queryeval::SearchIterator; using search::queryeval::SimpleBlueprint; using search::queryeval::SimpleResult; using search::queryeval::ParallelWeakAndBlueprint; +using search::queryeval::RankBlueprint; +using search::queryeval::AndBlueprint; +using search::queryeval::IntermediateBlueprint; +using search::queryeval::AndNotBlueprint; +using search::queryeval::SourceBlenderBlueprint; + using std::string; using std::vector; namespace fef_test = search::fef::test; @@ -106,6 +112,8 @@ class Test : public vespalib::TestApp { void requireThatWeakAndBlueprintsAreCreatedCorrectly(); void requireThatParallelWandBlueprintsAreCreatedCorrectly(); void requireThatWhiteListBlueprintCanBeUsed(); + void requireThatRankBlueprintStaysOnTopAfterWhiteListing(); + void requireThatAndNotBlueprintStaysOnTopAfterWhiteListing(); void requireThatSameElementTermsAreProperlyPrefixed(); void requireThatSameElementDoesNotAllocateMatchData(); void requireThatSameElementIteratorsCanBeBuilt(); @@ -879,6 +887,54 @@ Test::requireThatWhiteListBlueprintCanBeUsed() EXPECT_EQUAL(exp, act); } +template<typename T1, typename T2> +void verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing(QueryBuilder<ProtonNodeTypes> & builder) { + builder.addStringTerm("foo", field, field_id, string_weight); + builder.addStringTerm("bar", field, field_id, string_weight); + builder.addStringTerm("baz", field, field_id, string_weight); + std::string stackDump = StackDumpCreator::create(*builder.build()); + Query query; + query.buildTree(stackDump, "", ViewResolver(), plain_index_env); + FakeSearchContext context(42); + context.addIdx(0).idx(0).getFake() + .addResult(field, "foo", FakeResult().doc(1)); + context.setLimit(42); + + query.setWhiteListBlueprint(std::make_unique<SimpleBlueprint>(SimpleResult())); + + FakeRequestContext requestContext; + MatchDataLayout mdl; + query.reserveHandles(requestContext, context, mdl); + const IntermediateBlueprint * root = dynamic_cast<const T1 *>(query.peekRoot()); + ASSERT_TRUE(root != nullptr); + EXPECT_EQUAL(2u, root->childCnt()); + const IntermediateBlueprint * second = dynamic_cast<const T2 *>(&root->getChild(0)); + ASSERT_TRUE(second != nullptr); + EXPECT_EQUAL(2u, second->childCnt()); + const AndBlueprint * first = dynamic_cast<const AndBlueprint *>(&second->getChild(0)); + ASSERT_TRUE(first != nullptr); + EXPECT_EQUAL(2u, first->childCnt()); + EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&first->getChild(0))); + EXPECT_TRUE(dynamic_cast<const SimpleBlueprint *>(&first->getChild(1))); + EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&second->getChild(1))); + EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&root->getChild(1))); +} + +void Test::requireThatRankBlueprintStaysOnTopAfterWhiteListing() { + QueryBuilder<ProtonNodeTypes> builder; + builder.addRank(2); + builder.addAndNot(2); + verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing<RankBlueprint, AndNotBlueprint>(builder); +} + +void Test::requireThatAndNotBlueprintStaysOnTopAfterWhiteListing() { + QueryBuilder<ProtonNodeTypes> builder; + builder.addAndNot(2); + builder.addRank(2); + verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing<AndNotBlueprint, RankBlueprint>(builder); +} + + search::query::Node::UP make_same_element_stack_dump(const vespalib::string &prefix, const vespalib::string &term_prefix) { @@ -984,6 +1040,8 @@ Test::Main() TEST_CALL(requireThatWeakAndBlueprintsAreCreatedCorrectly); TEST_CALL(requireThatParallelWandBlueprintsAreCreatedCorrectly); TEST_CALL(requireThatWhiteListBlueprintCanBeUsed); + TEST_CALL(requireThatRankBlueprintStaysOnTopAfterWhiteListing); + TEST_CALL(requireThatAndNotBlueprintStaysOnTopAfterWhiteListing); TEST_CALL(requireThatSameElementTermsAreProperlyPrefixed); TEST_CALL(requireThatSameElementDoesNotAllocateMatchData); TEST_CALL(requireThatSameElementIteratorsCanBeBuilt); diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index 6607019cccc..297f75054b2 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -30,7 +30,6 @@ #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/vespalib/testkit/testapp.h> - #include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 73580416e65..7f065d0cc15 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -67,7 +67,9 @@ createDoc(const DocumentType &docType, const DocumentId &docId) document::DocumentUpdate::SP createUpd(const DocumentType& docType, const DocumentId &docId) { - return document::DocumentUpdate::SP(new document::DocumentUpdate(docType, docId)); + static std::vector<std::unique_ptr<document::DocumentTypeRepo>> repoList; + repoList.emplace_back(std::make_unique<document::DocumentTypeRepo>(docType)); + return std::make_shared<document::DocumentUpdate>(*repoList.back(), docType, docId); } storage::spi::ClusterState diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 045c8de9384..a89f2e3f0ff 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -12,7 +12,7 @@ #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/bootstrapconfigmanager.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> -#include <vespa/searchcore/proton/server/i_document_db_config_owner.h> +#include <vespa/searchcore/proton/server/document_db_config_owner.h> #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/proton_configurer.h> #include <vespa/searchcore/proton/server/i_proton_configurer_owner.h> @@ -208,13 +208,13 @@ struct ConfigFixture { struct MyProtonConfigurerOwner; -struct MyDocumentDBConfigOwner : public IDocumentDBConfigOwner +struct MyDocumentDBConfigOwner : public DocumentDBConfigOwner { vespalib::string _name; MyProtonConfigurerOwner &_owner; MyDocumentDBConfigOwner(const vespalib::string &name, MyProtonConfigurerOwner &owner) - : IDocumentDBConfigOwner(), + : DocumentDBConfigOwner(), _name(name), _owner(owner) { @@ -240,12 +240,12 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner } virtual ~MyProtonConfigurerOwner() { } - virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName &docTypeName, - document::BucketSpace bucketSpace, - const vespalib::string &configId, - const std::shared_ptr<BootstrapConfig> &bootstrapConfig, - const std::shared_ptr<DocumentDBConfig> &documentDBConfig, - InitializeThreads initializeThreads) override + virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, + document::BucketSpace bucketSpace, + const vespalib::string &configId, + const std::shared_ptr<BootstrapConfig> &bootstrapConfig, + const std::shared_ptr<DocumentDBConfig> &documentDBConfig, + InitializeThreads initializeThreads) override { (void) bucketSpace; (void) configId; @@ -257,7 +257,7 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner std::ostringstream os; os << "add db " << docTypeName.getName() << " " << documentDBConfig->getGeneration(); _log.push_back(os.str()); - return db.get(); + return db; } virtual void removeDocumentDB(const DocTypeName &docTypeName) override { ASSERT_FALSE(_dbs.find(docTypeName) == _dbs.end()); diff --git a/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp index fc9d2d770cd..9b268350b0d 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp @@ -1,6 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> - #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/reference/document_db_reference.h> #include <vespa/searchcore/proton/test/mock_attribute_manager.h> diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index 93c192b57e8..7be774f7291 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -306,6 +306,18 @@ TEST("require that added attribute aspect with flushed attribute after interrupt EXPECT_TRUE(f.assertAttributes({})); } +TEST_F("require that removed attribute aspect from struct field does not require document field populate", Fixture) +{ + f.addOldConfig({"array.a"}, {"array.a"}).addNewConfig({"array.a"}, {}).init(); + EXPECT_TRUE(f.assertFields({})); +} + +TEST_F("require that added attribute aspect to struct field requires attribute populate", Fixture) +{ + f.addOldConfig({"array.a"}, {}).addNewConfig({"array.a"}, {"array.a"}).init(); + EXPECT_TRUE(f.assertAttributes({"array.a"})); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index 69e90a6fdf8..f8ec0d20d33 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -33,6 +33,9 @@ #include <vespa/searchlib/attribute/stringbase.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("document_retriever_test"); + using document::ArrayFieldValue; using document::FieldValue; using document::BucketId; diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index dfa461e2b57..f206ffc9b17 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for feedstates. -#include <vespa/log/log.h> -LOG_SETUP("feedstates_test"); #include <vespa/document/base/documentid.h> #include <vespa/document/base/testdocrepo.h> @@ -11,6 +9,7 @@ LOG_SETUP("feedstates_test"); #include <vespa/searchcore/proton/server/feedstates.h> #include <vespa/searchcore/proton/server/ireplayconfig.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> +#include <vespa/searchcore/proton/feedoperation/removeoperation.h> #include <vespa/searchcore/proton/test/dummy_feed_view.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/objects/nbostream.h> @@ -18,6 +17,9 @@ LOG_SETUP("feedstates_test"); #include <vespa/vespalib/util/buffer.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> +#include <vespa/log/log.h> +LOG_SETUP("feedstates_test"); + using document::BucketId; using document::DocumentId; using document::DocumentTypeRepo; @@ -83,7 +85,7 @@ Fixture::Fixture() state("doctypename", feed_view_ptr, _bucketDBHandler, replay_config, config_store) { } -Fixture::~Fixture() {} +Fixture::~Fixture() = default; struct RemoveOperationContext @@ -107,7 +109,7 @@ RemoveOperationContext::RemoveOperationContext(search::SerialNum serial) packet.reset(new Packet()); packet->add(Packet::Entry(serial, FeedOperation::REMOVE, buf)); } -RemoveOperationContext::~RemoveOperationContext() {} +RemoveOperationContext::~RemoveOperationContext() = default; TEST_F("require that active FeedView can change during replay", Fixture) { RemoveOperationContext opCtx(10); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index e66466aa1cb..ca1ea67d288 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -231,6 +231,9 @@ summary.cache.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 ## 9 is a reasonable default for both summary.cache.compression.level int default=9 +## Control if cache entry is updated or ivalidated when changed. +summary.cache.update_strategy enum {INVALIDATE, UPDATE} default=INVALIDATE + ## Control compression type of the summary while in memory during compaction ## NB So far only stragey=LOG honours it. summary.log.compact.compression.type enum {NONE, LZ4, ZSTD} default=ZSTD @@ -467,7 +470,7 @@ hwinfo.cpu.cores int default = 0 restart ## max(ceil(hwinfo.cpu.cores * feeding.concurrency), summary.log.numthreads) ## The number of threads in each of pools 2-4 is calculated as: ## max(ceil((hwinfo.cpu.cores * feeding.concurrency)/3), indexing.threads) -feeding.concurrency double default = 0.5 restart +feeding.concurrency double default = 0.2 restart ## Adjustment to resource limit when determining if maintenance jobs can run. ## diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp index ea981d1cc01..d99b32ac138 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp @@ -198,16 +198,13 @@ FastS_DataSetCollection::GetDataSet() bool FastS_DataSetCollection::AreEnginesReady() { - bool ready = true; - - for (uint32_t datasetidx = 0; - ready && (datasetidx < GetMaxNumDataSets()); - datasetidx++) - { + for (uint32_t datasetidx = 0; datasetidx < GetMaxNumDataSets(); datasetidx++) { FastS_DataSetBase *dataset = PeekDataSet(datasetidx); - ready = (dataset != nullptr && !dataset->AreEnginesReady()); + if ((dataset != nullptr) && !dataset->AreEnginesReady()) { + return false; + } } - return ready; + return true; } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp index 9b596d6e992..577d6e7edf5 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp @@ -69,7 +69,7 @@ FastS_FNET_SearchNode::FastS_FNET_SearchNode(FastS_FNET_SearchNode &&) { // These objects are referenced everywhere and must never be either copied nor moved, // but as std::vector requires this to exist so we do this little trick. - assert(false); + LOG_ABORT("should not reach here"); } bool diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp index 48baba329ca..3ef5a0f63b1 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp @@ -21,9 +21,7 @@ GroupingManager::GroupingManager(GroupingContext & groupingContext) { } -GroupingManager::~GroupingManager() -{ -} +GroupingManager::~GroupingManager() = default; using search::expression::ExpressionNode; using search::expression::AttributeNode; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp index 9807faa5021..cf803ec0368 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp @@ -29,11 +29,15 @@ bool fastPartialUpdateAttribute(const search::attribute::Config &cfg) { (basicType != BasicType::Type::REFERENCE)); } +bool isStructFieldAttribute(const vespalib::string &name) { + return name.find('.') != vespalib::string::npos; +} + bool willTriggerReprocessOnAttributeAspectRemoval(const search::attribute::Config &cfg, const IIndexschemaInspector &indexschemaInspector, const vespalib::string &name) { - return fastPartialUpdateAttribute(cfg) && !indexschemaInspector.isStringIndex(name); + return fastPartialUpdateAttribute(cfg) && !indexschemaInspector.isStringIndex(name) && !isStructFieldAttribute(name); } @@ -73,6 +77,7 @@ handleNewAttributes(const AttributesConfig &oldAttributesConfig, SummarymapConfigBuilder &summarymapConfig) { vespalib::hash_set<vespalib::string> delayed; + vespalib::hash_set<vespalib::string> delayedStruct; AttributesConfigHash oldAttrs(oldAttributesConfig.attribute); for (const auto &newAttr : newAttributesConfig.attribute) { search::attribute::Config newCfg = ConfigConverter::convert(newAttr); @@ -102,6 +107,10 @@ handleNewAttributes(const AttributesConfig &oldAttributesConfig, } else { // Delay addition of attribute aspect delayed.insert(newAttr.name); + auto pos = newAttr.name.find('.'); + if (pos != vespalib::string::npos) { + delayedStruct.insert(newAttr.name.substr(0, pos)); + } } } } @@ -111,6 +120,11 @@ handleNewAttributes(const AttributesConfig &oldAttributesConfig, if (itr == delayed.end()) { summarymapConfig.override.emplace_back(override); } + } else if (override.command == "attributecombiner") { + auto itr = delayedStruct.find(override.field); + if (itr == delayedStruct.end()) { + summarymapConfig.override.emplace_back(override); + } } else { summarymapConfig.override.emplace_back(override); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp index f775f4443f8..f2e4ac4905d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp @@ -92,10 +92,9 @@ AttributeDirectory::saveSnapInfo() { if (!_snapInfo.save()) { vespalib::string dirName(getDirName()); - LOG(warning, - "Could not save meta-info file for attribute vector '%s' to disk", + LOG(warning, "Could not save meta-info file for attribute vector '%s' to disk", dirName.c_str()); - abort(); + LOG_ABORT("should not be reached"); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 239f91b449f..eb6020b8d5f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -4,19 +4,19 @@ #include "ifieldupdatecallback.h" #include "attributemanager.h" #include "document_field_extractor.h" -#include <vespa/document/base/exceptions.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/attrupdate.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/searchlib/common/idestructorcallback.h> - +#include <vespa/document/base/exceptions.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> -LOG_SETUP(".proton.server.attributeadapter"); +LOG_SETUP(".proton.attribute.attribute_writer"); using namespace document; using namespace search; @@ -47,6 +47,8 @@ AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) docType.buildFieldPath(fp, name); } catch (document::FieldNotFoundException & e) { fp = FieldPath(); + } catch (vespalib::IllegalArgumentException &e) { + fp = FieldPath(); } _fieldPath = std::move(fp); } @@ -230,7 +232,7 @@ public: FieldContext::FieldContext(ISequencedTaskExecutor &writer, AttributeVector *attr) : _name(attr->getName()), - _executorId(writer.getExecutorId(_name)), + _executorId(writer.getExecutorId(attr->getNamePrefix())), _attr(attr) { } @@ -412,7 +414,7 @@ AttributeWriter::setupWriteContexts() { std::vector<FieldContext> fieldContexts; assert(_writeContexts.empty()); - for (auto attr : _writableAttributes) { + for (auto attr : getWritableAttributes()) { fieldContexts.emplace_back(_attributeFieldWriter, attr); } std::sort(fieldContexts.begin(), fieldContexts.end()); @@ -469,14 +471,23 @@ AttributeWriter::internalRemove(SerialNum serialNum, DocumentIdT lid, bool immed AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr) : _mgr(mgr), _attributeFieldWriter(mgr->getAttributeFieldWriter()), - _writableAttributes(mgr->getWritableAttributes()), _writeContexts(), _dataType(nullptr), - _hasStructFieldAttribute(false) + _hasStructFieldAttribute(false), + _attrMap() { setupWriteContexts(); + setupAttriuteMapping(); } +void AttributeWriter::setupAttriuteMapping() { + for (auto attr : getWritableAttributes()) { + vespalib::stringref name = attr->getName(); + _attrMap[name] = AttrWithId(attr, _attributeFieldWriter.getExecutorId(attr->getNamePrefix())); + } +} + + AttributeWriter::~AttributeWriter() { _attributeFieldWriter.sync(); @@ -545,7 +556,8 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document for (const auto &fupd : upd.getUpdates()) { LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().c_str()); - AttributeVector *attrp = _mgr->getWritableAttribute(fupd.getField().getName()); + auto found = _attrMap.find(fupd.getField().getName()); + AttributeVector * attrp = (found != _attrMap.end()) ? found->second.first : nullptr; onUpdate.onUpdateField(fupd.getField().getName(), attrp); if (attrp == nullptr) { LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().c_str()); @@ -555,7 +567,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document // document and attribute. if (attrp->getStatus().getLastSyncToken() >= serialNum) continue; - args[_attributeFieldWriter.getExecutorId(attrp->getName()).getId()]->_updates.emplace_back(attrp, &fupd); + args[found->second.second.getId()]->_updates.emplace_back(attrp, &fupd); LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attrp->getName().c_str()); } // NOTE: The lifetime of the field update will be ensured by keeping the document update alive @@ -572,11 +584,10 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document void AttributeWriter::heartBeat(SerialNum serialNum) { - for (auto attrp : _writableAttributes) { - auto &attr = *attrp; - _attributeFieldWriter.execute(attr.getName(), - [serialNum, &attr]() - { applyHeartBeat(serialNum, attr); }); + for (auto entry : _attrMap) { + _attributeFieldWriter.execute(entry.second.second, + [serialNum, attr=entry.second.first]() + { applyHeartBeat(serialNum, *attr); }); } } @@ -601,11 +612,10 @@ AttributeWriter::forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) void AttributeWriter::onReplayDone(uint32_t docIdLimit) { - for (auto attrp : _writableAttributes) { - auto &attr = *attrp; - _attributeFieldWriter.execute(attr.getName(), - [docIdLimit, &attr]() - { applyReplayDone(docIdLimit, attr); }); + for (auto entry : _attrMap) { + _attributeFieldWriter.execute(entry.second.second, + [docIdLimit, attr = entry.second.first]() + { applyReplayDone(docIdLimit, *attr); }); } _attributeFieldWriter.sync(); } @@ -614,12 +624,11 @@ AttributeWriter::onReplayDone(uint32_t docIdLimit) void AttributeWriter::compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) { - for (auto attrp : _writableAttributes) { - auto &attr = *attrp; + for (auto entry : _attrMap) { _attributeFieldWriter. - execute(attr.getName(), - [wantedLidLimit, serialNum, &attr]() - { applyCompactLidSpace(wantedLidLimit, serialNum, attr); }); + execute(entry.second.second, + [wantedLidLimit, serialNum, attr=entry.second.first]() + { applyCompactLidSpace(wantedLidLimit, serialNum, *attr); }); } _attributeFieldWriter.sync(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index f89089ed335..4ea7f3fda6c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/document/base/fieldpath.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> +#include <vespa/vespalib/stllike/hash_map.h> namespace document { class DocumentType; } @@ -25,7 +26,6 @@ private: typedef document::FieldValue FieldValue; const IAttributeManager::SP _mgr; search::ISequencedTaskExecutor &_attributeFieldWriter; - const std::vector<search::AttributeVector *> &_writableAttributes; using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; public: class WriteField @@ -58,11 +58,15 @@ public: bool hasStructFieldAttribute() const { return _hasStructFieldAttribute; } }; private: + using AttrWithId = std::pair<search::AttributeVector *, ExecutorId>; + using AttrMap = vespalib::hash_map<vespalib::string, AttrWithId>; std::vector<WriteContext> _writeContexts; const DataType *_dataType; bool _hasStructFieldAttribute; + AttrMap _attrMap; void setupWriteContexts(); + void setupAttriuteMapping(); void buildFieldPaths(const DocumentType &docType, const DataType *dataType); void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone); @@ -73,13 +77,13 @@ public: AttributeWriter(const proton::IAttributeManager::SP &mgr); ~AttributeWriter(); + /* Only for in tests that add attributes after AttributeWriter construction. */ + /** * Implements IAttributeWriter. */ - std::vector<search::AttributeVector *> - getWritableAttributes() const override; - search::AttributeVector * - getWritableAttribute(const vespalib::string &name) const override; + std::vector<search::AttributeVector *> getWritableAttributes() const override; + search::AttributeVector *getWritableAttribute(const vespalib::string &name) const override; void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) override; void remove(SerialNum serialNum, DocumentIdT lid, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index ef818f7b407..5faf89aa149 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -78,8 +78,8 @@ std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector:: using Type = IFlushTarget::Type; using Component = IFlushTarget::Component; + auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter, attributeFieldWriter.getExecutorId(attr->getNamePrefix())); const vespalib::string &name = attr->getName(); - auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter, attributeFieldWriter.getExecutorId(name)); auto dir = diskLayout.createAttributeDir(name); search::SerialNum shrinkSerialNum = estimateShrinkSerialNum(*attr); return std::make_shared<ShrinkLidSpaceFlushTarget>("attribute.shrink." + name, Type::GC, Component::ATTRIBUTE, shrinkSerialNum, dir->getLastFlushTime(), shrinkwrap); @@ -569,16 +569,15 @@ AttributeManager::getWritableAttributes() const void -AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> - func) const +AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const { for (const auto &attr : _attributes) { if (attr.second.isExtra()) { continue; } AttributeVector::SP attrsp = attr.second.getAttribute(); - _attributeFieldWriter. - execute(attr.first, [attrsp, func]() { (*func)(*attrsp); }); + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(attrsp->getNamePrefix()), + [attrsp, func]() { (*func)(*attrsp); }); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp index d9a0ff3d8dd..d1d5b1c9af7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp @@ -37,8 +37,7 @@ ExclusiveAttributeReadAccessor(const AttributeVector::SP &attribute, namespace { void -attributeWriteBlockingTask(GateSP entranceGate, - GateSP exitGate) +attributeWriteBlockingTask(GateSP entranceGate, GateSP exitGate) { entranceGate->countDown(); exitGate->await(); @@ -51,9 +50,8 @@ ExclusiveAttributeReadAccessor::takeGuard() { GateSP entranceGate = std::make_shared<Gate>(); GateSP exitGate = std::make_shared<Gate>(); - _attributeFieldWriter.execute(_attribute->getName(), - [entranceGate, exitGate]() - { attributeWriteBlockingTask(entranceGate, exitGate); }); + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(_attribute->getNamePrefix()), + [entranceGate, exitGate]() { attributeWriteBlockingTask(entranceGate, exitGate); }); entranceGate->await(); return std::make_unique<Guard>(*_attribute, exitGate); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index d3a74bb9a98..8474efb15c9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -197,9 +197,8 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> search::AttributeVector::SP attrsp = guard.getSP(); // Name must be extracted in document db master thread or attribute // writer thread - vespalib::string attributeName = attrsp->getName(); - attributeFieldWriter. - execute(attributeName, [attrsp, func]() { (*func)(*attrsp); }); + attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(attrsp->getNamePrefix()), + [attrsp, func]() { (*func)(*attrsp); }); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index a658b11263a..7716fc5ee61 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -243,10 +243,8 @@ FlushableAttribute::initFlush(SerialNum currentSerial) // Called by document db executor std::promise<IFlushTarget::Task::UP> promise; std::future<IFlushTarget::Task::UP> future = promise.get_future(); - _attributeFieldWriter.execute(_attr->getName(), - [&]() { promise.set_value( - internalInitFlush(currentSerial)); - }); + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(_attr->getNamePrefix()), + [&]() { promise.set_value(internalInitFlush(currentSerial)); }); return future.get(); } @@ -257,5 +255,4 @@ FlushableAttribute::getApproxBytesToWriteToDisk() const return _attr->getEstimatedSaveByteSize(); } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index 6fa19f0f35e..b76451b377d 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -5,6 +5,8 @@ #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.common.attribute_field_value_node"); namespace proton { @@ -80,7 +82,7 @@ getValue(const Context &context) const } while (0); break; default: - abort(); + LOG_ABORT("should not be reached"); } return Value::UP(); diff --git a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp index 7c0a1815019..dac0fbc2cc7 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp @@ -17,13 +17,12 @@ #include <vespa/searchlib/common/base.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/attribute/reference_attribute.h> - #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/changevector.hpp> #include <sstream> #include <vespa/log/log.h> -LOG_SETUP(".attrupdate"); +LOG_SETUP(".proton.common.attrupdate"); using namespace document; using vespalib::make_string; diff --git a/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp index 6cff162ae08..e19fa5351c2 100644 --- a/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp @@ -1,6 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_type_inspector.h" +#include <vespa/document/base/exceptions.h> +#include <vespa/document/base/fieldpath.h> + +using document::FieldPath; +using document::FieldPathEntry; namespace proton { @@ -14,12 +19,28 @@ DocumentTypeInspector::DocumentTypeInspector(const document::DocumentType &oldDo bool DocumentTypeInspector::hasUnchangedField(const vespalib::string &name) const { - if (!_oldDocType.hasField(name) || !_newDocType.hasField(name)) { + FieldPath oldPath; + FieldPath newPath; + try { + _oldDocType.buildFieldPath(oldPath, name); + _newDocType.buildFieldPath(newPath, name); + } catch (document::FieldNotFoundException &e) { + return false; + } catch (vespalib::IllegalArgumentException &e) { return false; } - const document::Field &oldField = _oldDocType.getField(name); - const document::Field &newField = _newDocType.getField(name); - return oldField == newField; + if (oldPath.size() != newPath.size()) { + return false; + } + for (uint32_t i = 0; i < oldPath.size(); ++i) { + const auto &oldEntry = oldPath[i]; + const auto &newEntry = newPath[i]; + if (oldEntry.getType() != newEntry.getType() || + oldEntry.getDataType() != newEntry.getDataType()) { + return false; + } + } + return true; } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index c9bf81d1310..190e904ad68 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -10,6 +10,8 @@ #include <vespa/vespalib/io/fileutil.h> #include <experimental/filesystem> #include <thread> +#include <vespa/log/log.h> +LOG_SETUP(".proton.common.hw_info_sampler"); using config::ConfigHandle; using config::ConfigSubscriber; @@ -70,7 +72,7 @@ void writeConfig(const vespalib::string &path, builder.disk.sampletime = std::chrono::duration_cast<std::chrono::seconds>(sampleTime.time_since_epoch()).count(); config::FileConfigWriter writer(path + "/hwinfo.cfg"); if (!writer.write(builder)) { - abort(); + LOG_ABORT("should not be reached"); } } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index ead5c1d1ddf..54961d10fd3 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -161,7 +161,7 @@ SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, const LogDoc fileHeaderContext, tlSyncer, bucketizer); } -SummaryManager::~SummaryManager() {} +SummaryManager::~SummaryManager() = default; void SummaryManager::putDocument(uint64_t syncToken, search::DocumentIdT lid, const Document & doc) @@ -204,7 +204,7 @@ IFlushTarget::List SummaryManager::getFlushTargets(searchcorespi::index::IThread { IFlushTarget::List ret; ret.push_back(std::make_shared<SummaryFlushTarget>(getBackingStore(), summaryService)); - if (dynamic_cast<LogDocumentStore *>(_docStore.get()) != NULL) { + if (dynamic_cast<LogDocumentStore *>(_docStore.get()) != nullptr) { ret.push_back(std::make_shared<SummaryCompactTarget>(summaryService, getBackingStore())); } ret.push_back(createShrinkLidSpaceFlushTarget(summaryService, _docStore)); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp index c11326090dc..eee0be77c26 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp @@ -32,7 +32,7 @@ SummaryManagerInitializer(const search::GrowStrategy &grow, _result(result) { } -SummaryManagerInitializer::~SummaryManagerInitializer() {} +SummaryManagerInitializer::~SummaryManagerInitializer() = default; void SummaryManagerInitializer::run() diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index d2bbde1eadd..838efdfe5a4 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -21,6 +21,8 @@ #include <vespa/fastos/file.h> #include "document_meta_store_versions.h" +#include <vespa/log/log.h> +LOG_SETUP(".proton.documentmetastore"); using document::BucketId; using document::GlobalId; @@ -77,7 +79,7 @@ public: _headerLen = _header.readFile(*_datFile); _datFile->SetPosition(_headerLen); if (!search::ReaderBase::extractFileSize(_header, *_datFile, _datFileSize)) { - abort(); + LOG_ABORT("should not be reached"); } _docIdLimit = _header.getTag(DOCID_LIMIT).asInteger(); _version = _header.getTag(VERSION).asInteger(); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp index e10e6ed539b..5a4c15ed27a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp @@ -36,8 +36,9 @@ private: protected: const DocumentMetaStore & _store; public: - GidAllSearchIterator(TermFieldMatchData *matchData, const DocumentMetaStore &store) - : AttributeIteratorBase(matchData), + GidAllSearchIterator(const search::attribute::ISearchContext &baseSearchCtx, + TermFieldMatchData *matchData, const DocumentMetaStore &store) + : AttributeIteratorBase(baseSearchCtx, matchData), _store(store) { } @@ -64,9 +65,10 @@ private: } public: - GidStrictAllSearchIterator(TermFieldMatchData *matchData, + GidStrictAllSearchIterator(const search::attribute::ISearchContext &baseSearchCtx, + TermFieldMatchData *matchData, const DocumentMetaStore &store) - : GidAllSearchIterator(matchData, store), + : GidAllSearchIterator(baseSearchCtx, matchData, store), _numDocs(store.getNumDocs()) { } @@ -88,8 +90,9 @@ private: } } public: - GidSearchIterator(TermFieldMatchData *matchData, const DocumentMetaStore &store, const GlobalId &gid) - : GidAllSearchIterator(matchData, store), + GidSearchIterator(const search::attribute::ISearchContext &baseSearchCtx, + TermFieldMatchData *matchData, const DocumentMetaStore &store, const GlobalId &gid) + : GidAllSearchIterator(baseSearchCtx, matchData, store), _gid(gid) { } @@ -119,10 +122,10 @@ SearchIterator::UP SearchContext::createIterator(TermFieldMatchData *matchData, bool strict) { return _isWord - ? std::make_unique<GidSearchIterator>(matchData, getStore(), _gid) + ? std::make_unique<GidSearchIterator>(*this, matchData, getStore(), _gid) : strict - ? std::make_unique<GidStrictAllSearchIterator>(matchData, getStore()) - : std::make_unique<GidAllSearchIterator>(matchData, getStore()); + ? std::make_unique<GidStrictAllSearchIterator>(*this, matchData, getStore()) + : std::make_unique<GidAllSearchIterator>(*this, matchData, getStore()); } const DocumentMetaStore & diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp index cd906891b92..c2a3439388a 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "compact_lid_space_operation.h" +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> namespace proton { @@ -27,8 +28,7 @@ CompactLidSpaceOperation::serialize(vespalib::nbostream& os) const } void -CompactLidSpaceOperation::deserialize(vespalib::nbostream& is, - const document::DocumentTypeRepo&) +CompactLidSpaceOperation::deserialize(vespalib::nbostream& is, const document::DocumentTypeRepo&) { is >> _subDbId; is >> _lidLimit; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h index 8a00c739126..77b95547bd0 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h @@ -2,10 +2,10 @@ #pragma once #include <vespa/searchlib/common/serialnum.h> -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/stllike/string.h> namespace document { class DocumentTypeRepo; } - +namespace vespalib { class nbostream; } namespace proton { class FeedOperation diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp index 71a701bdb40..67e4d1b4287 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp @@ -2,6 +2,7 @@ #include "lidvectorcontext.h" #include <vespa/searchlib/common/bitvector.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".proton.feedoperation.lidvectorcontext"); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h index b432bbcf20a..b307e50da0a 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.h @@ -2,9 +2,10 @@ #pragma once #include <vespa/searchlib/query/base.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vector> +namespace vespalib { class nbostream; } + namespace proton { class LidVectorContext @@ -29,5 +30,4 @@ public: size_t getNumLids() const { return _result.size(); } }; -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h index b7b041135d4..144aa534bf7 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.h @@ -2,30 +2,29 @@ #pragma once #include "feedoperation.h" -#include <vespa/vespalib/objects/nbostream.h> namespace proton { + namespace feedoperation { + struct IStreamHandler { + virtual ~IStreamHandler() {} + virtual void serializeConfig(search::SerialNum serialNum, vespalib::nbostream &os) = 0; + virtual void deserializeConfig(search::SerialNum serialNum, vespalib::nbostream &is) = 0; + }; + } + class NewConfigOperation : public FeedOperation { public: - struct IStreamHandler { - virtual ~IStreamHandler() {} - virtual void serializeConfig(SerialNum serialNum, - vespalib::nbostream &os) = 0; - virtual void deserializeConfig(SerialNum serialNum, - vespalib::nbostream &is) = 0; - }; + using IStreamHandler = feedoperation::IStreamHandler; private: IStreamHandler &_streamHandler; public: - NewConfigOperation(SerialNum serialNum, - IStreamHandler &streamHandler); - virtual ~NewConfigOperation() {} - virtual void serialize(vespalib::nbostream &os) const override; - virtual void deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) override; - virtual vespalib::string toString() const override; + NewConfigOperation(SerialNum serialNum, IStreamHandler &streamHandler); + ~NewConfigOperation() override {} + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; + vespalib::string toString() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp index eea693e4680..69f754cd594 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp @@ -2,6 +2,7 @@ #include "pruneremoveddocumentsoperation.h" #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/objects/nbostream.h> #include <cassert> #include <vespa/log/log.h> @@ -22,8 +23,7 @@ PruneRemovedDocumentsOperation::PruneRemovedDocumentsOperation() PruneRemovedDocumentsOperation:: -PruneRemovedDocumentsOperation(DocumentIdT docIdLimit, - uint32_t subDbId) +PruneRemovedDocumentsOperation(DocumentIdT docIdLimit, uint32_t subDbId) : RemoveDocumentsOperation(FeedOperation::PRUNE_REMOVED_DOCUMENTS), _subDbId(subDbId) { @@ -44,8 +44,7 @@ PruneRemovedDocumentsOperation::serialize(vespalib::nbostream &os) const void -PruneRemovedDocumentsOperation::deserialize(vespalib::nbostream &is, - const DocumentTypeRepo &) +PruneRemovedDocumentsOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &) { is >> _subDbId; deserializeLidsToRemove(is); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp index 6aa479611d1..ef482a19ca3 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "removedocumentsoperation.h" +#include <vespa/vespalib/objects/nbostream.h> namespace proton { @@ -16,11 +17,9 @@ RemoveDocumentsOperation::serializeLidsToRemove(vespalib::nbostream &os) const { uint32_t mapSize = _lidsToRemoveMap.size(); os << mapSize; - for (LidsToRemoveMap::const_iterator - it = _lidsToRemoveMap.begin(), ite = _lidsToRemoveMap.end(); - it != ite; ++it) { - os << it->first; - it->second->serialize(os); + for (const auto & entry : _lidsToRemoveMap) { + os << entry.first; + entry.second->serialize(os); } } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp index f3ef73f4d90..16ddedc4745 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "spoolerreplayoperation.h" +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> using vespalib::make_string; @@ -14,10 +15,7 @@ SpoolerReplayOperation::SpoolerReplayOperation(Type type) { } - -SpoolerReplayOperation::SpoolerReplayOperation(Type type, - SerialNum serialNum, - SerialNum spoolerSerialNum) +SpoolerReplayOperation::SpoolerReplayOperation(Type type, SerialNum serialNum, SerialNum spoolerSerialNum) : FeedOperation(type), _spoolerSerialNum(spoolerSerialNum) { @@ -39,11 +37,8 @@ SpoolerReplayOperation::deserialize(vespalib::nbostream &is) } vespalib::string SpoolerReplayOperation::toString() const { - return make_string( - "SpoolerReplay%s(spoolerSerialNum=%" PRIu64 - ", serialNum=%" PRIu64 ")", - getType() == SPOOLER_REPLAY_START ? "Start" : "Complete", - _spoolerSerialNum, getSerialNum()); + return make_string("SpoolerReplay%s(spoolerSerialNum=%" PRIu64", serialNum=%" PRIu64 ")", + getType() == SPOOLER_REPLAY_START ? "Start" : "Complete", _spoolerSerialNum, getSerialNum()); } @@ -53,8 +48,7 @@ SpoolerReplayStartOperation::SpoolerReplayStartOperation() } -SpoolerReplayStartOperation::SpoolerReplayStartOperation(SerialNum serialNum, - SerialNum spoolerSerialNum) +SpoolerReplayStartOperation::SpoolerReplayStartOperation(SerialNum serialNum, SerialNum spoolerSerialNum) : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_START, serialNum, spoolerSerialNum) @@ -70,12 +64,8 @@ SpoolerReplayCompleteOperation::SpoolerReplayCompleteOperation() SpoolerReplayCompleteOperation::SpoolerReplayCompleteOperation(SerialNum serialNum, SerialNum spoolerSerialNum) - : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_COMPLETE, - serialNum, - spoolerSerialNum) + : SpoolerReplayOperation(FeedOperation::SPOOLER_REPLAY_COMPLETE, serialNum, spoolerSerialNum) { } - - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h index 028d01dec20..028ad1c6bfa 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.h @@ -11,15 +11,12 @@ private: SerialNum _spoolerSerialNum; protected: SpoolerReplayOperation(Type type); - SpoolerReplayOperation(Type type, - SerialNum serialNum, - SerialNum spoolerSerialNum); + SpoolerReplayOperation(Type type, SerialNum serialNum, SerialNum spoolerSerialNum); public: - virtual ~SpoolerReplayOperation() {} + ~SpoolerReplayOperation() override {} SerialNum getSpoolerSerialNum() const { return _spoolerSerialNum; } - virtual void serialize(vespalib::nbostream &os) const override; - virtual void deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &) override { + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &) override { deserialize(is); } void deserialize(vespalib::nbostream &is); @@ -38,8 +35,7 @@ public: * @param serialNum the current serial number of the transaction log. * @param spoolerSerialNum the serial number of the first entry of the spooler log replay. */ - SpoolerReplayStartOperation(SerialNum serialNum, - SerialNum spoolerSerialNum); + SpoolerReplayStartOperation(SerialNum serialNum, SerialNum spoolerSerialNum); }; @@ -54,8 +50,7 @@ public: * @param serialNum the current serial number of the transaction log. * @param spoolerSerialNum the serial number of the last entry of the spooler log replay. */ - SpoolerReplayCompleteOperation(SerialNum serialNum, - SerialNum spoolerSerialNum); + SpoolerReplayCompleteOperation(SerialNum serialNum, SerialNum spoolerSerialNum); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index d31f1faec77..dc83848152c 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -51,12 +51,11 @@ UpdateOperation::serializeUpdate(vespalib::nbostream &os) const } void -UpdateOperation::deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) +UpdateOperation::deserializeUpdate(vespalib::nbostream && is, const document::DocumentTypeRepo &repo) { - document::ByteBuffer buf(is.peek(), is.size()); - DocumentUpdate::UP update = (getType() == UPDATE_42) ? DocumentUpdate::create42(repo, buf) : DocumentUpdate::createHEAD(repo, buf); - is.adjustReadPos(buf.getPos()); - _upd = std::move(update); + _upd = (getType() == UPDATE_42) + ? DocumentUpdate::create42(repo, is) + : DocumentUpdate::createHEAD(repo, std::move(is)); } void @@ -73,7 +72,7 @@ UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &re { DocumentOperation::deserialize(is, repo); try { - deserializeUpdate(is, repo); + deserializeUpdate(std::move(is), repo); } catch (document::DocumentTypeNotFoundException &e) { LOG(warning, "Failed deserialize update operation using unknown document type '%s'", e.getDocumentTypeName().c_str()); @@ -83,18 +82,19 @@ UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &re } void -UpdateOperation::deserializeUpdate(const DocumentTypeRepo &repo) +UpdateOperation::verifyUpdate(const DocumentTypeRepo &repo) { vespalib::nbostream stream; serializeUpdate(stream); - deserializeUpdate(stream, repo); + deserializeUpdate(std::move(stream), repo); + _upd->eagerDeserialize(); // Will trigger exceptions if incompatible } -vespalib::string UpdateOperation::toString() const { +vespalib::string +UpdateOperation::toString() const { return make_string("%s(%s, %s)", ((getType() == FeedOperation::UPDATE_42) ? "Update42" : "Update"), - _upd.get() ? - _upd->getId().getScheme().toString().c_str() : "NULL", + _upd.get() ? _upd->getId().getScheme().toString().c_str() : "NULL", docArgsToString().c_str()); } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h index 99dcbfbce6c..83e87fea096 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h @@ -19,7 +19,7 @@ private: const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); void serializeUpdate(vespalib::nbostream &os) const; - void deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo); + void deserializeUpdate(vespalib::nbostream && is, const document::DocumentTypeRepo &repo); public: UpdateOperation(); UpdateOperation(Type type); @@ -30,7 +30,7 @@ public: const DocumentUpdateSP &getUpdate() const { return _upd; } void serialize(vespalib::nbostream &os) const override; void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; - void deserializeUpdate(const document::DocumentTypeRepo &repo); + void verifyUpdate(const document::DocumentTypeRepo &repo); vespalib::string toString() const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp index d7f38a0cc52..c5e6acd80ed 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "wipehistoryoperation.h" +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> using vespalib::make_string; @@ -9,29 +10,29 @@ namespace proton { WipeHistoryOperation::WipeHistoryOperation() : FeedOperation(FeedOperation::WIPE_HISTORY), - _wipeTimeLimit(0) { + _wipeTimeLimit(0) +{ } WipeHistoryOperation::WipeHistoryOperation(SerialNum serialNum, fastos::TimeStamp wipeTimeLimit) : FeedOperation(FeedOperation::WIPE_HISTORY), - _wipeTimeLimit(wipeTimeLimit) { + _wipeTimeLimit(wipeTimeLimit) +{ setSerialNum(serialNum); } void WipeHistoryOperation::serialize(vespalib::nbostream &str) const { str << _wipeTimeLimit; } -void WipeHistoryOperation::deserialize(vespalib::nbostream &str, - const document::DocumentTypeRepo &) { +void WipeHistoryOperation::deserialize(vespalib::nbostream &str, const document::DocumentTypeRepo &) { fastos::TimeStamp::TimeT t; str >> t; _wipeTimeLimit = t; } vespalib::string WipeHistoryOperation::toString() const { - return make_string("WipeHistory(wipeTimeLimit=%" PRIu64 - ", serialNum=%" PRIu64 ")", + return make_string("WipeHistory(wipeTimeLimit=%" PRIu64 ", serialNum=%" PRIu64 ")", _wipeTimeLimit.ns(), getSerialNum()); } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h index 80b551dbd92..0cf7256bf27 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.h @@ -12,14 +12,13 @@ class WipeHistoryOperation : public FeedOperation { public: WipeHistoryOperation(); WipeHistoryOperation(SerialNum serialNum, fastos::TimeStamp wipeTimeLimit); - virtual ~WipeHistoryOperation() {} + ~WipeHistoryOperation() override {} fastos::TimeStamp getWipeTimeLimit() const { return _wipeTimeLimit; } - virtual void serialize(vespalib::nbostream &str) const override; - virtual void deserialize(vespalib::nbostream &str, - const document::DocumentTypeRepo &) override; - virtual vespalib::string toString() const override; + void serialize(vespalib::nbostream &str) const override; + void deserialize(vespalib::nbostream &str, const document::DocumentTypeRepo &) override; + vespalib::string toString() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index 95b3008985b..0d2c556b4d6 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -8,6 +8,7 @@ #include "tls_stats_factory.h" #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/vespalib/util/jsonwriter.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP(".proton.flushengine.flushengine"); @@ -22,8 +23,7 @@ namespace proton { namespace { search::SerialNum -findOldestFlushedSerial(const IFlushTarget::List &lst, - const IFlushHandler &handler) +findOldestFlushedSerial(const IFlushTarget::List &lst, const IFlushHandler &handler) { search::SerialNum ret(handler.getCurrentSerialNumber()); for (const IFlushTarget::SP & target : lst) { @@ -33,42 +33,46 @@ findOldestFlushedSerial(const IFlushTarget::List &lst, return ret; } +void +logTarget(const char * text, const FlushContext & ctx) { + LOG(debug, "Target '%s' %s flush of transactions %" PRIu64 " through %" PRIu64 ".", + ctx.getName().c_str(), text, + ctx.getTarget()->getFlushedSerialNum() + 1, + ctx.getHandler()->getCurrentSerialNumber()); +} + } -FlushEngine::FlushMeta::FlushMeta(const vespalib::string & name, fastos::TimeStamp start, uint32_t id) : - _name(name), - _start(start), - _id(id) +FlushEngine::FlushMeta::FlushMeta(const vespalib::string & name, fastos::TimeStamp start, uint32_t id) + : _name(name), + _start(start), + _id(id) { } -FlushEngine::FlushMeta::~FlushMeta() { } +FlushEngine::FlushMeta::~FlushMeta() = default; -FlushEngine::FlushInfo::FlushInfo() : - FlushMeta("", fastos::ClockSystem::now(), 0), - _target() +FlushEngine::FlushInfo::FlushInfo() + : FlushMeta("", fastos::ClockSystem::now(), 0), + _target() { } -FlushEngine::FlushInfo::~FlushInfo() { } +FlushEngine::FlushInfo::~FlushInfo() = default; -FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, - const IFlushTarget::SP &target, - const vespalib::string & destination) : - FlushMeta(destination, fastos::ClockSystem::now(), taskId), - _target(target) +FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, const IFlushTarget::SP &target, const vespalib::string & destination) + : FlushMeta(destination, fastos::ClockSystem::now(), taskId), + _target(target) { } -FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> - tlsStatsFactory, - IFlushStrategy::SP strategy, uint32_t numThreads, - uint32_t idleIntervalMS) +FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory, + IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS) : _closed(false), _maxConcurrent(numThreads), _idleIntervalMS(idleIntervalMS), _taskId(0), _threadPool(128 * 1024), - _strategy(strategy), + _strategy(std::move(strategy)), _priorityStrategy(), _executor(numThreads, 128 * 1024), _lock(), @@ -78,11 +82,9 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> _setStrategyLock(), _strategyLock(), _strategyCond(), - _tlsStatsFactory(tlsStatsFactory), + _tlsStatsFactory(std::move(tlsStatsFactory)), _pendingPrune() -{ - // empty -} +{ } FlushEngine::~FlushEngine() { @@ -92,7 +94,7 @@ FlushEngine::~FlushEngine() FlushEngine & FlushEngine::start() { - if (_threadPool.NewThread(this) == NULL) { + if (_threadPool.NewThread(this) == nullptr) { throw vespalib::IllegalStateException("Failed to start engine thread."); } return *this; @@ -148,10 +150,8 @@ FlushEngine::wait(size_t minimumWaitTimeIfReady) } void -FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg) +FlushEngine::Run(FastOS_ThreadInterface *, void *) { - (void)thread; - (void)arg; bool shouldIdle = false; vespalib::string prevFlushName; while (wait(shouldIdle ? _idleIntervalMS : 0)) { @@ -161,13 +161,14 @@ FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg) } prevFlushName = flushNextTarget(prevFlushName); if ( ! prevFlushName.empty()) { - // Sleep at least 10 ms after a successful flush in order to avoid busy loop in case - // of strategy error or target error. - FastOS_Thread::Sleep(10); + // Sleep 1 ms after a successful flush in order to avoid busy loop in case + // of strategy or target error. + std::this_thread::sleep_for(1ms); } else { shouldIdle = true; } - LOG(debug, "Making another wait(idle=%s, timeMS=%d) last was '%s'", shouldIdle ? "true" : "false", shouldIdle ? _idleIntervalMS : 0, prevFlushName.c_str()); + LOG(debug, "Making another wait(idle=%s, timeMS=%d) last was '%s'", + shouldIdle ? "true" : "false", shouldIdle ? _idleIntervalMS : 0, prevFlushName.c_str()); } _executor.sync(); prune(); @@ -211,18 +212,16 @@ FlushEngine::getTargetList(bool includeFlushingTargets) const for (const auto & it : _handlers) { IFlushHandler & handler(*it.second); search::SerialNum serial(handler.getCurrentSerialNumber()); - LOG(spam, "Checking FlushHandler '%s' current serial = %ld", - handler.getName().c_str(), serial); + LOG(spam, "Checking FlushHandler '%s' current serial = %ld", handler.getName().c_str(), serial); IFlushTarget::List lst = handler.getFlushTargets(); for (const IFlushTarget::SP & target : lst) { - LOG(spam, "Checking target '%s' with flushedSerialNum = %ld", target->getName().c_str(), target->getFlushedSerialNum()); + LOG(spam, "Checking target '%s' with flushedSerialNum = %ld", + target->getName().c_str(), target->getFlushedSerialNum()); if (!isFlushing(guard, FlushContext::createName(handler, *target)) || includeFlushingTargets) { - ret.push_back(FlushContext::SP(new FlushContext(it.second, - IFlushTarget::SP(new CachedFlushTarget(target)), - serial))); + ret.push_back(std::make_shared<FlushContext>(it.second, std::make_shared<CachedFlushTarget>(target), serial)); } else { LOG(debug, "Target '%s' with flushedSerialNum = %ld already has a flush going. Local last serial = %ld.", - target->getName().c_str(), target->getFlushedSerialNum(), serial); + target->getName().c_str(), target->getFlushedSerialNum(), serial); } } } @@ -258,17 +257,12 @@ FlushEngine::initNextFlush(const FlushContext::List &lst) break; } } - if (ctx.get() != NULL) { - LOG(debug, "Target '%s' initiated flush of transactions %" PRIu64 " through %" PRIu64 ".", - ctx->getName().c_str(), - ctx->getTarget()->getFlushedSerialNum() + 1, - ctx->getHandler()->getCurrentSerialNumber()); + if (ctx) { + logTarget("initiated", *ctx); } return ctx; } - - void FlushEngine::flushAll(const FlushContext::List &lst) { @@ -276,19 +270,12 @@ FlushEngine::flushAll(const FlushContext::List &lst) for (const FlushContext::SP & ctx : lst) { if (wait(0)) { if (ctx->initFlush()) { - LOG(debug, "Target '%s' initiated flush of transactions %" PRIu64 " through %" PRIu64 ".", - ctx->getName().c_str(), - ctx->getTarget()->getFlushedSerialNum() + 1, - ctx->getHandler()->getCurrentSerialNumber()); - _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx))); + logTarget("initiated", *ctx); + _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx)); } else { - LOG(debug, "Target '%s' failed to initiate flush of transactions %" PRIu64 " through %" PRIu64 ".", - ctx->getName().c_str(), - ctx->getTarget()->getFlushedSerialNum() + 1, - ctx->getHandler()->getCurrentSerialNumber()); + logTarget("failed to initiate", *ctx); } } - } } @@ -311,17 +298,17 @@ FlushEngine::flushNextTarget(const vespalib::string & name) return ""; } FlushContext::SP ctx = initNextFlush(lst.first); - if (ctx.get() == NULL) { + if ( ! ctx) { LOG(debug, "All targets refused to flush."); return ""; } if ( name == ctx->getName()) { LOG(info, "The same target %s out of %ld has been asked to flush again. " - "This might indicate flush logic flaw so I will wait 1s before doing it.", + "This might indicate flush logic flaw so I will wait 100 ms before doing it.", name.c_str(), lst.first.size()); - FastOS_Thread::Sleep(1000); + std::this_thread::sleep_for(100ms); } - _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx))); + _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx)); return ctx->getName(); } @@ -330,12 +317,8 @@ FlushEngine::initFlush(const FlushContext &ctx) { if (LOG_WOULD_LOG(event)) { IFlushTarget::MemoryGain mgain(ctx.getTarget()->getApproxMemoryGain()); - EventLogger::flushStart(ctx.getName(), - mgain.getBefore(), - mgain.getAfter(), - mgain.gain(), - ctx.getTarget()->getFlushedSerialNum() + 1, - ctx.getHandler()->getCurrentSerialNumber()); + EventLogger::flushStart(ctx.getName(), mgain.getBefore(), mgain.getAfter(), mgain.gain(), + ctx.getTarget()->getFlushedSerialNum() + 1, ctx.getHandler()->getCurrentSerialNumber()); } return initFlush(ctx.getHandler(), ctx.getTarget()); } @@ -350,10 +333,7 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId) } if (LOG_WOULD_LOG(event)) { FlushStats stats = ctx.getTarget()->getLastFlushStats(); - EventLogger::flushComplete(ctx.getName(), - duration.ms(), - stats.getPath(), - stats.getPathElementsToLog()); + EventLogger::flushComplete(ctx.getName(), duration.ms(), stats.getPath(), stats.getPathElementsToLog()); } LOG(debug, "FlushEngine::flushDone(taskId='%d') took '%f' secs", taskId, duration.sec()); std::lock_guard<std::mutex> guard(_lock); @@ -366,8 +346,7 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId) } IFlushHandler::SP -FlushEngine::putFlushHandler(const DocTypeName &docTypeName, - const IFlushHandler::SP &flushHandler) +FlushEngine::putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler::SP &flushHandler) { std::lock_guard<std::mutex> guard(_lock); IFlushHandler::SP result(_handlers.putHandler(docTypeName, flushHandler)); @@ -379,13 +358,6 @@ FlushEngine::putFlushHandler(const DocTypeName &docTypeName, } IFlushHandler::SP -FlushEngine::getFlushHandler(const DocTypeName &docTypeName) const -{ - std::lock_guard<std::mutex> guard(_lock); - return _handlers.getHandler(docTypeName); -} - -IFlushHandler::SP FlushEngine::removeFlushHandler(const DocTypeName &docTypeName) { std::lock_guard<std::mutex> guard(_lock); @@ -430,7 +402,7 @@ FlushEngine::setStrategy(IFlushStrategy::SP strategy) return; } assert(!_priorityStrategy); - _priorityStrategy = strategy; + _priorityStrategy = std::move(strategy); { std::lock_guard<std::mutex> guard(_lock); _cond.notify_all(); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 19175f9ce2a..c1be05ba067 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -15,7 +15,7 @@ namespace proton { namespace flushengine { class ITlsStatsFactory; } -class FlushEngine : public FastOS_Runnable +class FlushEngine final : public FastOS_Runnable { public: class FlushMeta { @@ -37,9 +37,7 @@ private: struct FlushInfo : public FlushMeta { FlushInfo(); - FlushInfo(uint32_t taskId, - const IFlushTarget::SP &target, - const vespalib::string &destination); + FlushInfo(uint32_t taskId, const IFlushTarget::SP &target, const vespalib::string &destination); ~FlushInfo(); IFlushTarget::SP _target; @@ -96,14 +94,13 @@ public: * @param numThreads The number of worker threads to use. * @param idleInterval The interval between when flushes are checked whne there are no one progressing. */ - FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> - tlsStatsFactory, + FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory, IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS); /** * Destructor. Waits for all pending tasks to complete. */ - ~FlushEngine(); + ~FlushEngine() override; /** * Observe and reset internal executor stats @@ -145,19 +142,8 @@ public: * @param flushHandler The handler to register. * @return The replaced handler, if any. */ - IFlushHandler::SP - putFlushHandler(const DocTypeName &docTypeName, - const IFlushHandler::SP &flushHandler); + IFlushHandler::SP putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler::SP &flushHandler); - /** - * Returns the flush handler for the given document type. If no handler was - * registered, this method returns an empty shared pointer. - * - * @param docType The document type whose handler to return. - * @return The registered handler, if any. - */ - IFlushHandler::SP - getFlushHandler(const DocTypeName &docTypeName) const; /** * Removes and returns the flush handler for the given document type. If no @@ -166,10 +152,8 @@ public: * @param docType The document type whose handler to remove. * @return The removed handler, if any. */ - IFlushHandler::SP - removeFlushHandler(const DocTypeName &docTypeName); + IFlushHandler::SP removeFlushHandler(const DocTypeName &docTypeName); - // Implements FastOS_Runnable. void Run(FastOS_ThreadInterface *thread, void *arg) override; FlushMetaSet getCurrentlyFlushingSet() const; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h index 3a24330f8ec..914de9df30c 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h @@ -30,9 +30,7 @@ public: */ IFlushHandler(const vespalib::string &name) : _name(name) - { - // empty - } + { } /** * Virtual destructor required for inheritance. @@ -76,8 +74,7 @@ public: * This method is called to sync tls to stable media, up to and * including the given serial number. * - * @param syncTo The last serial number that has to be persisted to stable - * media. + * @param syncTo The last serial number that has to be persisted to stable media. */ virtual void syncTls(SerialNum syncTo) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.cpp index 088e202c027..c9ba495aa82 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_map.cpp @@ -2,6 +2,8 @@ #include "tls_stats_map.h" #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/log/log.h> +LOG_SETUP(".proton.flushengine.tls_stats_map"); namespace proton { namespace flushengine { @@ -18,7 +20,7 @@ TlsStatsMap::getTlsStats(const vespalib::string &domain) const { if (itr != _map.end()) { return itr->second; } - abort(); + LOG_ABORT("should not be reached"); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h index 04440831045..c22232d47db 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h +++ b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h @@ -2,14 +2,12 @@ #pragma once -#include <vespa/searchlib/common/feature.h> #include <vespa/searchlib/queryeval/scores.h> #include <utility> #include <cstddef> #include <vector> -namespace proton { -namespace matching { +namespace proton::matching { struct IMatchLoopCommunicator { typedef search::feature_t feature_t; @@ -31,6 +29,4 @@ struct IMatchLoopCommunicator { virtual ~IMatchLoopCommunicator() {} }; -} // namespace matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp index fa41c73838b..95148ef56e8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp @@ -3,15 +3,14 @@ #include "match_loop_communicator.h" #include <vespa/vespalib/util/priority_queue.h> -namespace proton { -namespace matching { +namespace proton:: matching { MatchLoopCommunicator::MatchLoopCommunicator(size_t threads, size_t topN) : _estimate_match_frequency(threads), _selectBest(threads, topN), _rangeCover(threads) {} -MatchLoopCommunicator::~MatchLoopCommunicator() {} +MatchLoopCommunicator::~MatchLoopCommunicator() = default; void MatchLoopCommunicator::EstimateMatchFrequency::mingle() @@ -72,5 +71,4 @@ MatchLoopCommunicator::RangeCover::mingle() } } -} // namespace matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h index 3b83cb471f7..c1eec37299f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h @@ -5,8 +5,7 @@ #include "i_match_loop_communicator.h" #include <vespa/vespalib/util/rendezvous.h> -namespace proton { -namespace matching { +namespace proton::matching { class MatchLoopCommunicator : public IMatchLoopCommunicator { @@ -14,13 +13,13 @@ private: struct EstimateMatchFrequency : vespalib::Rendezvous<Matches, double> { EstimateMatchFrequency(size_t n) : vespalib::Rendezvous<Matches, double>(n) {} - virtual void mingle() override; + void mingle() override; }; struct SelectBest : vespalib::Rendezvous<std::vector<feature_t>, size_t> { size_t topN; SelectBest(size_t n, size_t topN_in) : vespalib::Rendezvous<std::vector<feature_t>, size_t>(n), topN(topN_in) {} - virtual void mingle() override; + void mingle() override; bool cmp(const uint32_t &a, const uint32_t &b) { return (in(a)[out(a)] > in(b)[out(b)]); } @@ -34,8 +33,7 @@ private: }; struct RangeCover : vespalib::Rendezvous<RangePair, RangePair> { RangeCover(size_t n) - : vespalib::Rendezvous<RangePair, RangePair>(n) {} - virtual void mingle() override; + : vespalib::Rendezvous<RangePair, RangePair>(n) {}void mingle() override; }; EstimateMatchFrequency _estimate_match_frequency; SelectBest _selectBest; @@ -45,17 +43,15 @@ public: MatchLoopCommunicator(size_t threads, size_t topN); ~MatchLoopCommunicator(); - virtual double estimate_match_frequency(const Matches &matches) override { + double estimate_match_frequency(const Matches &matches) override { return _estimate_match_frequency.rendezvous(matches); } - virtual size_t selectBest(const std::vector<feature_t> &sortedScores) override { + size_t selectBest(const std::vector<feature_t> &sortedScores) override { return _selectBest.rendezvous(sortedScores); } - virtual RangePair rangeCover(const RangePair &ranges) override { + RangePair rangeCover(const RangePair &ranges) override { return _rangeCover.rendezvous(ranges); } }; -} // namespace matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 370c4b930e1..0eb49aec754 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -10,8 +10,7 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.matching.match_master"); -namespace proton { -namespace matching { +namespace proton::matching { using namespace search::fef; using search::queryeval::SearchIterator; @@ -23,15 +22,15 @@ struct TimedMatchLoopCommunicator : IMatchLoopCommunicator { IMatchLoopCommunicator &communicator; fastos::StopWatch rerank_time; TimedMatchLoopCommunicator(IMatchLoopCommunicator &com) : communicator(com) {} - virtual double estimate_match_frequency(const Matches &matches) override { + double estimate_match_frequency(const Matches &matches) override { return communicator.estimate_match_frequency(matches); } - virtual size_t selectBest(const std::vector<feature_t> &sortedScores) override { + size_t selectBest(const std::vector<feature_t> &sortedScores) override { size_t result = communicator.selectBest(sortedScores); rerank_time.start(); return result; } - virtual RangePair rangeCover(const RangePair &ranges) override { + RangePair rangeCover(const RangePair &ranges) override { RangePair result = communicator.rangeCover(ranges); rerank_time.stop(); return result; @@ -131,18 +130,15 @@ MatchMaster::getFeatureSet(const MatchToolsFactory &matchToolsFactory, if (search.seek(docs[i])) { uint32_t docId = search.getDocId(); search.unpack(docId); - search::feature_t * f = fs.getFeaturesByIndex( - fs.addDocId(docId)); + search::feature_t * f = fs.getFeaturesByIndex(fs.addDocId(docId)); for (uint32_t j = 0; j < featureNames.size(); ++j) { f[j] = resolver.resolve(j).as_number(docId); } } else { - LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. " - "Skipping hit", docs[i]); + LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. Skipping hit", docs[i]); } } return retval; } -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.h b/searchcore/src/vespa/searchcore/proton/matching/match_master.h index b9980023259..4c6463cf75d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.h @@ -8,8 +8,7 @@ namespace vespalib { class ThreadBundle; } namespace search { class FeatureSet; } -namespace proton { -namespace matching { +namespace proton::matching { class MatchToolsFactory; class MatchParams; @@ -37,6 +36,4 @@ public: static MatchingStats getStats(MatchMaster && rhs) { return std::move(rhs._stats); } }; -} // namespace proton::matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index 5c78ee59b1d..8384a45e0e2 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -103,7 +103,7 @@ public: ResultProcessor &rp, vespalib::DualMergeDirector &md, uint32_t distributionKey); - virtual void run() override; + void run() override; const MatchingStats::Partition &get_thread_stats() const { return thread_stats; } double get_match_time() const { return match_time_s; } PartialResult::UP extract_result() { return std::move(resultContext->result); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 998daed16c0..e7773c94d72 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -3,9 +3,8 @@ #include "match_tools.h" #include "querynodes.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> - #include <vespa/log/log.h> -LOG_SETUP(".searchcore.matching.match_tools"); +LOG_SETUP(".proton.matching.match_tools"); #include <vespa/searchlib/query/tree/querytreecreator.h> using search::attribute::IAttributeContext; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index ed58fcef977..bce13a96da7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -56,9 +56,10 @@ FeatureSet::SP findFeatureSet(const DocsumRequest &req, MatchToolsFactory &mtf, bool summaryFeatures) { std::vector<uint32_t> docs; - for (size_t i = 0; i < req.hits.size(); ++i) { - if (req.hits[i].docid != search::endDocId) { - docs.push_back(req.hits[i].docid); + docs.reserve(req.hits.size()); + for (const auto & hit : req.hits) { + if (hit.docid != search::endDocId) { + docs.push_back(hit.docid); } } std::sort(docs.begin(), docs.end()); @@ -102,7 +103,7 @@ Matcher::getFeatureSet(const DocsumRequest & req, ISearchContext & searchCtx, IA bool searchSessionCached = cache_props.lookup("query").found(); if (searchSessionCached) { SearchSession::SP session(sessionMgr.pickSearch(sessionId)); - if (session.get()) { + if (session) { MatchToolsFactory &mtf = session->getMatchToolsFactory(); FeatureSet::SP result = findFeatureSet(req, mtf, summaryFeatures); session->releaseEnumGuards(); @@ -117,7 +118,7 @@ Matcher::getFeatureSet(const DocsumRequest & req, ISearchContext & searchCtx, IA if (!mtf->valid()) { LOG(warning, "getFeatureSet(%s): query execution failed (invalid query). Returning empty feature set", (summaryFeatures ? "summary features" : "rank features")); - return FeatureSet::SP(new FeatureSet()); + return std::make_shared<FeatureSet>(); } return findFeatureSet(req, *mtf, summaryFeatures); } @@ -224,14 +225,14 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl shouldCacheSearchSession = cache_props.lookup("query").found(); if (shouldCacheGroupingSession) { GroupingSession::UP session(sessionMgr.pickGrouping(sessionId)); - if (session.get()) { + if (session) { return handleGroupingSession(sessionMgr, groupingContext, std::move(session)); } } } const Properties *feature_overrides = &request.propertiesMap.featureOverrides(); if (shouldCacheSearchSession) { - owned_objects.feature_overrides.reset(new Properties(*feature_overrides)); + owned_objects.feature_overrides = std::make_unique<Properties>(*feature_overrides); feature_overrides = owned_objects.feature_overrides.get(); } MatchToolsFactory::UP mtf = create_match_tools_factory(request, searchContext, attrContext, diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp index 0bc1807d714..4fb0e1d72e2 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "matching_stats.h" +#include <cmath> namespace proton::matching { @@ -13,6 +14,9 @@ MatchingStats::Partition &get_writable_partition(std::vector<MatchingStats::Part return state[id]; } +constexpr double MIN_TIMEOUT_SEC = 0.001; +constexpr double MAX_CHANGE_FACTOR = 5; + } // namespace proton::matching::<unnamed> MatchingStats::MatchingStats() @@ -62,6 +66,7 @@ MatchingStats::add(const MatchingStats &rhs) _docsReRanked += rhs._docsReRanked; _softDoomed += rhs.softDoomed(); + _queryCollateralTime.add(rhs._queryCollateralTime); _queryLatency.add(rhs._queryLatency); _matchTime.add(rhs._matchTime); @@ -75,10 +80,19 @@ MatchingStats::add(const MatchingStats &rhs) MatchingStats & MatchingStats::updatesoftDoomFactor(double hardLimit, double softLimit, double duration) { - if (duration < softLimit) { - _softDoomFactor += 0.01*(softLimit - duration)/hardLimit; - } else { - _softDoomFactor += 0.02*(softLimit - duration)/hardLimit; + // The safety capping here should normally not be necessary as all input numbers + // will normally be within reasonable values. + // It is merely a safety measure to avoid overflow on bad input as can happen with time senstive stuff + // in any soft real time system. + if ((hardLimit >= MIN_TIMEOUT_SEC) && (softLimit >= MIN_TIMEOUT_SEC)) { + double diff = (softLimit - duration)/hardLimit; + if (duration < softLimit) { + diff = std::min(diff, _softDoomFactor*MAX_CHANGE_FACTOR); + _softDoomFactor += 0.01*diff; + } else { + diff = std::max(diff, -_softDoomFactor*MAX_CHANGE_FACTOR); + _softDoomFactor += 0.02*diff; + } } return *this; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 38affde6075..e550ad8cad7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -28,6 +28,9 @@ using search::query::Node; using search::query::QueryTreeCreator; using search::query::Weight; using search::queryeval::AndBlueprint; +using search::queryeval::AndNotBlueprint; +using search::queryeval::RankBlueprint; +using search::queryeval::IntermediateBlueprint; using search::queryeval::Blueprint; using search::queryeval::IRequestContext; using search::queryeval::SearchIterator; @@ -73,6 +76,27 @@ void AddLocationNode(const string &location_str, Node::UP &query_tree, Location } query_tree = std::move(new_base); } + +IntermediateBlueprint * +asRankOrAndNot(Blueprint * blueprint) { + IntermediateBlueprint * rankOrAndNot = dynamic_cast<RankBlueprint*>(blueprint); + if (rankOrAndNot == nullptr) { + rankOrAndNot = dynamic_cast<AndNotBlueprint*>(blueprint); + } + return rankOrAndNot; +} + +IntermediateBlueprint * +lastConsequtiveRankOrAndNot(Blueprint * blueprint) { + IntermediateBlueprint * prev = nullptr; + IntermediateBlueprint * curr = asRankOrAndNot(blueprint); + while (curr != nullptr) { + prev = curr; + curr = asRankOrAndNot(&curr->getChild(0)); + } + return prev; +} + } // namespace Query::Query() = default; @@ -126,10 +150,18 @@ Query::reserveHandles(const IRequestContext & requestContext, ISearchContext &co LOG(debug, "original blueprint:\n%s\n", _blueprint->asString().c_str()); if (_whiteListBlueprint) { auto andBlueprint = std::make_unique<AndBlueprint>(); - (*andBlueprint) - .addChild(std::move(_blueprint)) - .addChild(std::move(_whiteListBlueprint)); - _blueprint = std::move(andBlueprint); + IntermediateBlueprint * rankOrAndNot = lastConsequtiveRankOrAndNot(_blueprint.get()); + if (rankOrAndNot != nullptr) { + (*andBlueprint) + .addChild(rankOrAndNot->removeChild(0)) + .addChild(std::move(_whiteListBlueprint)); + rankOrAndNot->insertChild(0, std::move(andBlueprint)); + } else { + (*andBlueprint) + .addChild(std::move(_blueprint)) + .addChild(std::move(_whiteListBlueprint)); + _blueprint = std::move(andBlueprint); + } _blueprint->setDocIdLimit(context.getDocIdLimit()); LOG(debug, "blueprint after white listing:\n%s\n", _blueprint->asString().c_str()); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index a3b512fc2b7..21365f75133 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -100,6 +100,8 @@ public: * @return estimate of hits produced. */ Blueprint::HitEstimate estimate() const; + + const Blueprint * peekRoot() const { return _blueprint.get(); } }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp index 325803d5aa6..dedda1504a5 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp @@ -24,7 +24,7 @@ ResultProcessor::Result::Result(std::unique_ptr<search::engine::SearchReply> rep _numFs4Hits(numFs4Hits) { } -ResultProcessor::Result::~Result() { } +ResultProcessor::Result::~Result() = default; ResultProcessor::Sort::Sort(uint32_t partitionId, const vespalib::Doom & doom, IAttributeContext &ac, const vespalib::string &ss) : sorter(FastS_DefaultResultSorter::instance()), @@ -43,7 +43,7 @@ ResultProcessor::Context::Context(Sort::UP s, PartialResult::UP r, GroupingConte groupingSource(grouping.get()) { } -ResultProcessor::Context::~Context() { } +ResultProcessor::Context::~Context() = default; void ResultProcessor::GroupingSource::merge(Source &s) { @@ -75,11 +75,11 @@ ResultProcessor::ResultProcessor(IAttributeContext &attrContext, _wasMerged(false) { if (!_groupingContext.empty()) { - _groupingSession.reset(new GroupingSession(sessionId, _groupingContext, attrContext)); + _groupingSession = std::make_unique<GroupingSession>(sessionId, _groupingContext, attrContext); } } -ResultProcessor::~ResultProcessor() { } +ResultProcessor::~ResultProcessor() = default; void ResultProcessor::prepareThreadContextCreation(size_t num_threads) @@ -95,19 +95,19 @@ ResultProcessor::prepareThreadContextCreation(size_t num_threads) ResultProcessor::Context::UP ResultProcessor::createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id, uint32_t distributionKey) { - Sort::UP sort(new Sort(distributionKey, hardDoom, _attrContext, _sortSpec)); - PartialResult::UP result(new PartialResult((_offset + _hits), sort->hasSortData())); + auto sort = std::make_unique<Sort>(distributionKey, hardDoom, _attrContext, _sortSpec); + auto result = std::make_unique<PartialResult>((_offset + _hits), sort->hasSortData()); search::grouping::GroupingContext::UP groupingContext; - if (_groupingSession.get() != 0) { + if (_groupingSession) { groupingContext = _groupingSession->createThreadContext(thread_id, _attrContext); } - return Context::UP(new Context(std::move(sort), std::move(result), std::move(groupingContext))); + return std::make_unique<Context>(std::move(sort), std::move(result), std::move(groupingContext)); } ResultProcessor::Result::UP ResultProcessor::makeReply(PartialResultUP full_result) { - search::engine::SearchReply::UP reply(new search::engine::SearchReply()); + auto reply = std::make_unique<search::engine::SearchReply>(); const search::IDocumentMetaStore &metaStore = _metaStore; search::engine::SearchReply &r = *reply; PartialResult &result = *full_result; @@ -158,7 +158,7 @@ ResultProcessor::makeReply(PartialResultUP full_result) assert(sortOffset == sortDataSize); } numFs4Hits += reply->hits.size(); - return Result::UP(new Result(std::move(reply), numFs4Hits)); + return std::make_unique<Result>(std::move(reply), numFs4Hits); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index bab3494ca5c..6475efdaabb 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -6,7 +6,9 @@ #include <vespa/metrics/loadmetric.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/base/exceptions.h> #include <vespa/log/log.h> @@ -25,6 +27,7 @@ using storage::spi::Result; using vespalib::IllegalStateException; using vespalib::Sequence; using vespalib::make_string; +using std::make_unique; using namespace std::chrono_literals; @@ -377,6 +380,18 @@ PersistenceEngine::update(const Bucket& b, Timestamp t, const DocumentUpdate::SP upd->getId().toString().c_str(), state.message().c_str())); } } + try { + upd->eagerDeserialize(); + } catch (document::FieldNotFoundException & e) { + return UpdateResult(Result::TRANSIENT_ERROR, + make_string("Update operation rejected for document '%s' of type '%s': 'Field not found'", + upd->getId().toString().c_str(), upd->getType().getName().c_str())); + } catch (document::DocumentTypeNotFoundException & e) { + return UpdateResult(Result::TRANSIENT_ERROR, + make_string("Update operation rejected for document '%s' of type '%s'.", + upd->getId().toString().c_str(), e.getDocumentTypeName().c_str())); + + } std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); DocTypeName docType(upd->getType()); LOG(spam, "update(%s, %" PRIu64 ", (\"%s\", \"%s\"), createIfNonExistent='%s')", diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index f2ab35e0ccf..a6c696d08fb 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -87,37 +87,36 @@ public: PersistenceEngine(IPersistenceEngineOwner &owner, const IResourceWriteFilter &writeFilter, ssize_t defaultSerializedSize, bool ignoreMaxBytes); - ~PersistenceEngine(); + ~PersistenceEngine() override; IPersistenceHandler::SP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, const IPersistenceHandler::SP &handler); IPersistenceHandler::SP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType); // Implements PersistenceProvider - virtual Result initialize() override; - virtual PartitionStateListResult getPartitionStates() const override; - virtual BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - virtual Result setClusterState(BucketSpace bucketSpace, const ClusterState& calc) override; - virtual Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; - virtual BucketInfoResult getBucketInfo(const Bucket&) const override; - virtual Result put(const Bucket&, Timestamp, const std::shared_ptr<document::Document>&, Context&) override; - virtual RemoveResult remove(const Bucket&, Timestamp, const document::DocumentId&, Context&) override; - virtual UpdateResult update(const Bucket&, Timestamp, - const std::shared_ptr<document::DocumentUpdate>&, Context&) override; - virtual GetResult get(const Bucket&, const document::FieldSet&, - const document::DocumentId&, Context&) const override; - virtual CreateIteratorResult createIterator(const Bucket&, const document::FieldSet&, const Selection&, - IncludedVersions, Context&) override; - virtual IterateResult iterate(IteratorId, uint64_t maxByteSize, Context&) const override; - virtual Result destroyIterator(IteratorId, Context&) override; - - virtual Result createBucket(const Bucket &bucketId, Context &) override ; - virtual Result deleteBucket(const Bucket&, Context&) override; - virtual BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; - virtual Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; - virtual Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; - - virtual Result maintain(const Bucket&, MaintenanceLevel) override; + Result initialize() override; + PartitionStateListResult getPartitionStates() const override; + BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; + Result setClusterState(BucketSpace bucketSpace, const ClusterState& calc) override; + Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; + BucketInfoResult getBucketInfo(const Bucket&) const override; + Result put(const Bucket&, Timestamp, const std::shared_ptr<document::Document>&, Context&) override; + RemoveResult remove(const Bucket&, Timestamp, const document::DocumentId&, Context&) override; + UpdateResult update(const Bucket&, Timestamp, + const std::shared_ptr<document::DocumentUpdate>&, Context&) override; + GetResult get(const Bucket&, const document::FieldSet&, const document::DocumentId&, Context&) const override; + CreateIteratorResult createIterator(const Bucket&, const document::FieldSet&, const Selection&, + IncludedVersions, Context&) override; + IterateResult iterate(IteratorId, uint64_t maxByteSize, Context&) const override; + Result destroyIterator(IteratorId, Context&) override; + + Result createBucket(const Bucket &bucketId, Context &) override ; + Result deleteBucket(const Bucket&, Context&) override; + BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; + Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; + Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; + + Result maintain(const Bucket&, MaintenanceLevel) override; void destroyIterators(); void propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler); diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp index da03adb3fe4..46096fead05 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp @@ -12,7 +12,7 @@ GidToLidChangeListener::GidToLidChangeListener(search::ISequencedTaskExecutor &a const vespalib::string &name, const vespalib::string &docTypeName) : _attributeFieldWriter(attributeFieldWriter), - _executorId(_attributeFieldWriter.getExecutorId(attr->getName())), + _executorId(_attributeFieldWriter.getExecutorId(attr->getNamePrefix())), _attr(std::move(attr)), _refCount(refCount), _name(name), diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp index 711f8ebdcaf..1df812b0c61 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp @@ -38,6 +38,9 @@ bool fastPartialUpdateAttribute(BasicType::Type attrType) { (attrType != BasicType::Type::REFERENCE)); } +bool isStructFieldAttribute(const vespalib::string &name) { + return name.find('.') != vespalib::string::npos; +} FilterAttributeManager::AttributeSet getAttributeSetToPopulate(const ARIConfig &newCfg, @@ -100,7 +103,8 @@ getFieldsToPopulate(const ARIConfig &newCfg, // keep the original in order to preserve annotations. bool wasStringIndexField = oldIndexschemaInspector.isStringIndex(name); bool populateField = !inNewAttrMgr && unchangedField && !wasStringIndexField && - fastPartialUpdateAttribute(attrType.type()); + fastPartialUpdateAttribute(attrType.type()) && + !isStructFieldAttribute(name); LOG(debug, "getFieldsToPopulate(): name='%s', inNewAttrMgr=%s, unchangedField=%s, " "wasStringIndexField=%s, dataType=%s, populate=%s", name.c_str(), toStr(inNewAttrMgr), toStr(unchangedField), diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index b371439dd0e..87aa19fb1b2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -13,6 +13,8 @@ vespa_add_library(searchcore_server STATIC disk_mem_usage_sampler.cpp disk_mem_usage_forwarder.cpp docstorevalidator.cpp + document_db_config_owner.cpp + document_db_directory_holder.cpp document_db_explorer.cpp document_db_flush_config.cpp document_db_maintenance_config.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index 987218beefa..20306e92ea8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -3,7 +3,7 @@ #include "combiningfeedview.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> -#include <vespa/searchcore/proton/feedoperation/moveoperation.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/log/log.h> @@ -13,22 +13,19 @@ using document::DocumentTypeRepo; using document::DocumentId; using search::IDestructorCallback; -namespace proton -{ +namespace proton { -namespace -{ +namespace { std::shared_ptr<const DocumentTypeRepo> getRepo(const std::vector<IFeedView::SP> &views) { for (const auto &view : views) { - if (view.get() == NULL) + if (view.get() == nullptr) continue; return view->getDocumentTypeRepo(); } - abort(); - return std::shared_ptr<const DocumentTypeRepo>(); + LOG_ABORT("should not be reached"); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h index 6644f21a3b7..3546fdbea71 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h @@ -3,21 +3,14 @@ #pragma once #include "ifeedview.h" -#include <vespa/searchcore/proton/common/feedtoken.h> -#include <vespa/searchcore/proton/feedoperation/deletebucketoperation.h> -#include <vespa/searchcore/proton/feedoperation/joinbucketsoperation.h> -#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> -#include <vespa/searchcore/proton/feedoperation/putoperation.h> -#include <vespa/searchcore/proton/feedoperation/removeoperation.h> -#include <vespa/searchcore/proton/feedoperation/splitbucketoperation.h> -#include <vespa/searchcore/proton/feedoperation/updateoperation.h> -#include <vespa/searchcore/proton/feedoperation/createbucketoperation.h> -#include <vespa/searchlib/common/serialnum.h> #include "replaypacketdispatcher.h" #include "ibucketstatecalculator.h" +#include <vespa/searchcore/proton/common/feedtoken.h> +#include <vespa/searchlib/common/serialnum.h> namespace proton { +class DocumentOperation; class CombiningFeedView : public IFeedView { diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp new file mode 100644 index 00000000000..28492c5d39a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_db_config_owner.h" +#include "document_db_directory_holder.h" + +namespace proton { + +DocumentDBConfigOwner::DocumentDBConfigOwner() + : _holder(std::make_shared<DocumentDBDirectoryHolder>()) +{ +} + +DocumentDBConfigOwner::~DocumentDBConfigOwner() = default; + +std::shared_ptr<DocumentDBDirectoryHolder> +DocumentDBConfigOwner::getDocumentDBDirectoryHolder() +{ + return _holder; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h new file mode 100644 index 00000000000..cc19bc14fab --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_document_db_config_owner.h" + +namespace proton { + +class DocumentDBDirectoryHolder; + +/* + * Abstract class meant to be a base class for DocumentDB where a + * directory holder exists until the document db instance is + * destroyed. + */ +class DocumentDBConfigOwner : public IDocumentDBConfigOwner +{ + std::shared_ptr<DocumentDBDirectoryHolder> _holder; +public: + DocumentDBConfigOwner(); + virtual ~DocumentDBConfigOwner(); + std::shared_ptr<DocumentDBDirectoryHolder> getDocumentDBDirectoryHolder(); +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp new file mode 100644 index 00000000000..d0807a3b8a7 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_db_directory_holder.h" +#include <mutex> +#include <condition_variable> + +namespace proton { + +namespace { + +std::mutex mutex; +std::condition_variable cv; + +} + +DocumentDBDirectoryHolder::DocumentDBDirectoryHolder() +{ +} + +DocumentDBDirectoryHolder::~DocumentDBDirectoryHolder() +{ + std::lock_guard guard(mutex); + cv.notify_all(); +} + +void +DocumentDBDirectoryHolder::waitUntilDestroyed(const std::weak_ptr<DocumentDBDirectoryHolder> &holder) +{ + std::unique_lock guard(mutex); + cv.wait(guard, [&]() { return !holder.lock(); }); +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h new file mode 100644 index 00000000000..65df1b7bd8e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h @@ -0,0 +1,20 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_document_db_config_owner.h" + +namespace proton { + +/* + * class holding onto a document db directory. + */ +class DocumentDBDirectoryHolder +{ +public: + DocumentDBDirectoryHolder(); + ~DocumentDBDirectoryHolder(); + static void waitUntilDestroyed(const std::weak_ptr<DocumentDBDirectoryHolder> &holder); +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 61ce0e6dc6f..24d40e15677 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -17,6 +17,7 @@ #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcore/proton/common/statusreport.h> +#include <vespa/searchcore/proton/feedoperation/noopoperation.h> #include <vespa/searchcore/proton/index/index_writer.h> #include <vespa/searchcore/proton/initializer/task_runner.h> #include <vespa/searchcore/proton/metrics/attribute_metrics_collection.h> @@ -89,7 +90,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, ConfigStore::UP config_store, InitializeThreads initializeThreads, const HwInfo &hwInfo) - : IDocumentDBConfigOwner(), + : DocumentDBConfigOwner(), IReplayConfig(), IFeedHandlerOwner(), IDocumentSubDBOwner(), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index e25f1fe66fc..1a64a4013de 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -10,7 +10,7 @@ #include "documentsubdbcollection.h" #include "executorthreadingservice.h" #include "feedhandler.h" -#include "i_document_db_config_owner.h" +#include "document_db_config_owner.h" #include "i_document_subdb_owner.h" #include "i_feed_handler_owner.h" #include "i_lid_space_compaction_handler.h" @@ -53,7 +53,7 @@ namespace matching { class SessionManager; } * to ensure that there are never multiple writers. Unless explicitly stated, * none of the methods of this class are thread-safe. */ -class DocumentDB : public IDocumentDBConfigOwner, +class DocumentDB : public DocumentDBConfigOwner, public IReplayConfig, public IFeedHandlerOwner, public IDocumentSubDBOwner, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index d5a9b09ead7..fd1f9f1155d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -110,7 +110,7 @@ DocumentDBConfig(const DocumentDBConfig &cfg) _delayedAttributeAspects(false) { } -DocumentDBConfig::~DocumentDBConfig() { } +DocumentDBConfig::~DocumentDBConfig() = default; bool DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index ec22d3293c4..f2230215c3d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -157,13 +157,26 @@ deriveCompression(const T & config) { return compression; } +DocumentStore::Config::UpdateStrategy +derive(ProtonConfig::Summary::Cache::UpdateStrategy strategy) { + switch (strategy) { + case ProtonConfig::Summary::Cache::UpdateStrategy::INVALIDATE: + return DocumentStore::Config::UpdateStrategy::INVALIDATE; + case ProtonConfig::Summary::Cache::UpdateStrategy::UPDATE: + return DocumentStore::Config::UpdateStrategy::UPDATE; + } + return DocumentStore::Config::UpdateStrategy::INVALIDATE; +} + DocumentStore::Config getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo) { size_t maxBytes = (cache.maxbytes < 0) ? (hwInfo.memory().sizeBytes()*std::min(50l, -cache.maxbytes))/100l : cache.maxbytes; - return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); + return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries) + .allowVisitCaching(cache.allowvisitcaching) + .updateStrategy(derive(cache.updateStrategy)); } LogDocumentStore::Config @@ -291,8 +304,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) search::index::Schema schema; search::index::SchemaBuilder::build(*newIndexschemaConfig, schema); if (!search::index::SchemaUtil::validateSchema(schema)) { - LOG(error, "Cannot use bad index schema, validation failed"); - abort(); + LOG_ABORT("Cannot use bad index schema, validation failed"); } } if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration)) { diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index 6364e772f94..1c2406b2acf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -5,6 +5,7 @@ #include "operationdonecontext.h" #include "removedonecontext.h" #include "putdonecontext.h" +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> using document::Document; @@ -71,7 +72,7 @@ FastAccessFeedView::FastAccessFeedView(const StoreOnlyFeedView::Context &storeOn _docIdLimit(ctx._docIdLimit) {} -FastAccessFeedView::~FastAccessFeedView() {} +FastAccessFeedView::~FastAccessFeedView() = default; void FastAccessFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 66e721f9222..97cc25e635f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -14,6 +14,7 @@ #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/searchcore/proton/persistenceengine/transport_latch.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/common/gatecallback.h> @@ -522,7 +523,7 @@ FeedHandler::considerUpdateOperationForRejection(FeedToken &token, UpdateOperati */ if (_documentType != &update.getType()) { try { - op.deserializeUpdate(*_repo); + op.verifyUpdate(*_repo); } catch (document::FieldNotFoundException &e) { if (token) { auto message = make_string("Update operation rejected for document '%s' of type '%s': 'Field not found'", diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.h b/searchcore/src/vespa/searchcore/proton/server/feedstate.h index 472f5cb224f..fa0a1702499 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstate.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.h @@ -25,6 +25,7 @@ private: Type _type; protected: + using FeedOperationUP = std::unique_ptr<FeedOperation>; void throwExceptionInReceive(const vespalib::string &docType, uint64_t serialRangeFrom, uint64_t serialRangeTo, size_t packetSize); void throwExceptionInHandleOperation(const vespalib::string &docType, const FeedOperation &op); @@ -38,7 +39,7 @@ public: Type getType() const { return _type; } vespalib::string getName() const; - virtual void handleOperation(FeedToken token, std::unique_ptr<FeedOperation> op) = 0; + virtual void handleOperation(FeedToken token, FeedOperationUP op) = 0; virtual void receive(const PacketWrapper::SP &wrap, vespalib::Executor &executor) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index f0866347f59..ae323bc93df 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -6,6 +6,7 @@ #include "ireplayconfig.h" #include "replaypacketdispatcher.h" #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/vespalib/util/closuretask.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.h b/searchcore/src/vespa/searchcore/proton/server/feedstates.h index 963a78d0d6b..a9224669d87 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.h @@ -28,16 +28,13 @@ public: { } - virtual void handleOperation(FeedToken, FeedOperation::UP op) override { + void handleOperation(FeedToken, FeedOperationUP op) override { throwExceptionInHandleOperation(_doc_type_name, *op); } - virtual void - receive(const PacketWrapper::SP &wrap, vespalib::Executor &) override { - throwExceptionInReceive(_doc_type_name.c_str(), - wrap->packet.range().from(), - wrap->packet.range().to(), - wrap->packet.size()); + void receive(const PacketWrapper::SP &wrap, vespalib::Executor &) override { + throwExceptionInReceive(_doc_type_name.c_str(), wrap->packet.range().from(), + wrap->packet.range().to(), wrap->packet.size()); } }; @@ -57,12 +54,11 @@ public: IReplayConfig &replay_config, FeedConfigStore &config_store); - virtual void handleOperation(FeedToken, FeedOperation::UP op) override { + void handleOperation(FeedToken, FeedOperationUP op) override { throwExceptionInHandleOperation(_doc_type_name, *op); } - virtual void receive(const PacketWrapper::SP &wrap, - vespalib::Executor &executor) override; + void receive(const PacketWrapper::SP &wrap, vespalib::Executor &executor) override; }; @@ -79,15 +75,13 @@ public: _handler(handler) { } - void handleOperation(FeedToken token, FeedOperation::UP op) override { + void handleOperation(FeedToken token, FeedOperationUP op) override { _handler.performOperation(std::move(token), std::move(op)); } void receive(const PacketWrapper::SP &wrap, vespalib::Executor &) override { - throwExceptionInReceive(_handler.getDocTypeName().c_str(), - wrap->packet.range().from(), - wrap->packet.range().to(), - wrap->packet.size()); + throwExceptionInReceive(_handler.getDocTypeName().c_str(), wrap->packet.range().from(), + wrap->packet.range().to(), wrap->packet.size()); } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h index 242e99fffbf..fec8430e41d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h @@ -10,7 +10,7 @@ namespace vespalib { class ThreadStackExecutorBase; } namespace proton { -class IDocumentDBConfigOwner; +class DocumentDBConfigOwner; /* * Interface class for owner of a proton configurer, with callback methods @@ -21,12 +21,12 @@ class IProtonConfigurerOwner using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; public: virtual ~IProtonConfigurerOwner() { } - virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName &docTypeName, - document::BucketSpace bucketSpace, - const vespalib::string &configId, - const std::shared_ptr<BootstrapConfig> &bootstrapConfig, - const std::shared_ptr<DocumentDBConfig> &documentDBConfig, - InitializeThreads initializeThreads) = 0; + virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, + document::BucketSpace bucketSpace, + const vespalib::string &configId, + const std::shared_ptr<BootstrapConfig> &bootstrapConfig, + const std::shared_ptr<DocumentDBConfig> &documentDBConfig, + InitializeThreads initializeThreads) = 0; virtual void removeDocumentDB(const DocTypeName &docTypeName) = 0; virtual void applyConfig(const std::shared_ptr<BootstrapConfig> &bootstrapConfig) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h b/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h index 210ece3a36a..e93821e2a36 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/ireplaypackethandler.h @@ -2,10 +2,28 @@ #pragma once -#include <vespa/searchcore/proton/feedoperation/operations.h> +namespace document { class DocumentTypeRepo; } namespace proton { +class PutOperation; +class RemoveOperation; +class UpdateOperation; +class NoopOperation; +class NewConfigOperation; +class WipeHistoryOperation; +class DeleteBucketOperation; +class SplitBucketOperation; +class JoinBucketsOperation; +class PruneRemovedDocumentsOperation; +class SpoolerReplayStartOperation; +class SpoolerReplayCompleteOperation; +class MoveOperation; +class CreateBucketOperation; +class CompactLidSpaceOperation; + +namespace feedoperation { class IStreamHandler; } + /** * Interface used to handle the various feed operations during * replay of the transaction log. @@ -29,7 +47,7 @@ struct IReplayPacketHandler virtual void replay(const CreateBucketOperation &op) = 0; virtual void replay(const CompactLidSpaceOperation &op) = 0; - virtual NewConfigOperation::IStreamHandler &getNewConfigStreamHandler() = 0; + virtual feedoperation::IStreamHandler &getNewConfigStreamHandler() = 0; virtual const document::DocumentTypeRepo &getDeserializeRepo() = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp index 3162f9a1c45..9af648917c4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp @@ -44,13 +44,13 @@ MatchView::MatchView(const Matchers::SP &matchers, _docIdLimit(docIdLimit) { } -MatchView::~MatchView() { } +MatchView::~MatchView() = default; Matcher::SP MatchView::getMatcher(const vespalib::string & rankProfile) const { Matcher::SP retval = _matchers->lookup(rankProfile); - if (retval.get() == NULL) { + if ( ! retval) { throw std::runtime_error(make_string("Failed locating Matcher for rank profile '%s'", rankProfile.c_str())); } LOG(debug, "Rankprofile = %s has termwise_limit=%f", rankProfile.c_str(), retval->get_termwise_limit()); @@ -60,8 +60,8 @@ MatchView::getMatcher(const vespalib::string & rankProfile) const MatchContext::UP MatchView::createContext() const { IAttributeContext::UP attrCtx = _attrMgr->createContext(); - ISearchContext::UP searchCtx(new SearchContext(_indexSearchable, _docIdLimit.get())); - return MatchContext::UP(new MatchContext(std::move(attrCtx), std::move(searchCtx))); + auto searchCtx = std::make_unique<SearchContext>(_indexSearchable, _docIdLimit.get()); + return std::make_unique<MatchContext>(std::move(attrCtx), std::move(searchCtx)); } diff --git a/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp index aef49c51f7a..1c930ef1ddc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp @@ -50,8 +50,7 @@ PrepareRestartHandler::performPrepareRestart(const ProtonConfig &protonCfg, std: { _running = true; lock.unlock(); - auto strategy = std::make_shared<PrepareRestartFlushStrategy>(createPrepareRestartConfig(protonCfg)); - _flushEngine.setStrategy(strategy); + _flushEngine.setStrategy(std::make_shared<PrepareRestartFlushStrategy>(createPrepareRestartConfig(protonCfg))); lock.lock(); _running = false; _cond.notify_all(); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 32dc711d5cf..9a7d5a5141e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -240,9 +240,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) const HwInfo & hwInfo = configSnapshot->getHwInfo(); setFS4Compression(protonConfig); - _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler> - (protonConfig.basedir, - diskMemUsageSamplerConfig(protonConfig, hwInfo)); + _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>(protonConfig.basedir, + diskMemUsageSamplerConfig(protonConfig, hwInfo)); _tls = std::make_unique<TLS>(_configUri.createWithNewId(protonConfig.tlsconfigid), _fileHeaderContext); _metricsEngine->addMetricsHook(_metricsHook); @@ -253,6 +252,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _distributionKey = protonConfig.distributionkey; _summaryEngine= std::make_unique<SummaryEngine>(protonConfig.numsummarythreads); _docsumBySlime = std::make_unique<DocsumBySlime>(*_summaryEngine); + IFlushStrategy::SP strategy; const ProtonConfig::Flush & flush(protonConfig.flush); switch (flush.strategy) { @@ -283,17 +283,15 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) LOG(debug, "Start proton server with root at %s and cwd at %s", protonConfig.basedir.c_str(), getcwd(tmp, sizeof(tmp))); - _persistenceEngine.reset(new PersistenceEngine(*this, - _diskMemUsageSampler->writeFilter(), - protonConfig.visit.defaultserializedsize, - protonConfig.visit.ignoremaxbytes)); - + _persistenceEngine = std::make_unique<PersistenceEngine>(*this, _diskMemUsageSampler->writeFilter(), + protonConfig.visit.defaultserializedsize, + protonConfig.visit.ignoremaxbytes); vespalib::string fileConfigId; - _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024)); + _warmupExecutor = std::make_unique<vespalib::ThreadStackExecutor>(4, 128*1024); const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); - _summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16)); + _summaryExecutor = std::make_unique<vespalib::BlockingThreadStackExecutor>(summaryThreads, 128*1024, summaryThreads*16); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128 * 1024); @@ -305,7 +303,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _prepareRestartHandler = std::make_unique<PrepareRestartHandler>(*_flushEngine); RPCHooks::Params rpcParams(*this, protonConfig.rpcport, _configUri.getConfigId()); rpcParams.slobrok_config = _configUri.createWithNewId(protonConfig.slobrokconfigid); - _rpcHooks.reset(new RPCHooks(rpcParams)); + _rpcHooks = std::make_unique<RPCHooks>(rpcParams); waitForInitDone(); @@ -354,7 +352,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) } } -IDocumentDBConfigOwner * +std::shared_ptr<DocumentDBConfigOwner> Proton::addDocumentDB(const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const vespalib::string &configId, @@ -368,21 +366,21 @@ Proton::addDocumentDB(const DocTypeName &docTypeName, if (docType != NULL) { LOG(info, "Add document database: doctypename(%s), configid(%s)", docTypeName.toString().c_str(), configId.c_str()); - return addDocumentDB(*docType, bucketSpace, bootstrapConfig, documentDBConfig, initializeThreads).get(); + return addDocumentDB(*docType, bucketSpace, bootstrapConfig, documentDBConfig, initializeThreads); } else { LOG(warning, "Did not find document type '%s' in the document manager. " "Skipping creating document database for this type", docTypeName.toString().c_str()); - return nullptr; + return std::shared_ptr<DocumentDBConfigOwner>(); } } catch (const document::DocumentTypeNotFoundException & e) { LOG(warning, "Did not find document type '%s' in the document manager. " "Skipping creating document database for this type", docTypeName.toString().c_str()); - return nullptr; + return std::shared_ptr<DocumentDBConfigOwner>(); } } @@ -528,23 +526,11 @@ Proton::addDocumentDB(const document::DocumentType &docType, // 1 thread per document type. initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(1, 128 * 1024); } - DocumentDB::SP ret(new DocumentDB(config.basedir + "/documents", - documentDBConfig, - config.tlsspec, - _queryLimiter, - _clock, - docTypeName, - bucketSpace, - config, - *this, - *_warmupExecutor, - *_summaryExecutor, - *_tls->getTransLogServer(), - *_metricsEngine, - _fileHeaderContext, - std::move(config_store), - initializeThreads, - bootstrapConfig->getHwInfo())); + auto ret = std::make_shared<DocumentDB>(config.basedir + "/documents", documentDBConfig, config.tlsspec, + _queryLimiter, _clock, docTypeName, bucketSpace, config, *this, + *_warmupExecutor, *_summaryExecutor, *_tls->getTransLogServer(), + *_metricsEngine, _fileHeaderContext, std::move(config_store), + initializeThreads, bootstrapConfig->getHwInfo()); try { ret->start(); } catch (vespalib::Exception &e) { @@ -571,10 +557,10 @@ Proton::addDocumentDB(const document::DocumentType &docType, // TODO: Fix race with new cluster state setting. _persistenceEngine->putHandler(bucketSpace, docTypeName, persistenceHandler); } - SearchHandlerProxy::SP searchHandler(new SearchHandlerProxy(ret)); + auto searchHandler = std::make_shared<SearchHandlerProxy>(ret); _summaryEngine->putSearchHandler(docTypeName, searchHandler); _matchEngine->putSearchHandler(docTypeName, searchHandler); - FlushHandlerProxy::SP flushHandler(new FlushHandlerProxy(ret)); + auto flushHandler = std::make_shared<FlushHandlerProxy>(ret); _flushEngine->putFlushHandler(docTypeName, flushHandler); _diskMemUsageSampler->notifier().addDiskMemUsageListener(ret->diskMemUsageListener()); return ret; @@ -624,7 +610,7 @@ Proton::MonitorReply::UP Proton::ping(MonitorRequest::UP request, MonitorClient & client) { (void) client; - MonitorReply::UP reply(new MonitorReply()); + auto reply = std::make_unique<MonitorReply>(); MonitorReply &ret = *reply; BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot(); @@ -807,8 +793,8 @@ struct DocumentDBMapExplorer : vespalib::StateExplorer { std::shared_timed_mutex &mutex; DocumentDBMapExplorer(const DocumentDBMap &documentDBMap_in, std::shared_timed_mutex &mutex_in) : documentDBMap(documentDBMap_in), mutex(mutex_in) {} - virtual void get_state(const vespalib::slime::Inserter &, bool) const override {} - virtual std::vector<vespalib::string> get_children_names() const override { + void get_state(const vespalib::slime::Inserter &, bool) const override {} + std::vector<vespalib::string> get_children_names() const override { std::shared_lock<std::shared_timed_mutex> guard(mutex); std::vector<vespalib::string> names; for (const auto &item: documentDBMap) { @@ -816,14 +802,14 @@ struct DocumentDBMapExplorer : vespalib::StateExplorer { } return names; } - virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override { + std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override { typedef std::unique_ptr<StateExplorer> Explorer_UP; std::shared_lock<std::shared_timed_mutex> guard(mutex); auto result = documentDBMap.find(DocTypeName(vespalib::string(name))); if (result == documentDBMap.end()) { return Explorer_UP(nullptr); } - return Explorer_UP(new DocumentDBExplorer(result->second)); + return std::make_unique<DocumentDBExplorer>(result->second); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index dd2b42d94cf..a3c596d7b0b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -130,7 +130,7 @@ private: std::mutex _nodeUpLock; std::set<BucketSpace> _nodeUp; // bucketspaces where node is up - IDocumentDBConfigOwner * + std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName & docTypeName, BucketSpace bucketSpace, const vespalib::string & configid, const BootstrapConfig::SP & bootstrapConfig, const std::shared_ptr<DocumentDBConfig> &documentDBConfig, InitializeThreads initializeThreads) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp index 07ed83fd8c6..f871f36b042 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp @@ -4,7 +4,8 @@ #include "proton_config_snapshot.h" #include "bootstrapconfig.h" #include "i_proton_configurer_owner.h" -#include "i_document_db_config_owner.h" +#include "document_db_config_owner.h" +#include "document_db_directory_holder.h" #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutorbase.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> @@ -162,13 +163,15 @@ ProtonConfigurer::configureDocumentDB(const ProtonConfigSnapshot &configSnapshot const auto &documentDBConfig = cfgitr->second; auto dbitr(_documentDBs.find(docTypeName)); if (dbitr == _documentDBs.end()) { - auto *newdb = _owner.addDocumentDB(docTypeName, bucketSpace, configId, bootstrapConfig, documentDBConfig, initializeThreads); - if (newdb != nullptr) { - auto insres = _documentDBs.insert(std::make_pair(docTypeName, newdb)); + auto newdb = _owner.addDocumentDB(docTypeName, bucketSpace, configId, bootstrapConfig, documentDBConfig, initializeThreads); + if (newdb) { + auto insres = _documentDBs.insert(std::make_pair(docTypeName, std::make_pair(newdb, newdb->getDocumentDBDirectoryHolder()))); assert(insres.second); } } else { - dbitr->second->reconfigure(documentDBConfig); + auto documentDB = dbitr->second.first.lock(); + assert(documentDB); + documentDB->reconfigure(documentDBConfig); } } @@ -189,6 +192,7 @@ ProtonConfigurer::pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot) auto found(newDocTypes.find(dbitr->first)); if (found == newDocTypes.end()) { _owner.removeDocumentDB(dbitr->first); + DocumentDBDirectoryHolder::waitUntilDestroyed(dbitr->second.second); dbitr = _documentDBs.erase(dbitr); } else { ++dbitr; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h index 149be3a9e62..ac0f6197fd7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h @@ -12,6 +12,7 @@ namespace proton { +class DocumentDBDirectoryHolder; class IDocumentDBConfigOwner; class IProtonConfigurerOwner; class BootstrapConfig; @@ -22,7 +23,7 @@ class BootstrapConfig; */ class ProtonConfigurer : public IProtonConfigurer { - using DocumentDBs = std::map<DocTypeName, IDocumentDBConfigOwner *>; + using DocumentDBs = std::map<DocTypeName, std::pair<std::weak_ptr<IDocumentDBConfigOwner>, std::weak_ptr<DocumentDBDirectoryHolder>>>; using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; ExecutorThreadService _executor; diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp index 6b3ee548ad5..42451f08315 100644 --- a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "replaypacketdispatcher.h" +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/util/serializableexceptions.h> @@ -106,9 +107,7 @@ ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry) } -ReplayPacketDispatcher::~ReplayPacketDispatcher() -{ -} +ReplayPacketDispatcher::~ReplayPacketDispatcher() = default; void diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h index fc6f99471d8..771a79ddad9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.h @@ -7,6 +7,7 @@ namespace proton { +class FeedOperation; /** * Utility class that deserializes packet entries into feed operations * during replay from the transaction log and dispatches the feed operations @@ -22,14 +23,11 @@ private: void replay(OperationType &op, vespalib::nbostream &is, const Packet::Entry &entry); protected: - virtual void - store(const FeedOperation &op); + virtual void store(const FeedOperation &op); public: ReplayPacketDispatcher(IReplayPacketHandler &handler); - - virtual - ~ReplayPacketDispatcher(); + virtual ~ReplayPacketDispatcher(); void replayEntry(const Packet::Entry &entry); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 29615b0daf9..22ff1b90de4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -13,6 +13,7 @@ #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> +#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/document/datatype/documenttype.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 825e14b4368..37f67399f4d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -13,7 +13,7 @@ #include <vespa/searchcore/proton/common/feeddebugger.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> -#include <vespa/searchcore/proton/feedoperation/feedoperation.h> +#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> #include <vespa/searchcore/proton/persistenceengine/resulthandler.h> #include <vespa/searchcore/proton/reference/pending_notify_remove_done.h> #include <vespa/searchcorespi/index/ithreadingservice.h> @@ -23,7 +23,6 @@ namespace search { class IDestructorCallback; } -namespace document { class GLobalId; } namespace proton { @@ -35,6 +34,8 @@ class RemoveDoneContext; class CommitTimeTracker; class IGidToLidChangeHandler; class IFieldUpdateCallback; +class RemoveDocumentsOperation; +class DocumentOperation; namespace documentmetastore { class ILidReuseDelayer; } diff --git a/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h b/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h index 6229e989cba..f1d661a3ad0 100644 --- a/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h +++ b/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/util/hdr_abort.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/integerbase.h> @@ -24,7 +25,7 @@ struct AttributeUtils search::IntegerAttribute &ia = static_cast<search::IntegerAttribute &>(*attr); while (ia.getNumDocs() < to) { uint32_t docId; - if (!ia.addDoc(docId)) { abort(); } + if (!ia.addDoc(docId)) { HDR_ABORT("should not be reached"); } } for (uint32_t i = from; i < to; ++i) { ia.update(i, value); diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 9dad0d32b61..b0b999e0d30 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/util/hdr_abort.h> #include <vespa/searchlib/test/mock_attribute_manager.h> #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> @@ -54,16 +55,16 @@ public: virtual void pruneRemovedFields(search::SerialNum) override { } virtual const IAttributeFactory::SP &getFactory() const override { - abort(); + HDR_ABORT("should not be reached"); } virtual search::ISequencedTaskExecutor &getAttributeFieldWriter() const override { - abort(); + HDR_ABORT("should not be reached"); } virtual search::AttributeVector *getWritableAttribute(const vespalib::string &) const override { return nullptr; } virtual const std::vector<search::AttributeVector *> &getWritableAttributes() const override { - abort(); + HDR_ABORT("should not be reached"); } virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>) const override { } |