From 4ad220bb0e9d62562ae631ad5d25fa783f608a6a Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 18 Feb 2024 09:07:03 +0000 Subject: Modernize key/value interface of fs4::FS4Properties --- .../go/internal/admin/jvm/application_container.go | 1 + searchlib/src/vespa/searchlib/common/packets.cpp | 23 ++++++++++++++++------ searchlib/src/vespa/searchlib/common/packets.h | 19 +++++++++--------- .../src/vespa/searchvisitor/searchvisitor.cpp | 22 ++++++++++----------- 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/client/go/internal/admin/jvm/application_container.go b/client/go/internal/admin/jvm/application_container.go index 3806dfea302..4db70d364f4 100644 --- a/client/go/internal/admin/jvm/application_container.go +++ b/client/go/internal/admin/jvm/application_container.go @@ -118,6 +118,7 @@ func (a *ApplicationContainer) configureMemory(qc *QrStartConfig) { opts.AddOption(fmt.Sprintf("-Xmx%dm", jvm_heapsize)) opts.AddOption(fmt.Sprintf("-XX:ThreadStackSize=%d", jvm_stacksize)) opts.AddOption(fmt.Sprintf("-XX:MaxDirectMemorySize=%dm", maxDirectMemorySize)) + opts.AddOption(fmt.Sprintf("-XX:Tier4CompileThreshold=1000000")) opts.MaybeAddHugepages(MegaBytesOfMemory(jvm_heapsize)) if jvm_compressedClassSpaceSize > 0 { opts.AddOption(fmt.Sprintf("-XX:CompressedClassSpaceSize=%dm", jvm_compressedClassSpaceSize)) diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 3f2b5750631..af5dc5fc110 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -35,14 +35,14 @@ FS4Properties::FS4Properties() _backing() { } -FS4Properties::FS4Properties(FS4Properties && rhs) +FS4Properties::FS4Properties(FS4Properties && rhs) noexcept : _entries(std::move(rhs._entries)), _name(std::move(rhs._name)), _backing(std::move(rhs._backing)) { } FS4Properties & -FS4Properties::operator=(FS4Properties && rhs) +FS4Properties::operator=(FS4Properties && rhs) noexcept { _entries = std::move(rhs._entries); _name = std::move(rhs._name); @@ -72,14 +72,25 @@ FS4Properties::setValue(uint32_t entry, const char *value, uint32_t valueSize) } uint32_t -FS4Properties::getLength() +FS4Properties::getLength() const noexcept { - uint32_t len = sizeof(uint32_t) * 2 + getNameLen(); + uint32_t len = sizeof(uint32_t) * 2 + name().size(); len += _backing.size(); len += _entries.size() * sizeof(uint32_t) * 2; return len; } +vespalib::stringref +FS4Properties::key(uint32_t entry) const noexcept { + auto pair = _entries[entry].first; + return {c_str(pair.first), pair.second}; +} +vespalib::stringref +FS4Properties::value(uint32_t entry) const noexcept { + auto pair = _entries[entry].second; + return {c_str(pair.first), pair.second}; +} + vespalib::string FS4Properties::toString(uint32_t indent) const { @@ -90,8 +101,8 @@ FS4Properties::toString(uint32_t indent) const s += "\n"; for (uint32_t i = 0; i < size(); ++i) { s += make_string("%*s Entry[%d] {\n", indent, "", i); - s += make_string("%*s key : %s\n", indent, "", vespalib::string(getKey(i), getKeyLen(i)).c_str()); - s += make_string("%*s value: %s\n", indent, "", vespalib::string(getValue(i), getValueLen(i)).c_str()); + s += make_string("%*s key : %s\n", indent, "", string(key(i)).c_str()); + s += make_string("%*s value: %s\n", indent, "", string(value(i)).c_str()); s += make_string("%*s }\n", indent, ""); } s += make_string("%*s}\n", indent, ""); diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index b04e1fb44d0..fbdae8b51ec 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -50,8 +50,10 @@ private: void set(StringRef & e, vespalib::stringref s); void allocEntries(uint32_t cnt); public: - FS4Properties(FS4Properties &&); - FS4Properties &operator=(FS4Properties &&); + FS4Properties(FS4Properties &&) noexcept; + FS4Properties &operator=(FS4Properties &&) noexcept; + FS4Properties(const FS4Properties &) = delete; + FS4Properties &operator=(const FS4Properties &) = delete; FS4Properties(); ~FS4Properties(); @@ -67,16 +69,13 @@ public: void setValue(uint32_t entry, vespalib::stringref val) { setValue(entry, val.data(), val.size()); } - uint32_t size() const { return _entries.size(); } - const char *getName() const { return _name.c_str(); } - uint32_t getNameLen() const { return _name.size(); } - const char *getKey(uint32_t entry) const { return c_str(_entries[entry].first.first); } - uint32_t getKeyLen(uint32_t entry) const { return _entries[entry].first.second; } - const char *getValue(uint32_t entry) const { return c_str(_entries[entry].second.first); } - uint32_t getValueLen(uint32_t entry) const { return _entries[entry].second.second; } + uint32_t size() const noexcept { return _entries.size(); } + const vespalib::string & name() const noexcept { return _name; } + vespalib::stringref key(uint32_t entry) const noexcept; + vespalib::stringref value(uint32_t entry) const noexcept; // sub-packet methods below - uint32_t getLength(); + uint32_t getLength() const noexcept; void encode(FNET_DataBuffer &dst); bool decode(FNET_DataBuffer &src, uint32_t &len); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 1875664f6c4..68c5f5cfdc8 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -51,6 +51,7 @@ using vsm::DocsumFilter; using vsm::FieldPath; using vsm::StorageDocument; using vsm::StringFieldIdTMap; +using vespalib::string; namespace { @@ -429,21 +430,18 @@ SearchVisitor::init(const Parameters & params) if (!prop.decode(src, len)) { LOG(warning, "Could not decode rank properties"); } else { - LOG(debug, "Properties[%u]: name '%s', size '%u'", i, prop.getName(), prop.size()); - if (strcmp(prop.getName(), "rank") == 0) { // pick up rank properties + LOG(debug, "Properties[%u]: name '%s', size '%u'", i, prop.name().c_str(), prop.size()); + if (prop.name() == "rank") { // pick up rank properties for (uint32_t j = 0; j < prop.size(); ++j) { - vespalib::string k{prop.getKey(j), prop.getKeyLen(j)}; - vespalib::string v{prop.getValue(j), prop.getValueLen(j)}; - LOG(debug, "Properties[%u][%u]: key '%s' -> value '%s'", i, j, k.c_str(), v.c_str()); - _rankController.getQueryProperties().add(k, v); + LOG(debug, "Properties[%u][%u]: key '%s' -> value '%s'", + i, j, string(prop.key(j)).c_str(), string(prop.value(j)).c_str()); + _rankController.getQueryProperties().add(prop.key(j), prop.value(j)); } - } - if (strcmp(prop.getName(), "feature") == 0) { // pick up feature overrides + } else if (prop.name() == "feature") { // pick up feature overrides for (uint32_t j = 0; j < prop.size(); ++j) { - vespalib::string k{prop.getKey(j), prop.getKeyLen(j)}; - vespalib::string v{prop.getValue(j), prop.getValueLen(j)}; - LOG(debug, "Feature override[%u][%u]: key '%s' -> value '%s'", i, j, k.c_str(), v.c_str()); - _rankController.getFeatureOverrides().add(k, v); + LOG(debug, "Feature override[%u][%u]: key '%s' -> value '%s'", + i, j, string(prop.key(j)).c_str(), string(prop.value(j)).c_str()); + _rankController.getFeatureOverrides().add(prop.key(j), prop.value(j)); } } } -- cgit v1.2.3 From 9486db5d9672be5a5e4316af368e2b604f7b068d Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 18 Feb 2024 09:36:40 +0000 Subject: Propagate highlighterms. --- searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h | 1 + streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp | 8 ++++++++ streamingvisitors/src/vespa/searchvisitor/searchvisitor.h | 2 ++ 3 files changed, 11 insertions(+) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h index 0051af29c21..6e0d610da97 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h @@ -46,6 +46,7 @@ public: bool dumpFeatures() const { return _dumpFeatures; } const fef::Properties &highlightTerms() const { return _highlightTerms; } + void highlightTerms(fef::Properties & terms) { _highlightTerms = terms; } void set_fields(const FieldSet& fields_in) { _fields = fields_in; } const FieldSet& get_fields() const { return _fields; } bool need_field(vespalib::stringref field) const; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 68c5f5cfdc8..e6f682321d8 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -187,6 +187,7 @@ SearchVisitor::SummaryGenerator::SummaryGenerator(const search::IAttributeManage _dump_features(), _location(), _stack_dump(), + _highlight_terms(), _attr_manager(attr_manager), _query_normalization(query_normalization) { @@ -219,6 +220,7 @@ SearchVisitor::SummaryGenerator::get_streaming_docsums_state(vespalib::stringref if (_stack_dump.has_value()) { ds._args.setStackDump(_stack_dump.value().size(), _stack_dump.value().data()); } + ds._args.highlightTerms(_highlight_terms); _docsumWriter->initState(_attr_manager, ds, state->get_resolve_class_info()); auto insres = _docsum_states.insert(std::make_pair(summary_class, std::move(state))); return *insres.first->second; @@ -443,6 +445,12 @@ SearchVisitor::init(const Parameters & params) i, j, string(prop.key(j)).c_str(), string(prop.value(j)).c_str()); _rankController.getFeatureOverrides().add(prop.key(j), prop.value(j)); } + } else if (prop.name() == "highlightterms") { // pick up feature overrides + for (uint32_t j = 0; j < prop.size(); ++j) { + LOG(debug, "Hightligthterms[%u][%u]: key '%s' -> value '%s'", + i, j, string(prop.key(j)).c_str(), string(prop.value(j)).c_str()); + _summaryGenerator.highlightTerms().add(prop.key(j), prop.value(j)); + } } } } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 567cb1b1f2f..7d73a159f6f 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -423,6 +423,7 @@ private: void set_location(const vespalib::string& location) { _location = location; } void set_stack_dump(std::vector stack_dump) { _stack_dump = std::move(stack_dump); } void add_summary_field(vespalib::stringref field) { _summaryFields.emplace_back(field); } + search::fef::Properties & highlightTerms() { return _highlight_terms;} private: StreamingDocsumsState& get_streaming_docsums_state(vespalib::stringref summary_class); vsm::GetDocsumsStateCallback _callback; @@ -434,6 +435,7 @@ private: std::optional _dump_features; std::optional _location; std::optional> _stack_dump; + search::fef::Properties _highlight_terms; const search::IAttributeManager& _attr_manager; const search::QueryNormalization & _query_normalization; }; -- cgit v1.2.3 From c1ca22cba9db0424afc82cbd9dc8c6d5dfa6b617 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 18 Feb 2024 09:49:06 +0000 Subject: Normalize highlight terms --- streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index e6f682321d8..63f11f2ad6a 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -449,7 +449,10 @@ SearchVisitor::init(const Parameters & params) for (uint32_t j = 0; j < prop.size(); ++j) { LOG(debug, "Hightligthterms[%u][%u]: key '%s' -> value '%s'", i, j, string(prop.key(j)).c_str(), string(prop.value(j)).c_str()); - _summaryGenerator.highlightTerms().add(prop.key(j), prop.value(j)); + vespalib::stringref index = prop.key(j); + vespalib::stringref term = prop.value(j); + vespalib::string norm_term = QueryNormalization::optional_fold(term, search::TermType::WORD, normalizing_mode(index)); + _summaryGenerator.highlightTerms().add(index, norm_term); } } } -- cgit v1.2.3 From 37a5e1ad7a0ff7fd07ec0ef377058f4a6cded93d Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 19 Feb 2024 13:41:14 +0000 Subject: Revert experimental unintended change. --- client/go/internal/admin/jvm/application_container.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client/go/internal/admin/jvm/application_container.go b/client/go/internal/admin/jvm/application_container.go index 4db70d364f4..3806dfea302 100644 --- a/client/go/internal/admin/jvm/application_container.go +++ b/client/go/internal/admin/jvm/application_container.go @@ -118,7 +118,6 @@ func (a *ApplicationContainer) configureMemory(qc *QrStartConfig) { opts.AddOption(fmt.Sprintf("-Xmx%dm", jvm_heapsize)) opts.AddOption(fmt.Sprintf("-XX:ThreadStackSize=%d", jvm_stacksize)) opts.AddOption(fmt.Sprintf("-XX:MaxDirectMemorySize=%dm", maxDirectMemorySize)) - opts.AddOption(fmt.Sprintf("-XX:Tier4CompileThreshold=1000000")) opts.MaybeAddHugepages(MegaBytesOfMemory(jvm_heapsize)) if jvm_compressedClassSpaceSize > 0 { opts.AddOption(fmt.Sprintf("-XX:CompressedClassSpaceSize=%dm", jvm_compressedClassSpaceSize)) -- cgit v1.2.3 From 7059a097c51a313ff9a7ef7e16894ce7832e0edc Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 19 Feb 2024 13:50:00 +0000 Subject: GC comment not adding any value, besides being incorrect. --- streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 63f11f2ad6a..e2eb3535223 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -445,7 +445,7 @@ SearchVisitor::init(const Parameters & params) i, j, string(prop.key(j)).c_str(), string(prop.value(j)).c_str()); _rankController.getFeatureOverrides().add(prop.key(j), prop.value(j)); } - } else if (prop.name() == "highlightterms") { // pick up feature overrides + } else if (prop.name() == "highlightterms") { for (uint32_t j = 0; j < prop.size(); ++j) { LOG(debug, "Hightligthterms[%u][%u]: key '%s' -> value '%s'", i, j, string(prop.key(j)).c_str(), string(prop.value(j)).c_str()); -- cgit v1.2.3