diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-09-07 12:03:07 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-09-07 12:03:07 +0200 |
commit | 2123fbfb5ad5564fb853822bdb3d7f12f9e2b69e (patch) | |
tree | a57fe1732b3f36937987fa47ac492181d5100990 | |
parent | 88a5e95f8ce908e7f15c6c2b3f08a1fb1bd2fddd (diff) | |
parent | f04a4642e52ac293f2a4b5248bc2d1c14233f38e (diff) |
Merge branch 'master' into balder/unify-access-to-ranking-assets
62 files changed, 310 insertions, 973 deletions
diff --git a/client/go/vespa/tls_options.go b/client/go/vespa/tls_options.go new file mode 100644 index 00000000000..46b0171fae5 --- /dev/null +++ b/client/go/vespa/tls_options.go @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package vespa + +import ( + "bytes" + "encoding/json" + "os" +) + +type VespaTlsConfig struct { + DisableHostnameValidation bool `json:"disable-hostname-validation"` + Files struct { + PrivateKey string `json:"private-key"` + CaCertificates string `json:"ca-certificates"` + Certificates string `json:"certificates"` + } `json:"files"` +} + +func LoadTlsConfig() (*VespaTlsConfig, error) { + fn := os.Getenv("VESPA_TLS_CONFIG_FILE") + if fn == "" { + return nil, nil + } + contents, err := os.ReadFile(fn) + if err != nil { + return nil, err + } + codec := json.NewDecoder(bytes.NewReader(contents)) + var parsedJson VespaTlsConfig + err = codec.Decode(&parsedJson) + if err != nil { + return nil, err + } + return &parsedJson, nil +} + +func ExportSecurityEnvToSh() { + LoadDefaultEnv() + cfg, _ := LoadTlsConfig() + helper := newShellEnvExporter() + if cfg == nil { + helper.unsetVar("VESPA_TLS_ENABLED") + } else { + if fn := cfg.Files.PrivateKey; fn != "" { + helper.overrideVar("VESPA_TLS_PRIVATE_KEY", fn) + } + if fn := cfg.Files.CaCertificates; fn != "" { + helper.overrideVar("VESPA_TLS_CA_CERT", fn) + } + if fn := cfg.Files.Certificates; fn != "" { + helper.overrideVar("VESPA_TLS_CERT", fn) + } + if cfg.DisableHostnameValidation { + helper.overrideVar("VESPA_TLS_HOSTNAME_VALIDATION_DISABLED", "1") + } else { + helper.unsetVar("VESPA_TLS_HOSTNAME_VALIDATION_DISABLED") + } + if os.Getenv("VESPA_TLS_INSECURE_MIXED_MODE") == "" { + helper.overrideVar("VESPA_TLS_ENABLED", "1") + } + } + helper.dump() +} diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 33ceb5338d3..22c67b89829 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -60,7 +60,6 @@ vespa_generate_config(configdefinitions stor-filestor.def) install_config_definition(stor-filestor.def vespa.config.content.stor-filestor.def) vespa_generate_config(configdefinitions summary.def) install_config_definition(summary.def vespa.config.search.summary.def) -vespa_generate_config(configdefinitions summarymap.def) install_config_definition(summarymap.def vespa.config.search.summarymap.def) vespa_generate_config(configdefinitions upgrading.def) install_config_definition(upgrading.def vespa.config.content.upgrading.def) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java index 2211e83c51a..5953c51782a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java @@ -40,7 +40,12 @@ public class TsdbQueryRewriter { JsonNode execution = executionGraph.get(i); // Will be handled by rewriteFilters() - if (execution.has("filterId") && filterExists(root, execution.get("filterId").asText())) continue; + if (execution.has("filterId")) { + if (filterExists(root, execution.get("filterId").asText())) + continue; + else + throw new IllegalArgumentException("Invalid filterId: " + execution.get("filterId").asText()); + } rewriteFilter((ObjectNode) execution, tenantNames, operator, systemName); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2cf9ff94b35..eef051c64b8 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -86,7 +86,7 @@ public class Flags { public static final UnboundBooleanFlag KEEP_STORAGE_NODE_UP = defineFeatureFlag( "keep-storage-node-up", true, - List.of("hakonhall"), "2022-07-07", "2022-09-07", + List.of("hakonhall"), "2022-07-07", "2022-10-07", "Whether to leave the storage node (with wanted state) UP while the node is permanently down.", "Takes effect immediately for nodes transitioning to permanently down.", ZONE_ID, APPLICATION_ID); diff --git a/screwdriver/release-java-artifacts.sh b/screwdriver/release-java-artifacts.sh index 0c8ae0e4eb1..dc288989441 100755 --- a/screwdriver/release-java-artifacts.sh +++ b/screwdriver/release-java-artifacts.sh @@ -59,6 +59,7 @@ echo "No javadoc available for module" > container-spifly/src/main/javadoc/READM # Workaround for broken nexus-staging-maven-plugin instead of swapping JDK export MAVEN_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.desktop/java.awt.font=ALL-UNNAMED" export VESPA_MAVEN_EXTRA_OPTS="--show-version --batch-mode" +export FACTORY_VESPA_VERSION=$VESPA_RELEASE ./bootstrap.sh COMMON_MAVEN_OPTS="$VESPA_MAVEN_EXTRA_OPTS --no-snapshot-updates --settings $(pwd)/screwdriver/settings-publish.xml --activate-profiles ossrh-deploy-vespa -DskipTests" diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index a9c18d4b8a9..4c3acb403e8 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -9,7 +9,6 @@ #include <vespa/config-indexschema.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/config/subscription/sourcespec.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/config/documenttypes_config_fwd.h> @@ -140,7 +139,6 @@ public: indexschema, attributes, summary, - std::make_shared<SummarymapConfig>(), std::make_shared<JuniperrcConfig>(), _typeCfg, _repo, diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp index 4ff459a5a9d..f2433b2adf6 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp @@ -3,7 +3,6 @@ #include <vespa/config-attributes.h> #include <vespa/config-indexschema.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/searchcore/proton/attribute/attribute_aspect_delayer.h> #include <vespa/searchcore/proton/common/i_document_type_inspector.h> #include <vespa/searchcore/proton/common/indexschema_inspector.h> @@ -21,18 +20,8 @@ using vespa::config::search::IndexschemaConfig; using vespa::config::search::IndexschemaConfigBuilder; using vespa::config::search::SummaryConfig; using vespa::config::search::SummaryConfigBuilder; -using vespa::config::search::SummarymapConfig; -using vespa::config::search::SummarymapConfigBuilder; using search::attribute::Config; -namespace std { - -ostream &operator<<(ostream &os, const SummarymapConfig::Override &override) { - return os << "{field=" << override.field << ", command=" << override.command << ", arguments=" << override.arguments << "}"; -} - -} - namespace vespa::config::search::internal { std::ostream &operator<<(std::ostream &os, const SummaryConfig::Classes::Fields &field) { @@ -104,61 +93,6 @@ AttributesConfig::Attribute make_fa(const AttributesConfig::Attribute &cfg) return attr; } -SummarymapConfig::Override make_attribute_override(const vespalib::string &name, const vespalib::string &source) -{ - SummarymapConfig::Override override; - override.field = name; - override.command = "attribute"; - override.arguments = source; - return override; -} - -SummarymapConfig::Override make_attribute_override(const vespalib::string &name) -{ - return make_attribute_override(name, name); -} - -SummarymapConfig::Override make_geopos_override(const vespalib::string &name) -{ - SummarymapConfig::Override override; - override.field = name; - override.command = "geopos"; - override.arguments = name; - return override; -} - -SummarymapConfig::Override make_attribute_combiner_override(const vespalib::string &name, const vespalib::string &source) -{ - SummarymapConfig::Override override; - override.field = name; - override.command = "attributecombiner"; - override.arguments = source; - return override; -} - -SummarymapConfig::Override make_attribute_combiner_override(const vespalib::string &name) -{ - return make_attribute_combiner_override(name, ""); -} - -SummarymapConfig::Override make_matched_attribute_elements_filter_override(const vespalib::string &name, const vespalib::string &source) -{ - SummarymapConfig::Override override; - override.field = name; - override.command = "matchedattributeelementsfilter"; - override.arguments = source; - return override; -} - -SummarymapConfig::Override make_matched_elements_filter_override(const vespalib::string &name, const vespalib::string &source) -{ - SummarymapConfig::Override override; - override.field = name; - override.command = "matchedelementsfilter"; - override.arguments = source; - return override; -} - SummaryConfig::Classes::Fields make_summary_field(const vespalib::string &name, const vespalib::string &type) { SummaryConfig::Classes::Fields field; @@ -187,13 +121,6 @@ SummaryConfig sCfg(std::vector<SummaryConfig::Classes::Fields> fields) return result; } -SummarymapConfig smCfg(std::vector<SummarymapConfig::Override> overrides) -{ - SummarymapConfigBuilder result; - result.override = overrides; - return result; -} - class MyInspector : public IDocumentTypeInspector { std::set<vespalib::string> _unchanged; @@ -236,11 +163,11 @@ public: field.name = name; _oldIndexSchema.indexfield.emplace_back(field); } - void setup(const AttributesConfig &oldAttributesConfig, const SummarymapConfig &oldSummarymapConfig, - const AttributesConfig &newAttributesConfig, const SummaryConfig &newSummaryConfig, const SummarymapConfig &newSummarymapConfig) { + void setup(const AttributesConfig &oldAttributesConfig, + const AttributesConfig &newAttributesConfig, const SummaryConfig &newSummaryConfig) { IndexschemaInspector indexschemaInspector(_oldIndexSchema); - _delayer.setup(oldAttributesConfig, oldSummarymapConfig, - newAttributesConfig, newSummaryConfig, newSummarymapConfig, + _delayer.setup(oldAttributesConfig, + newAttributesConfig, newSummaryConfig, indexschemaInspector, _inspector); } void assertAttributeConfig(const std::vector<AttributesConfig::Attribute> &exp) @@ -248,11 +175,6 @@ public: auto actConfig = _delayer.getAttributesConfig(); EXPECT_EQ(exp, actConfig->attribute); } - void assertSummarymapConfig(const std::vector<SummarymapConfig::Override> &exp) - { - auto summarymapConfig = _delayer.getSummarymapConfig(); - EXPECT_EQ(exp, summarymapConfig->override); - } void assertSummaryConfig(const std::vector<SummaryConfig::Classes::Fields> &exp) { auto summaryConfig = _delayer.getSummaryConfig(); @@ -263,87 +185,77 @@ public: TEST_F(DelayerTest, require_that_empty_config_is_ok) { - setup(attrCfg({}), smCfg({}), attrCfg({}), sCfg({}), smCfg({})); + setup(attrCfg({}), attrCfg({}), sCfg({})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({}); } TEST_F(DelayerTest, require_that_simple_attribute_config_is_ok) { - setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({make_int32_sv_cfg()}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_if_field_type_is_unchanged) { addFields({"a"}); - setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("a", "integer")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_if_field_type_is_unchanged_geopos_override) { addFields({"a"}); - setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "geopos", "a")}), smCfg({make_geopos_override("a")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "geopos", "a")})); assertAttributeConfig({}); - assertSummarymapConfig({make_geopos_override("a")}); assertSummaryConfig({make_summary_field("a", "integer", "geopos", "a")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_if_field_type_is_unchanged_mapped_summary) { addFields({"a"}); - setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a_mapped", "integer", "attribute", "a")}), smCfg({make_attribute_override("a_mapped", "a")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a_mapped", "integer", "attribute", "a")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("a_mapped", "integer", "copy", "a")}); } TEST_F(DelayerTest, require_that_adding_attribute_is_not_delayed_if_field_type_changed) { - setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_delayed_if_field_type_is_unchanged) { addFields({"a"}); - setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "integer")}), smCfg({})); + setup(attrCfg({make_int32_sv_cfg()}), attrCfg({}), sCfg({make_summary_field("a", "integer")})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); } TEST_F(DelayerTest, require_that_summary_map_override_is_removed_when_summary_aspect_is_removed_even_if_removing_attribute_aspect_is_delayed) { addFields({"a"}); - setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({}), smCfg({})); + setup(attrCfg({make_int32_sv_cfg()}), attrCfg({}), sCfg({})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummarymapConfig({}); assertSummaryConfig({}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_delayed_if_field_type_is_unchanged_gepos_override) { addFields({"a"}); - setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_geopos_override("a")}), attrCfg({}), sCfg({}), smCfg({})); + setup(attrCfg({make_int32_sv_cfg()}), attrCfg({}), sCfg({})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummarymapConfig({}); assertSummaryConfig({}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_not_delayed_if_field_type_changed) { - setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "integer")}), smCfg({})); + setup(attrCfg({make_int32_sv_cfg()}), attrCfg({}), sCfg({make_summary_field("a", "integer")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("a", "integer")}); } @@ -351,85 +263,75 @@ TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_not_delayed_if_als { addFields({"a"}); addOldIndexField("a"); - setup(attrCfg({make_string_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "string")}), smCfg({})); + setup(attrCfg({make_string_sv_cfg()}), attrCfg({}), sCfg({make_summary_field("a", "string")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("a", "string")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_for_tensor_field) { addFields({"a"}); - setup(attrCfg({}), smCfg({}), - attrCfg({make_tensor_cfg("tensor(x[10])")}), sCfg({make_summary_field("a", "tensor", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({}), + attrCfg({make_tensor_cfg("tensor(x[10])")}), sCfg({make_summary_field("a", "tensor", "attribute", "a")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("a", "tensor")}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_delayed_for_tensor_field) { addFields({"a"}); - setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), - attrCfg({}), sCfg({make_summary_field("a", "tensor")}), smCfg({})); + setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), + attrCfg({}), sCfg({make_summary_field("a", "tensor")})); assertAttributeConfig({make_tensor_cfg("tensor(x[10])")}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "tensor", "attribute", "a")}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_not_delayed_for_predicate) { addFields({"a"}); - setup(attrCfg({make_predicate_cfg(4)}), smCfg({}), attrCfg({}), sCfg({make_summary_field("a", "string")}), smCfg({})); + setup(attrCfg({make_predicate_cfg(4)}), attrCfg({}), sCfg({make_summary_field("a", "string")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("a", "string")}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_not_delayed_for_reference) { addFields({"a"}); - setup(attrCfg({make_reference_cfg()}), smCfg({}), attrCfg({}), sCfg({make_summary_field("a", "longstring")}), smCfg({})); + setup(attrCfg({make_reference_cfg()}), attrCfg({}), sCfg({make_summary_field("a", "longstring")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("a", "longstring")}); } TEST_F(DelayerTest, require_that_fast_access_flag_change_is_delayed_false_true_edge) { addFields({"a"}); - setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_int32_sv_cfg())}), sCfg({make_summary_field("a", "integer", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({make_int32_sv_cfg()}), attrCfg({make_fa(make_int32_sv_cfg())}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); } TEST_F(DelayerTest, require_that_fast_access_flag_change_is_delayed_true_false_edge) { addFields({"a"}); - setup(attrCfg({make_fa(make_int32_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({make_fa(make_int32_sv_cfg())}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); assertAttributeConfig({make_fa(make_int32_sv_cfg())}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); } TEST_F(DelayerTest, require_that_fast_access_flag_change_is_delayed_false_true_edge_on_tensor_attribute) { addFields({"a"}); - setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), - attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), sCfg({make_summary_field("a", "tensor", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), sCfg({make_summary_field("a", "tensor", "attribute", "a")})); assertAttributeConfig({make_tensor_cfg("tensor(x[10])")}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "tensor", "attribute", "a")}); } TEST_F(DelayerTest, require_that_fast_access_flag_change_is_delayed_true_false_edge_on_tensor_attribute) { addFields({"a"}); - setup(attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), smCfg({make_attribute_override("a")}), - attrCfg({make_tensor_cfg("tensor(x[10])")}), sCfg({make_summary_field("a", "tensor", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), + attrCfg({make_tensor_cfg("tensor(x[10])")}), sCfg({make_summary_field("a", "tensor", "attribute", "a")})); assertAttributeConfig({make_fa(make_tensor_cfg("tensor(x[10])"))}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "tensor", "attribute", "a")}); } @@ -437,44 +339,39 @@ TEST_F(DelayerTest, require_that_fast_access_flag_change_is_not_delayed_true_fal { addFields({"a"}); addOldIndexField("a"); - setup(attrCfg({make_fa(make_string_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_string_sv_cfg()}), sCfg({make_summary_field("a", "string", "attribute", "a")}), smCfg({make_attribute_override("a")})); + setup(attrCfg({make_fa(make_string_sv_cfg())}), attrCfg({make_string_sv_cfg()}), sCfg({make_summary_field("a", "string", "attribute", "a")})); assertAttributeConfig({make_string_sv_cfg()}); - assertSummarymapConfig({make_attribute_override("a")}); assertSummaryConfig({make_summary_field("a", "string", "attribute", "a")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_to_struct_field_is_not_delayed_if_field_type_is_changed) { - setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array")}), smCfg({make_attribute_combiner_override("array")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array")})); assertAttributeConfig({make_int32_sv_cfg("array.a")}); - assertSummarymapConfig({make_attribute_combiner_override("array")}); assertSummaryConfig({make_summary_field("array", "jsonstring", "attributecombiner", "array")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_to_struct_field_is_delayed_if_field_type_is_unchanged) { addFields({"array.a"}); - setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array")}), smCfg({make_attribute_combiner_override("array")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("array", "jsonstring")}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_from_struct_field_is_not_delayed) { addFields({"array.a"}); - setup(attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")}), attrCfg({}), sCfg({make_summary_field("array", "jsonstring")}), smCfg({})); + setup(attrCfg({make_int32_sv_cfg("array.a")}), attrCfg({}), sCfg({make_summary_field("array", "jsonstring")})); assertAttributeConfig({}); - assertSummarymapConfig({}); assertSummaryConfig({make_summary_field("array", "jsonstring")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_to_struct_field_is_delayed_if_field_type_is_unchanged_with_filtering_docsum) { addFields({"array.a"}); - setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array"), make_summary_field("array_filtered", "jsonstring", "matchedattributeelementsfilter", "array")}), smCfg({make_attribute_combiner_override("array"), make_matched_attribute_elements_filter_override("array_filtered", "array")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array"), make_summary_field("array_filtered", "jsonstring", "matchedattributeelementsfilter", "array")})); assertAttributeConfig({}); - assertSummarymapConfig({make_matched_elements_filter_override("array_filtered", "array")}); assertSummaryConfig({make_summary_field("array", "jsonstring"), make_summary_field("array_filtered", "jsonstring", "matchedelementsfilter", "array")}); } diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index b034b77086c..62dd1610147 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -45,6 +45,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/config-summary.h> #include <filesystem> #include <regex> diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index e9fb8be9600..53ea1b3542f 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -209,7 +209,7 @@ Fixture::initViewSet(ViewSet &views) IndexSearchable::SP indexSearchable; auto matchView = std::make_shared<MatchView>(matchers, indexSearchable, attrMgr, sesMgr, metaStore, views._docIdLimit); views.searchView.set(SearchView::create - (summaryMgr->createSummarySetup(SummaryConfig(), SummarymapConfig(), + (summaryMgr->createSummarySetup(SummaryConfig(), JuniperrcConfig(), views.repo, attrMgr), std::move(matchView))); views.feedView.set( @@ -554,7 +554,7 @@ TEST_F("require that reconfigure returns reprocessing initializer", FastAccessFi TEST_F("require that we can reconfigure summary manager", Fixture) { ViewPtrs o = f._views.getViewPtrs(); - ReconfigParams params(CCR().setSummarymapChanged(true)); + ReconfigParams params(CCR().setSummaryChanged(true)); // Use new config snapshot == old config snapshot (only relevant for reprocessing) f._configurer->reconfigure(*createConfig(), *createConfig(), params, f._resolver); @@ -641,7 +641,6 @@ TEST("require that maintenance controller should change if some config has chang TEST_DO(assertMaintenanceControllerShouldChange(CCR().setIndexschemaChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setAttributesChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setSummaryChanged(true))); - TEST_DO(assertMaintenanceControllerShouldChange(CCR().setSummarymapChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setJuniperrcChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setDocumenttypesChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setDocumentTypeRepoChanged(true))); @@ -677,7 +676,6 @@ TEST("require that subdbs should change if relevant config changed") TEST_DO(assertSubDbsShouldChange(CCR().setDocumenttypesChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setDocumentTypeRepoChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setSummaryChanged(true))); - TEST_DO(assertSubDbsShouldChange(CCR().setSummarymapChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setJuniperrcChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setAttributesChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setImportedFieldsChanged(true))); diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index fd47cc9b264..c2d3da1b4f6 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -7,7 +7,6 @@ #include <vespa/searchcore/proton/test/documentdb_config_builder.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/datatype.h> @@ -100,7 +99,7 @@ public: _builder.attributes(make_shared<AttributesConfig>(builder)); return *this; } - MyConfigBuilder &addSummary(bool hasField) { + MyConfigBuilder &addSummary(bool hasField, bool has_attribute) { SummaryConfigBuilder builder; builder.defaultsummaryid = 0; builder.classes.resize(1); @@ -110,18 +109,14 @@ public: builder.classes.back().fields.resize(1); builder.classes.back().fields.back().name = "my_attribute"; builder.classes.back().fields.back().type = "integer"; + if (has_attribute) { + builder.classes.back().fields.back().command = "attribute"; + builder.classes.back().fields.back().source = "my_attribute"; + } } _builder.summary(make_shared<SummaryConfig>(builder)); return *this; } - MyConfigBuilder &addSummarymap() { - SummarymapConfigBuilder builder; - builder.override.resize(1); - builder.override.back().field = "my_attribute"; - builder.override.back().command = "attribute"; - _builder.summarymap(make_shared<SummarymapConfig>(builder)); - return *this; - } ConfigSP build() { return _builder.build(); } @@ -142,15 +137,14 @@ struct Fixture { replayCfg(), nullCfg() { - basicCfg = MyConfigBuilder(4, schema, repo).addAttribute().addSummary(false).build(); + basicCfg = MyConfigBuilder(4, schema, repo).addAttribute().addSummary(false, false).build(); fullCfg = MyConfigBuilder(4, schema, repo).addAttribute(). addRankProfile(). addRankingConstant(). addRankingExpression(). addOnnxModel(). addImportedField(). - addSummary(true). - addSummarymap(). + addSummary(true, true). build(); replayCfg = DocumentDBConfig::makeReplayConfig(fullCfg); } @@ -185,30 +179,29 @@ struct DelayAttributeAspectFixture { addRankingExpression(). addOnnxModel(). addImportedField(). - addSummary(true). - addSummarymap(). + addSummary(true, true). build(); noAttrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(hasDocField)).addRankProfile(). addRankingConstant(). addRankingExpression(). addOnnxModel(). addImportedField(). - addSummary(hasDocField). + addSummary(hasDocField, false). build(); } void assertDelayedConfig(const DocumentDBConfig &testCfg) { EXPECT_FALSE(noAttrCfg->getAttributesConfig() == testCfg.getAttributesConfig()); - EXPECT_FALSE(noAttrCfg->getSummarymapConfig() == testCfg.getSummarymapConfig()); + EXPECT_FALSE(noAttrCfg->getSummaryConfig() == testCfg.getSummaryConfig()); EXPECT_TRUE(attrCfg->getAttributesConfig() == testCfg.getAttributesConfig()); - EXPECT_TRUE(attrCfg->getSummarymapConfig() == testCfg.getSummarymapConfig()); + EXPECT_TRUE(attrCfg->getSummaryConfig() == testCfg.getSummaryConfig()); EXPECT_TRUE(testCfg.getDelayedAttributeAspects()); } void assertNotDelayedConfig(const DocumentDBConfig &testCfg) { EXPECT_TRUE(noAttrCfg->getAttributesConfig() == testCfg.getAttributesConfig()); - EXPECT_TRUE(noAttrCfg->getSummarymapConfig() == testCfg.getSummarymapConfig()); + EXPECT_TRUE(noAttrCfg->getSummaryConfig() == testCfg.getSummaryConfig()); EXPECT_FALSE(attrCfg->getAttributesConfig() == testCfg.getAttributesConfig()); - EXPECT_FALSE(attrCfg->getSummarymapConfig() == testCfg.getSummarymapConfig()); + EXPECT_FALSE(attrCfg->getSummaryConfig() == testCfg.getSummaryConfig()); EXPECT_FALSE(testCfg.getDelayedAttributeAspects()); } }; diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 3f254558740..e1ec5916291 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -6,7 +6,6 @@ #include <vespa/config-indexschema.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/repo/documenttyperepo.h> @@ -89,7 +88,6 @@ assertEqualSnapshot(const DocumentDBConfig &exp, const DocumentDBConfig &act) EXPECT_TRUE(exp.getIndexschemaConfig() == act.getIndexschemaConfig()); EXPECT_TRUE(exp.getAttributesConfig() == act.getAttributesConfig()); EXPECT_TRUE(exp.getSummaryConfig() == act.getSummaryConfig()); - EXPECT_TRUE(exp.getSummarymapConfig() == act.getSummarymapConfig()); EXPECT_TRUE(exp.getJuniperrcConfig() == act.getJuniperrcConfig()); EXPECT_TRUE(exp.getImportedFieldsConfig() == act.getImportedFieldsConfig()); EXPECT_EQUAL(0u, exp.getImportedFieldsConfig().attribute.size()); diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index f6c1518ba90..afd224c55de 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -27,7 +27,6 @@ #include <vespa/config-indexschema.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <map> #include <thread> @@ -58,7 +57,6 @@ struct DoctypeFixture { OnnxModelsConfigBuilder onnxModelsBuilder; IndexschemaConfigBuilder indexschemaBuilder; SummaryConfigBuilder summaryBuilder; - SummarymapConfigBuilder summarymapBuilder; JuniperrcConfigBuilder juniperrcBuilder; ImportedFieldsConfigBuilder importedFieldsBuilder; }; @@ -118,7 +116,6 @@ struct ConfigTestFixture { set.addBuilder(db.configid, &fixture->onnxModelsBuilder); set.addBuilder(db.configid, &fixture->indexschemaBuilder); set.addBuilder(db.configid, &fixture->summaryBuilder); - set.addBuilder(db.configid, &fixture->summarymapBuilder); set.addBuilder(db.configid, &fixture->juniperrcBuilder); set.addBuilder(db.configid, &fixture->importedFieldsBuilder); return dbConfig.emplace(std::make_pair(name, std::move(fixture))).first->second.get(); @@ -155,7 +152,6 @@ struct ConfigTestFixture { fixture->rankProfilesBuilder == dbc->getRankProfilesConfig() && fixture->indexschemaBuilder == dbc->getIndexschemaConfig() && fixture->summaryBuilder == dbc->getSummaryConfig() && - fixture->summarymapBuilder == dbc->getSummarymapConfig() && fixture->juniperrcBuilder == dbc->getJuniperrcConfig()); } @@ -272,7 +268,7 @@ TEST_FF("require that documentdb config manager subscribes for config", DocumentDBConfigManager(f1.configId + "/typea", "typea")) { f1.addDocType("typea"); const ConfigKeySet keySet(f2.createConfigKeySet()); - ASSERT_EQUAL(10u, keySet.size()); + ASSERT_EQUAL(9u, keySet.size()); ASSERT_TRUE(f1.configEqual("typea", getDocumentDBConfig(f1, f2))); } diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 3aa014ca8ac..07fd4ac6fc3 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -6,7 +6,6 @@ #include <vespa/config-indexschema.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/config-bucketspaces.h> #include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/repo/documenttyperepo.h> @@ -54,7 +53,6 @@ struct DBConfigFixture { RankProfilesConfigBuilder _rankProfilesBuilder; IndexschemaConfigBuilder _indexschemaBuilder; SummaryConfigBuilder _summaryBuilder; - SummarymapConfigBuilder _summarymapBuilder; JuniperrcConfigBuilder _juniperrcBuilder; ImportedFieldsConfigBuilder _importedFieldsBuilder; @@ -97,7 +95,6 @@ struct DBConfigFixture { std::make_shared<IndexschemaConfig>(_indexschemaBuilder), std::make_shared<AttributesConfig>(_attributesBuilder), std::make_shared<SummaryConfig>(_summaryBuilder), - std::make_shared<SummarymapConfig>(_summarymapBuilder), std::make_shared<JuniperrcConfig>(_juniperrcBuilder), documentTypes, repo, diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index b18ea6dd89d..2af8f909dac 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -20,7 +20,6 @@ #include <vespa/config-stor-distribution.h> #include <vespa/config-stor-filestor.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/config-upgrading.h> #include <vespa/config/common/configcontext.h> #include <vespa/document/bucket/bucketspace.h> @@ -121,7 +120,6 @@ using vespa::config::search::ImportedFieldsConfig; using vespa::config::search::IndexschemaConfig; using vespa::config::search::RankProfilesConfig; using vespa::config::search::SummaryConfig; -using vespa::config::search::SummarymapConfig; using vespa::config::search::core::ProtonConfig; using vespa::config::search::core::ProtonConfigBuilder; using vespa::config::search::summary::JuniperrcConfig; @@ -193,7 +191,6 @@ std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<Docume indexschema, attributes, summary, - std::make_shared<SummarymapConfig>(), std::make_shared<JuniperrcConfig>(), document_types, repo, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp index 2b954cf4dac..e25bc4c71d5 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp @@ -3,7 +3,6 @@ #include "attribute_aspect_delayer.h" #include <vespa/config-attributes.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/searchcommon/attribute/attribute_utils.h> #include <vespa/searchcore/proton/common/config_hash.hpp> #include <vespa/searchcore/proton/common/i_document_type_inspector.h> @@ -19,8 +18,6 @@ using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespa::config::search::SummaryConfig; using vespa::config::search::SummaryConfigBuilder; -using vespa::config::search::SummarymapConfig; -using vespa::config::search::SummarymapConfigBuilder; namespace proton { @@ -42,39 +39,6 @@ bool willTriggerReprocessOnAttributeAspectRemoval(const search::attribute::Confi !indexschemaInspector.isStringIndex(name); } -class KnownSummaryFields -{ - vespalib::hash_set<vespalib::string> _fields; - -public: - KnownSummaryFields(const SummaryConfig &summaryConfig); - ~KnownSummaryFields(); - - bool known(const vespalib::string &fieldName) const { - return _fields.find(fieldName) != _fields.end(); - } -}; - -KnownSummaryFields::KnownSummaryFields(const SummaryConfig &summaryConfig) - : _fields() -{ - for (const auto &summaryClass : summaryConfig.classes) { - for (const auto &summaryField : summaryClass.fields) { - _fields.insert(summaryField.name); - } - } -} - -KnownSummaryFields::~KnownSummaryFields() = default; - -vespalib::string source_field(const SummarymapConfig::Override &override) { - if (override.arguments == "") { - return override.field; - } else { - return override.arguments; - } -} - vespalib::string source_field(const SummaryConfig::Classes::Fields& summary_field) { @@ -126,10 +90,6 @@ public: ~AttributeAspectConfigRewriter(); void calculate_delayed_attribute_aspects(); void build_attributes_config(AttributesConfigBuilder& attributes_config_builder) const; - void build_summary_map_config(const SummarymapConfig& old_summarymap_config, - const SummarymapConfig& new_summarymap_config, - const SummaryConfig& new_summary_config, - SummarymapConfigBuilder& summary_map_config_builder) const; void build_summary_config(const SummaryConfig& new_summary_config, SummaryConfigBuilder& summary_config_builder) const; }; @@ -259,43 +219,6 @@ AttributeAspectConfigRewriter::build_attributes_config(AttributesConfigBuilder& } void -AttributeAspectConfigRewriter::build_summary_map_config(const SummarymapConfig& old_summarymap_config, - const SummarymapConfig& new_summarymap_config, - const SummaryConfig& new_summary_config, - SummarymapConfigBuilder& summarymap_config_builder) const -{ - KnownSummaryFields knownSummaryFields(new_summary_config); - for (const auto &override : new_summarymap_config.override) { - if (override.command == attribute_dfw_string) { - if (!is_delayed_add_attribute_aspect(source_field(override))) { - summarymap_config_builder.override.emplace_back(override); - } - } else if (override.command == attribute_combiner_dfw_string) { - if (!is_delayed_add_attribute_aspect_struct(source_field(override))) { - summarymap_config_builder.override.emplace_back(override); - } - } else if (override.command == matched_attribute_elements_filter_dfw_string) { - if (!is_delayed_add_attribute_aspect_struct(source_field(override))) { - summarymap_config_builder.override.emplace_back(override); - } else { - SummarymapConfig::Override mutated_override(override); - mutated_override.command = matched_elements_filter_dfw_string; - summarymap_config_builder.override.emplace_back(mutated_override); - } - } else { - summarymap_config_builder.override.emplace_back(override); - } - } - for (const auto &override : old_summarymap_config.override) { - if (override.command == attribute_dfw_string) { - if (is_delayed_remove_attribute_aspect(source_field(override)) && knownSummaryFields.known(override.field)) { - summarymap_config_builder.override.emplace_back(override); - } - } - } -} - -void AttributeAspectConfigRewriter::build_summary_config(const SummaryConfig& new_summary_config, SummaryConfigBuilder& summary_config_builder) const { @@ -338,7 +261,6 @@ AttributeAspectConfigRewriter::build_summary_config(const SummaryConfig& new_sum AttributeAspectDelayer::AttributeAspectDelayer() : _attributesConfig(std::make_shared<AttributesConfigBuilder>()), - _summarymapConfig(std::make_shared<SummarymapConfigBuilder>()), _summaryConfig(std::make_shared<SummaryConfigBuilder>()) { } @@ -353,12 +275,6 @@ AttributeAspectDelayer::getAttributesConfig() const return _attributesConfig; } -std::shared_ptr<AttributeAspectDelayer::SummarymapConfig> -AttributeAspectDelayer::getSummarymapConfig() const -{ - return _summarymapConfig; -} - std::shared_ptr<AttributeAspectDelayer::SummaryConfig> AttributeAspectDelayer::getSummaryConfig() const { @@ -367,10 +283,8 @@ AttributeAspectDelayer::getSummaryConfig() const void AttributeAspectDelayer::setup(const AttributesConfig &oldAttributesConfig, - const SummarymapConfig &oldSummarymapConfig, const AttributesConfig &newAttributesConfig, const SummaryConfig &newSummaryConfig, - const SummarymapConfig &newSummarymapConfig, const IIndexschemaInspector &oldIndexschemaInspector, const IDocumentTypeInspector &inspector) { @@ -379,10 +293,6 @@ AttributeAspectDelayer::setup(const AttributesConfig &oldAttributesConfig, oldIndexschemaInspector, inspector); cfg_rewriter.build_attributes_config(*_attributesConfig); - cfg_rewriter.build_summary_map_config(oldSummarymapConfig, - newSummarymapConfig, - newSummaryConfig, - *_summarymapConfig); cfg_rewriter.build_summary_config(newSummaryConfig, *_summaryConfig); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h index 9d8338349d1..dc9bcd5d963 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h @@ -8,7 +8,6 @@ namespace document { class DocumentType; } namespace vespa::config::search::internal { class InternalAttributesType; class InternalIndexschemaType; -class InternalSummarymapType; class InternalSummaryType; } @@ -27,13 +26,10 @@ class AttributeAspectDelayer using AttributesConfig = const vespa::config::search::internal::InternalAttributesType; using DocumentType = document::DocumentType; using IndexschemaConfig = const vespa::config::search::internal::InternalIndexschemaType; - using SummarymapConfigBuilder = vespa::config::search::internal::InternalSummarymapType; - using SummarymapConfig = const vespa::config::search::internal::InternalSummarymapType; using SummaryConfigBuilder = vespa::config::search::internal::InternalSummaryType; using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; std::shared_ptr<AttributesConfigBuilder> _attributesConfig; - std::shared_ptr<SummarymapConfigBuilder> _summarymapConfig; std::shared_ptr<SummaryConfigBuilder> _summaryConfig; public: @@ -45,15 +41,12 @@ public: * config before applying new config when system is online. */ void setup(const AttributesConfig &oldAttributesConfig, - const SummarymapConfig &oldSummarymapConfig, const AttributesConfig &newAttributesConfig, const SummaryConfig &newSummaryConfig, - const SummarymapConfig &newSummarymapConfig, const IIndexschemaInspector &oldIndexschemaInspector, const IDocumentTypeInspector &inspector); std::shared_ptr<AttributesConfig> getAttributesConfig() const; - std::shared_ptr<SummarymapConfig> getSummarymapConfig() const; std::shared_ptr<SummaryConfig> getSummaryConfig() const; }; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h index 946c45feb4b..e4e32209e3a 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h @@ -1,15 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/docstore/idocumentstore.h> #include <vespa/searchsummary/docsummary/docsumstore.h> #include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> #include <vespa/searchsummary/docsummary/resultconfig.h> -#include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> namespace document { class DocumentTypeRepo; } @@ -21,6 +18,8 @@ namespace proton { class ISummaryManager { public: + using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; + using JuniperrcConfig = const vespa::config::search::summary::internal::InternalJuniperrcType; ISummaryManager(const ISummaryManager &) = delete; ISummaryManager & operator = (const ISummaryManager &) = delete; /** @@ -41,12 +40,11 @@ public: typedef std::unique_ptr<ISummaryManager> UP; typedef std::shared_ptr<ISummaryManager> SP; - virtual ~ISummaryManager() {} + virtual ~ISummaryManager() = default; virtual ISummarySetup::SP - createSummarySetup(const vespa::config::search::SummaryConfig &summaryCfg, - const vespa::config::search::SummarymapConfig &summarymapCfg, - const vespa::config::search::summary::JuniperrcConfig &juniperCfg, + createSummarySetup(const SummaryConfig &summaryCfg, + const JuniperrcConfig &juniperCfg, const std::shared_ptr<const document::DocumentTypeRepo> &repo, const std::shared_ptr<search::IAttributeManager> &attributeMgr) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 6a0133e913d..d801465372a 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -13,6 +13,7 @@ #include <vespa/searchsummary/docsummary/keywordextractor.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/fastlib/text/normwordfolder.h> +#include <vespa/config-summary.h> #include <sstream> @@ -22,8 +23,6 @@ LOG_SETUP(".proton.docsummary.summarymanager"); using namespace config; using namespace document; using namespace search::docsummary; -using namespace vespa::config::search::summary; -using namespace vespa::config::search; using vespalib::make_string; using vespalib::IllegalArgumentException; using vespalib::compression::CompressionConfig; @@ -81,7 +80,7 @@ ShrinkSummaryLidSpaceFlushTarget::initFlush(SerialNum currentSerial, std::shared SummaryManager::SummarySetup:: SummarySetup(const vespalib::string & baseDir, const SummaryConfig & summaryCfg, - const SummarymapConfig & summarymapCfg, const JuniperrcConfig & juniperCfg, + const JuniperrcConfig & juniperCfg, search::IAttributeManager::SP attributeMgr, search::IDocumentStore::SP docStore, std::shared_ptr<const DocumentTypeRepo> repo) : _docsumWriter(), @@ -106,7 +105,6 @@ SummarySetup(const vespalib::string & baseDir, const SummaryConfig & summaryCfg, docsum_field_writer_factory.reset(); _docsumWriter = std::make_unique<DynamicDocsumWriter>(std::move(resultConfig), std::unique_ptr<KeywordExtractor>()); - (void) summarymapCfg; } IDocsumStore::UP @@ -117,11 +115,11 @@ SummaryManager::SummarySetup::createDocsumStore() ISummaryManager::ISummarySetup::SP -SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg, const SummarymapConfig & summarymapCfg, +SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg, const JuniperrcConfig & juniperCfg, const std::shared_ptr<const DocumentTypeRepo> &repo, const search::IAttributeManager::SP &attributeMgr) { - return std::make_shared<SummarySetup>(_baseDir, summaryCfg, summarymapCfg, + return std::make_shared<SummarySetup>(_baseDir, summaryCfg, juniperCfg, attributeMgr, _docStore, repo); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h index 8be949a7351..39df4c5ca70 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h @@ -31,9 +31,8 @@ public: const std::shared_ptr<const document::DocumentTypeRepo> _repo; public: SummarySetup(const vespalib::string & baseDir, - const vespa::config::search::SummaryConfig & summaryCfg, - const vespa::config::search::SummarymapConfig & summarymapCfg, - const vespa::config::search::summary::JuniperrcConfig & juniperCfg, + const SummaryConfig & summaryCfg, + const JuniperrcConfig & juniperCfg, search::IAttributeManager::SP attributeMgr, search::IDocumentStore::SP docStore, std::shared_ptr<const document::DocumentTypeRepo> repo); @@ -70,9 +69,8 @@ public: searchcorespi::IFlushTarget::List getFlushTargets(vespalib::Executor & summaryService); ISummarySetup::SP - createSummarySetup(const vespa::config::search::SummaryConfig &summaryCfg, - const vespa::config::search::SummarymapConfig &summarymapCfg, - const vespa::config::search::summary::JuniperrcConfig &juniperCfg, + createSummarySetup(const SummaryConfig &summaryCfg, + const JuniperrcConfig &juniperCfg, const std::shared_ptr<const document::DocumentTypeRepo> &repo, const search::IAttributeManager::SP &attributeMgr) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index b93f96aa3bb..70478be5b70 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -6,7 +6,6 @@ #include <vespa/config-imported-fields.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/config/config-documenttypes.h> @@ -25,7 +24,6 @@ using namespace vespa::config::search; using document::DocumentTypeRepo; using search::TuneFileDocumentDB; using search::index::Schema; -using vespa::config::search::SummarymapConfig; using vespa::config::search::core::RankingConstantsConfig; using vespa::config::search::core::OnnxModelsConfig; @@ -39,7 +37,6 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() indexschemaChanged(false), attributesChanged(false), summaryChanged(false), - summarymapChanged(false), juniperrcChanged(false), documenttypesChanged(false), documentTypeRepoChanged(false), @@ -62,7 +59,6 @@ DocumentDBConfig::DocumentDBConfig( const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, const SummaryConfigSP &summary, - const SummarymapConfigSP &summarymap, const JuniperrcConfigSP &juniperrc, const DocumenttypesConfigSP &documenttypes, const std::shared_ptr<const DocumentTypeRepo> &repo, @@ -85,7 +81,6 @@ DocumentDBConfig::DocumentDBConfig( _indexschema(indexschema), _attributes(attributes), _summary(summary), - _summarymap(summarymap), _juniperrc(juniperrc), _documenttypes(documenttypes), _repo(repo), @@ -113,7 +108,6 @@ DocumentDBConfig(const DocumentDBConfig &cfg) _indexschema(cfg._indexschema), _attributes(cfg._attributes), _summary(cfg._summary), - _summarymap(cfg._summarymap), _juniperrc(cfg._juniperrc), _documenttypes(cfg._documenttypes), _repo(cfg._repo), @@ -140,7 +134,6 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const equals<IndexschemaConfig>(_indexschema.get(), rhs._indexschema.get()) && equals<AttributesConfig>(_attributes.get(), rhs._attributes.get()) && equals<SummaryConfig>(_summary.get(), rhs._summary.get()) && - equals<SummarymapConfig>(_summarymap.get(), rhs._summarymap.get()) && equals<JuniperrcConfig>(_juniperrc.get(), rhs._juniperrc.get()) && equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()) && _repo.get() == rhs._repo.get() && @@ -165,7 +158,6 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const retval.indexschemaChanged = !equals<IndexschemaConfig>(_indexschema.get(), rhs._indexschema.get()); retval.attributesChanged = !equals<AttributesConfig>(_attributes.get(), rhs._attributes.get()); retval.summaryChanged = !equals<SummaryConfig>(_summary.get(), rhs._summary.get()); - retval.summarymapChanged = !equals<SummarymapConfig>(_summarymap.get(), rhs._summarymap.get()); retval.juniperrcChanged = !equals<JuniperrcConfig>(_juniperrc.get(), rhs._juniperrc.get()); retval.documenttypesChanged = !equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()); retval.documentTypeRepoChanged = _repo.get() != rhs._repo.get(); @@ -191,7 +183,6 @@ DocumentDBConfig::valid() const _indexschema && _attributes && _summary && - _summarymap && _juniperrc && _documenttypes && _repo && @@ -252,7 +243,6 @@ DocumentDBConfig::makeReplayConfig(const SP & orig) o._indexschema, o._attributes, emptyConfig(o._summary), - std::make_shared<SummarymapConfig>(), o._juniperrc, o._documenttypes, o._repo, @@ -296,7 +286,6 @@ DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes) _indexschema, attributes, _summary, - _summarymap, _juniperrc, _documenttypes, _repo, @@ -318,13 +307,12 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum AttributeAspectDelayer attributeAspectDelayer; DocumentTypeInspector inspector(*oldCfg.getDocumentType(), *n.getDocumentType()); IndexschemaInspector oldIndexschemaInspector(oldCfg.getIndexschemaConfig()); - attributeAspectDelayer.setup(oldCfg.getAttributesConfig(), oldCfg.getSummarymapConfig(), - n.getAttributesConfig(), n.getSummaryConfig(), n.getSummarymapConfig(), + attributeAspectDelayer.setup(oldCfg.getAttributesConfig(), + n.getAttributesConfig(), n.getSummaryConfig(), oldIndexschemaInspector, inspector); bool attributes_config_changed = (n.getAttributesConfig() != *attributeAspectDelayer.getAttributesConfig()); - bool summarymap_config_changed = (n.getSummarymapConfig() != *attributeAspectDelayer.getSummarymapConfig()); bool summary_config_changed = (n.getSummaryConfig() != *attributeAspectDelayer.getSummaryConfig()); - bool delayedAttributeAspects = (attributes_config_changed || summarymap_config_changed || summary_config_changed); + bool delayedAttributeAspects = (attributes_config_changed || summary_config_changed); if (!delayedAttributeAspects) { return newCfg; } @@ -337,7 +325,6 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum n._indexschema, (attributes_config_changed ? attributeAspectDelayer.getAttributesConfig() : n._attributes), (summary_config_changed ? attributeAspectDelayer.getSummaryConfig() : n._summary), - (summarymap_config_changed ? attributeAspectDelayer.getSummarymapConfig() : n._summarymap), n._juniperrc, n._documenttypes, n._repo, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 5f5887df6ea..4c814de0b90 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -18,7 +18,6 @@ namespace vespa::config::search::internal { class InternalSummaryType; - class InternalSummarymapType; class InternalRankProfilesType; class InternalAttributesType; class InternalIndexschemaType; @@ -46,7 +45,6 @@ public: bool indexschemaChanged; bool attributesChanged; bool summaryChanged; - bool summarymapChanged; bool juniperrcChanged; bool documenttypesChanged; bool documentTypeRepoChanged; @@ -67,7 +65,6 @@ public: ComparisonResult &setIndexschemaChanged(bool val) { indexschemaChanged = val; return *this; } ComparisonResult &setAttributesChanged(bool val) { attributesChanged = val; return *this; } ComparisonResult &setSummaryChanged(bool val) { summaryChanged = val; return *this; } - ComparisonResult &setSummarymapChanged(bool val) { summarymapChanged = val; return *this; } ComparisonResult &setJuniperrcChanged(bool val) { juniperrcChanged = val; return *this; } ComparisonResult &setDocumenttypesChanged(bool val) { documenttypesChanged = val; return *this; } ComparisonResult &setDocumentTypeRepoChanged(bool val) { documentTypeRepoChanged = val; return *this; } @@ -106,8 +103,6 @@ public: using OnnxModels = matching::OnnxModels; using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; using SummaryConfigSP = std::shared_ptr<SummaryConfig>; - using SummarymapConfig = const vespa::config::search::internal::InternalSummarymapType; - using SummarymapConfigSP = std::shared_ptr<SummarymapConfig>; using JuniperrcConfig = const vespa::config::search::summary::internal::InternalJuniperrcType; using JuniperrcConfigSP = std::shared_ptr<JuniperrcConfig>; using DocumenttypesConfigSP = std::shared_ptr<DocumenttypesConfig>; @@ -126,7 +121,6 @@ private: IndexschemaConfigSP _indexschema; AttributesConfigSP _attributes; SummaryConfigSP _summary; - SummarymapConfigSP _summarymap; JuniperrcConfigSP _juniperrc; DocumenttypesConfigSP _documenttypes; std::shared_ptr<const document::DocumentTypeRepo> _repo; @@ -166,7 +160,6 @@ public: const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, const SummaryConfigSP &summary, - const SummarymapConfigSP &summarymap, const JuniperrcConfigSP &juniperrc, const DocumenttypesConfigSP &documenttypesConfig, const std::shared_ptr<const document::DocumentTypeRepo> &repo, @@ -197,7 +190,6 @@ public: const IndexschemaConfig &getIndexschemaConfig() const { return *_indexschema; } const AttributesConfig &getAttributesConfig() const { return *_attributes; } const SummaryConfig &getSummaryConfig() const { return *_summary; } - const SummarymapConfig &getSummarymapConfig() const { return *_summarymap; } const JuniperrcConfig &getJuniperrcConfig() const { return *_juniperrc; } const DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; } const RankProfilesConfigSP &getRankProfilesConfigSP() const { return _rankProfiles; } @@ -207,7 +199,6 @@ public: const IndexschemaConfigSP &getIndexschemaConfigSP() const { return _indexschema; } const AttributesConfigSP &getAttributesConfigSP() const { return _attributes; } const SummaryConfigSP &getSummaryConfigSP() const { return _summary; } - const SummarymapConfigSP &getSummarymapConfigSP() const { return _summarymap; } const JuniperrcConfigSP &getJuniperrcConfigSP() const { return _juniperrc; } const DocumenttypesConfigSP &getDocumenttypesConfigSP() const { return _documenttypes; } const std::shared_ptr<const document::DocumentTypeRepo> &getDocumentTypeRepoSP() const { return _repo; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 4e6988034ad..8de142034d1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -11,7 +11,6 @@ #include <vespa/config/common/exceptions.h> #include <vespa/config-imported-fields.h> #include <vespa/config-rank-profiles.h> -#include <vespa/config-summarymap.h> #include <vespa/config/file_acquirer/file_acquirer.h> #include <vespa/config/common/configcontext.h> #include <vespa/config/retriever/configretriever.h> @@ -68,7 +67,6 @@ DocumentDBConfigManager::createConfigKeySet() const IndexschemaConfig, AttributesConfig, SummaryConfig, - SummarymapConfig, JuniperrcConfig, ImportedFieldsConfig>(_configId); return set; @@ -291,7 +289,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot using OnnxModelsConfigSP = std::shared_ptr<vespa::config::search::core::OnnxModelsConfig>; using IndexschemaConfigSP = DocumentDBConfig::IndexschemaConfigSP; using SummaryConfigSP = DocumentDBConfig::SummaryConfigSP; - using SummarymapConfigSP = DocumentDBConfig::SummarymapConfigSP; using JuniperrcConfigSP = DocumentDBConfig::JuniperrcConfigSP; using ImportedFieldsConfigSP = DocumentDBConfig::ImportedFieldsConfigSP; using MaintenanceConfigSP = DocumentDBConfig::MaintenanceConfigSP; @@ -392,7 +389,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot } AttributesConfigSP newAttributesConfig = snapshot.getConfig<AttributesConfig>(_configId); SummaryConfigSP newSummaryConfig = snapshot.getConfig<SummaryConfig>(_configId); - SummarymapConfigSP newSummarymapConfig = snapshot.getConfig<SummarymapConfig>(_configId); JuniperrcConfigSP newJuniperrcConfig = snapshot.getConfig<JuniperrcConfig>(_configId); ImportedFieldsConfigSP newImportedFieldsConfig = snapshot.getConfig<ImportedFieldsConfig>(_configId); @@ -411,7 +407,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot newIndexschemaConfig, filterImportedAttributes(newAttributesConfig), newSummaryConfig, - newSummarymapConfig, newJuniperrcConfig, _bootstrapConfig->getDocumenttypesConfigSP(), _bootstrapConfig->getDocumentTypeRepoSP(), diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 5ee70575916..c65114c43ac 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -10,7 +10,6 @@ #include <vespa/document/repo/document_type_repo_factory.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/config-summarymap.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-attributes.h> #include <vespa/config-imported-fields.h> @@ -37,7 +36,6 @@ using vespa::config::search::AttributesConfig; using vespa::config::search::IndexschemaConfig; using vespa::config::search::RankProfilesConfig; using vespa::config::search::SummaryConfig; -using vespa::config::search::SummarymapConfig; using vespa::config::search::core::ProtonConfig; using vespa::config::search::summary::JuniperrcConfig; using vespa::config::content::core::BucketspacesConfig; @@ -210,6 +208,23 @@ getFileList(const vespalib::string &snapDir) return res; } +// add an empty file if it's not already present +void addEmptyFile(vespalib::string snapDir, vespalib::string fileName) +{ + vespalib::string path = snapDir + "/" + fileName; + if (access(path.c_str(), R_OK) == 0) { + // exists OK + return; + } + int fd = creat(path.c_str(), 0444); + if (fd < 0) { + LOG(error, "Could not create empty file '%s': %s", path.c_str(), strerror(errno)); + return; + } + fsync(fd); + close(fd); +} + } FileConfigManager::FileConfigManager(FNET_Transport & transport, @@ -274,9 +289,9 @@ FileConfigManager::saveConfig(const DocumentDBConfig &snapshot, SerialNum serial save(snapDir, snapshot.getIndexschemaConfig()); save(snapDir, snapshot.getAttributesConfig()); save(snapDir, snapshot.getSummaryConfig()); - save(snapDir, snapshot.getSummarymapConfig()); save(snapDir, snapshot.getJuniperrcConfig()); save(snapDir, snapshot.getDocumenttypesConfig()); + addEmptyFile(snapDir, "summarymap.cfg"); bool saveSchemaRes = snapshot.getSchemaSP()->saveToFile(snapDir + "/schema.txt"); assert(saveSchemaRes); @@ -296,26 +311,6 @@ FileConfigManager::saveConfig(const DocumentDBConfig &snapshot, SerialNum serial (void) saveValidSnap; } -namespace { - -// add an empty file if it's not already present -void addEmptyFile(vespalib::string snapDir, vespalib::string fileName) -{ - vespalib::string path = snapDir + "/" + fileName; - if (access(path.c_str(), R_OK) == 0) { - // exists OK - return; - } - int fd = creat(path.c_str(), 0444); - if (fd < 0) { - LOG(error, "Could not create empty file '%s': %s", path.c_str(), strerror(errno)); - return; - } - close(fd); -} - -} - void FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, search::SerialNum serialNum, DocumentDBConfig::SP &loadedSnapshot) diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index 5ed190bad46..2705ef24455 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -20,7 +20,6 @@ ReconfigParams::configHasChanged() const _res.indexschemaChanged || _res.attributesChanged || _res.summaryChanged || - _res.summarymapChanged || _res.juniperrcChanged || _res.documenttypesChanged || _res.documentTypeRepoChanged || @@ -59,7 +58,7 @@ ReconfigParams::shouldAttributeManagerChange() const bool ReconfigParams::shouldSummaryManagerChange() const { - return _res.summaryChanged || _res.summarymapChanged || _res.juniperrcChanged + return _res.summaryChanged || _res.juniperrcChanged || _res.documentTypeRepoChanged || _res.documenttypesChanged || _res.storeChanged; } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 390ed9a0d7a..af562f88a6f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -219,7 +219,6 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, ISummaryManager::SP sumMgr(_summaryMgr); ISummaryManager::ISummarySetup::SP newSumSetup = sumMgr->createSummarySetup(newConfig.getSummaryConfig(), - newConfig.getSummarymapConfig(), newConfig.getJuniperrcConfig(), newConfig.getDocumentTypeRepoSP(), attrMgr); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h index 29ab94f0708..2021caaed87 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h @@ -18,7 +18,6 @@ #include <vespa/config-indexschema.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/vespalib/util/varholder.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 209a78cb00c..79d5abd9f16 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -203,7 +203,6 @@ SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const Sess _rSearchView.set(SearchView::create( getSummaryManager()->createSummarySetup( configSnapshot.getSummaryConfig(), - configSnapshot.getSummarymapConfig(), configSnapshot.getJuniperrcConfig(), configSnapshot.getDocumentTypeRepoSP(), attrMgr), diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp index 5c57f821fe1..afaf62d043c 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp @@ -2,7 +2,6 @@ #include "documentdb_config_builder.h" #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-attributes.h> #include <vespa/config-indexschema.h> @@ -20,7 +19,6 @@ using vespa::config::search::RankProfilesConfig; using vespa::config::search::IndexschemaConfig; using vespa::config::search::AttributesConfig; using vespa::config::search::SummaryConfig; -using vespa::config::search::SummarymapConfig; using vespa::config::search::summary::JuniperrcConfig; using vespa::config::search::ImportedFieldsConfig; using proton::ThreadingServiceConfig; @@ -39,7 +37,6 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation, _indexschema(std::make_shared<IndexschemaConfig>()), _attributes(std::make_shared<AttributesConfig>()), _summary(std::make_shared<SummaryConfig>()), - _summarymap(std::make_shared<SummarymapConfig>()), _juniperrc(std::make_shared<JuniperrcConfig>()), _documenttypes(std::make_shared<DocumenttypesConfig>()), _repo(std::make_shared<document::DocumentTypeRepo>()), @@ -64,7 +61,6 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(const DocumentDBConfig &cfg) _indexschema(cfg.getIndexschemaConfigSP()), _attributes(cfg.getAttributesConfigSP()), _summary(cfg.getSummaryConfigSP()), - _summarymap(cfg.getSummarymapConfigSP()), _juniperrc(cfg.getJuniperrcConfigSP()), _documenttypes(cfg.getDocumenttypesConfigSP()), _repo(cfg.getDocumentTypeRepoSP()), @@ -93,7 +89,6 @@ DocumentDBConfigBuilder::build() _indexschema, _attributes, _summary, - _summarymap, _juniperrc, _documenttypes, _repo, diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index c8d49c3175b..3f84128c7f9 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -19,7 +19,6 @@ private: DocumentDBConfig::IndexschemaConfigSP _indexschema; DocumentDBConfig::AttributesConfigSP _attributes; DocumentDBConfig::SummaryConfigSP _summary; - DocumentDBConfig::SummarymapConfigSP _summarymap; DocumentDBConfig::JuniperrcConfigSP _juniperrc; DocumentDBConfig::DocumenttypesConfigSP _documenttypes; std::shared_ptr<const document::DocumentTypeRepo> _repo; @@ -74,10 +73,6 @@ public: _summary = summary_in; return *this; } - DocumentDBConfigBuilder &summarymap(const DocumentDBConfig::SummarymapConfigSP &summarymap_in) { - _summarymap = summarymap_in; - return *this; - } DocumentDBConfig::SP build(); }; diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_summary_manager.h b/searchcore/src/vespa/searchcore/proton/test/dummy_summary_manager.h index f826cbbe921..a40d89d6edf 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_summary_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_summary_manager.h @@ -3,23 +3,17 @@ #include <vespa/searchcore/proton/docsummary/isummarymanager.h> -namespace proton { - -namespace test { +namespace proton::test { struct DummySummaryManager : public ISummaryManager { ISummarySetup::SP - createSummarySetup(const vespa::config::search::SummaryConfig &, - const vespa::config::search::SummarymapConfig &, - const vespa::config::search::summary::JuniperrcConfig &, + createSummarySetup(const SummaryConfig &, + const JuniperrcConfig &, const std::shared_ptr<const document::DocumentTypeRepo> &, const std::shared_ptr<search::IAttributeManager> &) override { - return ISummarySetup::SP(); + return {}; } }; -} // namespace test - -} // namespace proton - +} diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index f39018846a0..4d0f520f666 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -217,7 +217,6 @@ vespa_define_module( src/tests/sort src/tests/sortresults src/tests/sortspec - src/tests/stringenum src/tests/tensor/dense_tensor_store src/tests/tensor/direct_tensor_store src/tests/tensor/distance_functions diff --git a/searchlib/src/tests/stringenum/.gitignore b/searchlib/src/tests/stringenum/.gitignore deleted file mode 100644 index 7a2f1dd659f..00000000000 --- a/searchlib/src/tests/stringenum/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -*.core -.depend -Makefile -core -core.* -stringenum -tmp.enum -searchlib_stringenum_test_app diff --git a/searchlib/src/tests/stringenum/CMakeLists.txt b/searchlib/src/tests/stringenum/CMakeLists.txt deleted file mode 100644 index d9c5a133c88..00000000000 --- a/searchlib/src/tests/stringenum/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_stringenum_test_app TEST - SOURCES - stringenum_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_stringenum_test_app COMMAND searchlib_stringenum_test_app) diff --git a/searchlib/src/tests/stringenum/stringenum_test.cpp b/searchlib/src/tests/stringenum/stringenum_test.cpp deleted file mode 100644 index f5915db6df0..00000000000 --- a/searchlib/src/tests/stringenum/stringenum_test.cpp +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/log/log.h> -#include <vespa/searchlib/util/stringenum.h> - -LOG_SETUP("stringenum"); -#include <vespa/vespalib/testkit/testapp.h> - -using namespace vespalib; - -void -CheckLookup( search::util::StringEnum *strEnum, const char *str, int value) -{ - EXPECT_EQUAL(0, strcmp(str, strEnum->Lookup(value))); - EXPECT_EQUAL(value, strEnum->Lookup(str)); -} - - -TEST("test StringEnum Add and Lookup") -{ - - search::util::StringEnum enum1; - - // check number of entries - EXPECT_EQUAL(enum1.GetNumEntries(), 0u); - - // check add non-duplicates - EXPECT_EQUAL(enum1.Add("zero"), 0); - EXPECT_EQUAL(enum1.Add("one"), 1); - EXPECT_EQUAL(enum1.Add("two"), 2); - EXPECT_EQUAL(enum1.Add("three"), 3); - EXPECT_EQUAL(enum1.Add("four"), 4); - EXPECT_EQUAL(enum1.Add("five"), 5); - EXPECT_EQUAL(enum1.Add("six"), 6); - EXPECT_EQUAL(enum1.Add("seven"), 7); - EXPECT_EQUAL(enum1.Add("eight"), 8); - EXPECT_EQUAL(enum1.Add("nine"), 9); - - // check add duplicates - EXPECT_EQUAL(enum1.Add("four"), 4); - EXPECT_EQUAL(enum1.Add("eight"), 8); - EXPECT_EQUAL(enum1.Add("six"), 6); - EXPECT_EQUAL(enum1.Add("seven"), 7); - EXPECT_EQUAL(enum1.Add("one"), 1); - EXPECT_EQUAL(enum1.Add("nine"), 9); - EXPECT_EQUAL(enum1.Add("five"), 5); - EXPECT_EQUAL(enum1.Add("zero"), 0); - EXPECT_EQUAL(enum1.Add("two"), 2); - EXPECT_EQUAL(enum1.Add("three"), 3); - - // check add non-duplicate - EXPECT_EQUAL(enum1.Add("ten"), 10); - - // check mapping and reverse mapping - EXPECT_EQUAL(enum1.GetNumEntries(), 11u); - TEST_DO(CheckLookup(&enum1, "zero", 0)); - TEST_DO(CheckLookup(&enum1, "one", 1)); - TEST_DO(CheckLookup(&enum1, "two", 2)); - TEST_DO(CheckLookup(&enum1, "three", 3)); - TEST_DO(CheckLookup(&enum1, "four", 4)); - TEST_DO(CheckLookup(&enum1, "five", 5)); - TEST_DO(CheckLookup(&enum1, "six", 6)); - TEST_DO(CheckLookup(&enum1, "seven", 7)); - TEST_DO(CheckLookup(&enum1, "eight", 8)); - TEST_DO(CheckLookup(&enum1, "nine", 9)); - TEST_DO(CheckLookup(&enum1, "ten", 10)); - - // clear - enum1.Clear(); - - // check number of entries - EXPECT_EQUAL(enum1.GetNumEntries(), 0u); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/util/CMakeLists.txt b/searchlib/src/vespa/searchlib/util/CMakeLists.txt index 619913f9ca6..0d8c8ecb2c2 100644 --- a/searchlib/src/vespa/searchlib/util/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/util/CMakeLists.txt @@ -17,7 +17,6 @@ vespa_add_library(searchlib_util OBJECT rawbuf.cpp slime_output_raw_buf_adapter.cpp state_explorer_utils.cpp - stringenum.cpp url.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/util/filekit.h b/searchlib/src/vespa/searchlib/util/filekit.h index 9720fc1d4a2..8c994ff5866 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.h +++ b/searchlib/src/vespa/searchlib/util/filekit.h @@ -9,8 +9,6 @@ namespace search { class FileKit { -private: - static bool _syncFiles; public: static bool createStamp(const vespalib::string &name); static bool hasStamp(const vespalib::string &name); diff --git a/searchlib/src/vespa/searchlib/util/filesizecalculator.h b/searchlib/src/vespa/searchlib/util/filesizecalculator.h index 590893a0704..be795b84c8b 100644 --- a/searchlib/src/vespa/searchlib/util/filesizecalculator.h +++ b/searchlib/src/vespa/searchlib/util/filesizecalculator.h @@ -6,8 +6,7 @@ namespace vespalib { class GenericHeader; } -namespace search -{ +namespace search { /* * Class to calculate logical file size of a file based on header tags diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index 7d65e298767..cdeca9ce95c 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -115,15 +115,6 @@ void FileReaderBase::handleError(ssize_t numRead, size_t wanted) } } -void FileWriterBase::handleError(ssize_t numRead, size_t wanted) -{ - if (numRead == 0) { - throw std::runtime_error(vespalib::make_string("Failed writing anything to file %s", _file.GetFileName())); - } else { - throw std::runtime_error(vespalib::make_string("Partial read(%zd of %zu) of file %s", numRead, wanted, _file.GetFileName())); - } -} - ssize_t FileReaderBase::read(void *buf, size_t sz) { ssize_t numRead = _file->Read(buf, sz); @@ -133,13 +124,4 @@ FileReaderBase::read(void *buf, size_t sz) { return numRead; } -ssize_t -FileWriterBase::write(const void *buf, size_t sz) { - ssize_t numWritten = _file.Write2(buf, sz); - if (numWritten != ssize_t(sz)) { - handleError(numWritten, sz); - } - return numWritten; -} - } diff --git a/searchlib/src/vespa/searchlib/util/fileutil.h b/searchlib/src/vespa/searchlib/util/fileutil.h index bb74aa47484..3ebb66c5b5e 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.h +++ b/searchlib/src/vespa/searchlib/util/fileutil.h @@ -9,47 +9,46 @@ using vespalib::GenericHeader; -namespace search { +namespace search::fileutil { + +class LoadedBuffer +{ +protected: + void * _buffer; + size_t _size; + std::unique_ptr<GenericHeader> _header; +public: + LoadedBuffer(const LoadedBuffer & rhs) = delete; + LoadedBuffer & operator =(const LoadedBuffer & rhs) = delete; + typedef std::unique_ptr<LoadedBuffer> UP; + + LoadedBuffer(void * buf, size_t sz) + : _buffer(buf), + _size(sz), + _header(nullptr) + { } + + virtual ~LoadedBuffer() = default; + const void * buffer() const { return _buffer; } + const char * c_str() const { return static_cast<const char *>(_buffer); } + size_t size() const { return _size; } + bool empty() const { return _size == 0; } + size_t size(size_t elemSize) const { return _size/elemSize; } + const GenericHeader &getHeader() const { return *_header; } +}; - namespace fileutil { - - class LoadedBuffer - { - protected: - void * _buffer; - size_t _size; - std::unique_ptr<GenericHeader> _header; - public: - LoadedBuffer(const LoadedBuffer & rhs) = delete; - LoadedBuffer & operator =(const LoadedBuffer & rhs) = delete; - typedef std::unique_ptr<LoadedBuffer> UP; - - LoadedBuffer(void * buf, size_t sz) - : _buffer(buf), - _size(sz), - _header(nullptr) - { } - - virtual ~LoadedBuffer() { } - const void * buffer() const { return _buffer; } - const char * c_str() const { return static_cast<const char *>(_buffer); } - size_t size() const { return _size; } - bool empty() const { return _size == 0; } - size_t size(size_t elemSize) const { return _size/elemSize; } - const GenericHeader &getHeader() const { return *_header; } - }; - - class LoadedMmap : public LoadedBuffer - { - void * _mapBuffer; - size_t _mapSize; - public: - LoadedMmap(const vespalib::string &fileName); - - virtual ~LoadedMmap(); - }; +class LoadedMmap : public LoadedBuffer +{ + void * _mapBuffer; + size_t _mapSize; +public: + explicit LoadedMmap(const vespalib::string &fileName); + ~LoadedMmap() override; +}; - } +} + +namespace search { /** * Util class with static functions for handling attribute data files. **/ @@ -73,29 +72,18 @@ public: class FileReaderBase { public: - FileReaderBase(FastOS_FileInterface * file) : _file(file) { } + explicit FileReaderBase(FastOS_FileInterface * file) : _file(file) { } ssize_t read(void *buf, size_t sz); private: void handleError(ssize_t numRead, size_t wanted); FastOS_FileInterface * _file; }; -class FileWriterBase -{ -public: - FileWriterBase(FastOS_FileInterface & file) : _file(file) { } - ssize_t write(const void *buf, size_t sz); -protected: - void handleError(ssize_t numWritten, size_t wanted); -private: - FastOS_FileInterface & _file; -}; - template <typename T> class FileReader : public FileReaderBase { public: - FileReader(FastOS_FileInterface * file) : FileReaderBase(file) { } + explicit FileReader(FastOS_FileInterface * file) : FileReaderBase(file) { } T readHostOrder() { T result; read(&result, sizeof(result)); @@ -108,7 +96,7 @@ class SequentialReadModifyWriteInterface { public: typedef T Type; - virtual ~SequentialReadModifyWriteInterface() { } + virtual ~SequentialReadModifyWriteInterface() = default; virtual const T & read() = 0; virtual void write(const T & v) = 0; virtual bool next() = 0; @@ -124,8 +112,8 @@ private: typedef vespalib::Array<T> Vector; public: SequentialReadModifyWriteVector(); - SequentialReadModifyWriteVector(size_t sz); - ~SequentialReadModifyWriteVector(); + explicit SequentialReadModifyWriteVector(size_t sz); + ~SequentialReadModifyWriteVector() override; const T & read() override { return (*this)[_rp]; } void write(const T & v) override { (*this)[_wp++] = v; } bool next() override { _rp++; return _rp < Vector::size(); } diff --git a/searchlib/src/vespa/searchlib/util/inline.h b/searchlib/src/vespa/searchlib/util/inline.h deleted file mode 100644 index f8485f22de7..00000000000 --- a/searchlib/src/vespa/searchlib/util/inline.h +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/util/inline.h> - diff --git a/searchlib/src/vespa/searchlib/util/stringenum.cpp b/searchlib/src/vespa/searchlib/util/stringenum.cpp deleted file mode 100644 index 116e400083a..00000000000 --- a/searchlib/src/vespa/searchlib/util/stringenum.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "stringenum.h" -#include <vespa/vespalib/stllike/hashtable.hpp> -#include <cassert> - -#include <vespa/log/log.h> -LOG_SETUP(".seachlib.util.stringenum"); - -namespace search::util { - -StringEnum::StringEnum() - : _numEntries(0), - _mapping(), - _reverseMap() -{ -} - -StringEnum::~StringEnum() = default; - -void -StringEnum::CreateReverseMapping() const -{ - _reverseMap.resize(_numEntries); - - for (Map::const_iterator it = _mapping.begin(); - it != _mapping.end(); - it++) - { - assert(it->second >= 0); - assert(it->second < (int)_numEntries); - _reverseMap[it->second] = it->first.c_str(); - } -} - -void -StringEnum::Clear() -{ - _reverseMap.clear(); - _mapping.clear(); - _numEntries = 0; -} - -int -StringEnum::Add(const char *str) -{ - Map::const_iterator found(_mapping.find(str)); - if (found != _mapping.end()) { - return found->second; - } else { - int value = _numEntries++; - _mapping[str] = value; - return value; - } -} - -int -StringEnum::Lookup(const char *str) const -{ - Map::const_iterator found(_mapping.find(str)); - return (found != _mapping.end()) ? found->second : -1; -} - -const char * -StringEnum::Lookup(uint32_t value) const -{ - if (value >= _numEntries) - return nullptr; - - if (_numEntries > _reverseMap.size()) - CreateReverseMapping(); - - return _reverseMap[value]; -} - -} diff --git a/searchlib/src/vespa/searchlib/util/stringenum.h b/searchlib/src/vespa/searchlib/util/stringenum.h deleted file mode 100644 index 0da79db323a..00000000000 --- a/searchlib/src/vespa/searchlib/util/stringenum.h +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vector> -#include <vespa/vespalib/stllike/hash_map.h> - -namespace search::util { - -/** - * An object of this class represents an enumeration of a set of - * strings. This is useful for mapping a set of strings into a - * continuous range of integers. - **/ -class StringEnum -{ -private: - StringEnum(const StringEnum &); - StringEnum& operator=(const StringEnum &); - typedef vespalib::hash_map<vespalib::string, int> Map; - - uint32_t _numEntries; - Map _mapping; - mutable std::vector<const char *> _reverseMap; - - /** - * Create a reverse mapping that enables the user to map integers - * into strings. This method is called by the Lookup(int) method. - **/ - void CreateReverseMapping() const; - -public: - - /** - * Create an empty string enumeration. - **/ - StringEnum(); - - /** - * Destructor. - **/ - ~StringEnum(); - - /** - * Discard all entries held by this object. - **/ - void Clear(); - - /** - * Add a string to this enumeration. Equal strings will get the same - * enumerated value. Different string will get different enumerated - * values. The set of values returned from multiple invocations of - * this method will always be a contiuous range beginning at 0. - * - * @return the enumerated value for the given string. - * @param str string you want to add. - **/ - int Add(const char *str); - - /** - * Obtain the enumerated value for the given string. - * - * @return enumerated value or -1 if not present. - * @param str the string to look up. - **/ - int Lookup(const char *str) const; - - /** - * Obtain the string for the given enumerated value. - * - * @return string or NULL if out of range. - * @param value the enumerated value to look up. - **/ - const char *Lookup(uint32_t value) const; - - /** - * Obtain the number of entries currently present in this - * enumeration. - * - * @return current number of entries. - **/ - uint32_t GetNumEntries() const { return _numEntries; } -}; - -} diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp index 06a20563161..1c4e201e745 100644 --- a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp +++ b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp @@ -74,7 +74,6 @@ DocumentIdDFWTest::DocumentIdDFWTest() { auto* cfg = _result_config->AddResultClass("default", 0); cfg->AddConfigEntry(_field_name.c_str(), search::docsummary::RES_LONG_STRING); - _result_config->CreateEnumMaps(); } diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 4fcf8c10401..d002a9a2748 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -100,7 +100,6 @@ public: EXPECT_TRUE(result_class->AddConfigEntry("array", ResType::RES_JSONSTRING)); EXPECT_TRUE(result_class->AddConfigEntry("map", ResType::RES_JSONSTRING)); EXPECT_TRUE(result_class->AddConfigEntry("map2", ResType::RES_JSONSTRING)); - _config.CreateEnumMaps(); } ~DocsumStore(); const ResultConfig& get_config() const { return _config; } diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index a845a39207f..cbde3d77b4a 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -117,7 +117,6 @@ DocsumFixture::DocsumFixture() EXPECT_TRUE(cfg->AddConfigEntry("longdata_field", RES_LONG_DATA)); EXPECT_TRUE(cfg->AddConfigEntry("int_pair_field", RES_JSONSTRING)); config->set_default_result_class_id(0); - config->CreateEnumMaps(); writer = std::make_unique<DynamicDocsumWriter>(std::move(config), std::unique_ptr<KeywordExtractor>()); int_pair_type.addField(Field("foo", *DataType::INT)); int_pair_type.addField(Field("bar", *DataType::INT)); diff --git a/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp b/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp index 1d7795d26dc..0eff397bc10 100644 --- a/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp +++ b/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp @@ -40,7 +40,6 @@ #include <vespa/searchsummary/docsummary/linguisticsannotation.h> #include <vespa/searchsummary/docsummary/searchdatatype.h> #include <vespa/searchcommon/common/schema.h> -#include <vespa/config-summarymap.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/json_format.h> @@ -90,8 +89,6 @@ using document::WeightedSetFieldValue; using search::index::Schema; using search::linguistics::SPANTREE_NAME; using search::linguistics::TERM; -using vespa::config::search::SummarymapConfig; -using vespa::config::search::SummarymapConfigBuilder; using vespalib::Slime; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; @@ -136,7 +133,6 @@ FieldBlock::~FieldBlock() = default; class Test : public vespalib::TestApp { std::unique_ptr<Schema> _schema; - std::unique_ptr<SummarymapConfigBuilder> _summarymap; std::shared_ptr<const DocumentTypeRepo> _documentRepo; const DocumentType *_documentType; document::FixedTypeRepo _fixedRepo; @@ -286,7 +282,6 @@ Test::Main() void Test::setUp() { _schema = std::make_unique<Schema>(); - _summarymap = std::make_unique<SummarymapConfigBuilder>(); } void Test::tearDown() { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 181af0ca9a3..422400dc2ef 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -61,8 +61,8 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *resCfg = rci.outputClass->GetEntry(i); const DocsumFieldWriter *writer = resCfg->_docsum_field_writer.get(); - if (state->_args.needField(resCfg->_bindname) && ! writer->isDefaultValue(docid, state)) { - const Memory field_name(resCfg->_bindname.data(), resCfg->_bindname.size()); + if (state->_args.needField(resCfg->_name) && ! writer->isDefaultValue(docid, state)) { + const Memory field_name(resCfg->_name.data(), resCfg->_name.size()); ObjectInserter inserter(docsum, field_name); writer->insertField(docid, nullptr, state, resCfg->_type, inserter); } @@ -77,9 +77,9 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, vespalib::slime::Cursor & docsum = topInserter.insertObject(); for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *outCfg = rci.outputClass->GetEntry(i); - if ( ! state->_args.needField(outCfg->_bindname)) continue; + if ( ! state->_args.needField(outCfg->_name)) continue; const DocsumFieldWriter *writer = outCfg->_docsum_field_writer.get(); - const Memory field_name(outCfg->_bindname.data(), outCfg->_bindname.size()); + const Memory field_name(outCfg->_name.data(), outCfg->_name.size()); ObjectInserter inserter(docsum, field_name); if (writer != nullptr) { if (! writer->isDefaultValue(docid, state)) { @@ -87,7 +87,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, } } else { if (doc) { - doc->insert_summary_field(outCfg->_bindname, inserter); + doc->insert_summary_field(outCfg->_name, inserter); } } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp index 8f6d6ed02ea..feb4c63df34 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp @@ -2,8 +2,8 @@ #include "juniperproperties.h" #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/searchsummary/config/config-juniperrc.h> -using vespa::config::search::summary::JuniperrcConfig; using vespalib::make_string; namespace search::docsummary { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h index 88e819056cf..98c8a8eadb2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/juniper/IJuniperProperties.h> +#include <vespa/vespalib/stllike/string.h> #include <map> +namespace vespa::config::search::summary::internal { + class InternalJuniperrcType; +} namespace search::docsummary { class JuniperProperties : public IJuniperProperties { @@ -19,6 +22,7 @@ private: public: + using JuniperrcConfig = const vespa::config::search::summary::internal::InternalJuniperrcType;; /** * Constructs a juniper property object with default values set. */ @@ -26,7 +30,7 @@ public: /** * Constructs a juniper property object with default values set. */ - explicit JuniperProperties(const vespa::config::search::summary::JuniperrcConfig &cfg); + explicit JuniperProperties(const JuniperrcConfig &cfg); ~JuniperProperties() override; @@ -35,7 +39,7 @@ public: * * @param cfg The configuration object. */ - void configure(const vespa::config::search::summary::JuniperrcConfig &cfg); + void configure(const JuniperrcConfig &cfg); // Inherit doc from IJuniperProperties. const char *GetProperty(const char *name, const char *def) const override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.cpp b/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.cpp index b00b5d975a2..0bf9e61fc36 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.cpp @@ -7,8 +7,7 @@ namespace search::docsummary { ResConfigEntry::ResConfigEntry() noexcept : _type(RES_BAD), - _bindname(), - _enumValue(0), + _name(), _docsum_field_writer() { } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.h b/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.h index 6277e955c3d..b5b5c620c88 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.h @@ -15,8 +15,7 @@ class DocsumFieldWriter; **/ struct ResConfigEntry { ResType _type; - vespalib::string _bindname; - int _enumValue; + vespalib::string _name; std::unique_ptr<DocsumFieldWriter> _docsum_field_writer; ResConfigEntry() noexcept; ~ResConfigEntry(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index 78a9804ccd4..781cd62a818 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -8,12 +8,10 @@ namespace search::docsummary { -ResultClass::ResultClass(const char *name, util::StringEnum & fieldEnum) +ResultClass::ResultClass(const char *name) : _name(name), _entries(), _nameMap(), - _fieldEnum(fieldEnum), - _enumMap(), _dynInfo(), _omit_summary_features(false), _num_field_writer_states(0) @@ -37,10 +35,8 @@ ResultClass::AddConfigEntry(const char *name, ResType type, std::unique_ptr<Docs _nameMap[name] = _entries.size(); ResConfigEntry e; - e._type = type; - e._bindname = name; - e._enumValue = _fieldEnum.Add(name); - assert(e._enumValue >= 0); + e._type = type; + e._name = name; if (docsum_field_writer) { docsum_field_writer->setIndex(_entries.size()); bool generated = docsum_field_writer->IsGenerated(); @@ -60,17 +56,4 @@ ResultClass::AddConfigEntry(const char *name, ResType type) return AddConfigEntry(name, type, {}); } -void -ResultClass::CreateEnumMap() -{ - _enumMap.resize(_fieldEnum.GetNumEntries()); - - for (int & value : _enumMap) { - value = -1; - } - for (uint32_t i(0); i < _entries.size(); i++) { - _enumMap[_entries[i]._enumValue] = i; - } -} - } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index e11032d1aac..7b4168a3ed9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -5,7 +5,6 @@ #include "res_config_entry.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/searchlib/util/stringenum.h> namespace search::docsummary { @@ -36,16 +35,12 @@ public: }; private: - ResultClass(const ResultClass &); - ResultClass& operator=(const ResultClass &); - typedef vespalib::hash_map<vespalib::string, int> NameIdMap; - typedef std::vector<ResConfigEntry> Configs; + using NameIdMap = vespalib::hash_map<vespalib::string, int>; + using Configs = std::vector<ResConfigEntry>; vespalib::string _name; // name of this class Configs _entries; // config entries for this result class NameIdMap _nameMap; // fieldname -> entry index - util::StringEnum &_fieldEnum; // fieldname -> f.n. enum value [SHARED] - std::vector<int> _enumMap; // fieldname enum value -> entry index DynamicInfo _dynInfo; // fields overridden and generated // Whether or not summary features should be omitted when filling this summary class. // As default, summary features are always included. @@ -53,16 +48,17 @@ private: size_t _num_field_writer_states; public: - typedef std::unique_ptr<ResultClass> UP; + using UP = std::unique_ptr<ResultClass>; /** * Constructor. Assign name and id to this result class. Also gain * ref. to shared string enum object and insert into linked list. * * @param name the name of this result class. - * @param fieldEnum shared object used to enumerate field names. **/ - ResultClass(const char *name, util::StringEnum & fieldEnum); + explicit ResultClass(const char *name); + ResultClass(const ResultClass &) = delete; + ResultClass& operator=(const ResultClass &) = delete; /** * Destructor. Delete internal structures. @@ -101,25 +97,6 @@ public: bool AddConfigEntry(const char *name, ResType type, std::unique_ptr<DocsumFieldWriter> docsum_field_writer); bool AddConfigEntry(const char *name, ResType type); - - - /** - * This method may be called to create an internal mapping from - * field name enumerated value to field index. When building up a - * result configuration possibly containing several result classes, - * all field names are enumerated (across all result classes), - * assigning a single unique integer value to each field name. This - * is done with the StringEnum object given to the - * constructor. This way, fastserver components that want to - * reference a unique field name may use the enumerated value - * instead of the string itself. NOTE: This method must be called in - * order to use the GetIndexFromEnumValue method. NOTE2: This method - * is called by the ResultConfig::CreateEnumMaps method; no - * need to call it directly. - **/ - void CreateEnumMap(); - - /** * Obtain the field index from the field name. The field index may * be used to look up a config entry in this object, or to look up a @@ -135,34 +112,12 @@ public: **/ int GetIndexFromName(const char* name) const; - - /** - * Obtain the field index from the field name enumerated value. The - * field index may be used to look up a config entry in this object, - * or to look up a result entry in a GeneralResult - * object. NOTE: When using the return value from this method to - * look up a result entry in a GeneralResult object, make sure - * that the GeneralResult object has this object as it's - * result class. NOTE2: This method is called by the - * GeneralResult::GetEntryFromEnumValue method; no need to - * call it directly. NOTE3: You need to call the CreateEnumMap - * method before calling this one. - * - * @return field index or -1 if not found - **/ - int GetIndexFromEnumValue(uint32_t value) const - { - return (value < _enumMap.size()) ? _enumMap[value] : -1; - } - - /** * Obtain config entry by field index. * * @return config entry or NULL if not found. **/ - const ResConfigEntry *GetEntry(uint32_t offset) const - { + const ResConfigEntry *GetEntry(uint32_t offset) const { return (offset < _entries.size()) ? &_entries[offset] : nullptr; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index 63ccf80d4e7..0a4f607b962 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -4,8 +4,8 @@ #include "docsum_field_writer.h" #include "docsum_field_writer_factory.h" #include "resultclass.h" -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/config-summary.h> #include <atomic> #include <vespa/log/log.h> @@ -23,11 +23,9 @@ ResultConfig::Clean() ResultConfig::ResultConfig() : _defaultSummaryId(-1), - _useV8geoPositions(false), _classLookup(), _nameLookup() { - } @@ -40,7 +38,7 @@ ResultConfig::~ResultConfig() void ResultConfig::Reset() { - if (! _classLookup.empty() || _fieldEnum.GetNumEntries() > 0) { + if (! _classLookup.empty()) { Clean(); } } @@ -52,7 +50,7 @@ ResultConfig::AddResultClass(const char *name, uint32_t id) ResultClass *ret = nullptr; if (id != NoClassID() && (_classLookup.find(id) == _classLookup.end())) { - auto rc = std::make_unique<ResultClass>(name, _fieldEnum); + auto rc = std::make_unique<ResultClass>(name); ret = rc.get(); _classLookup[id] = std::move(rc); if (_nameLookup.find(name) != _nameLookup.end()) { @@ -84,14 +82,6 @@ ResultConfig::LookupResultClassId(const vespalib::string &name) const } -void -ResultConfig::CreateEnumMaps() -{ - for (auto & entry : _classLookup) { - entry.second->CreateEnumMap(); - } -} - namespace { std::atomic<bool> global_useV8geoPositions = false; } @@ -101,13 +91,12 @@ bool ResultConfig::wantedV8geoPositions() { } bool -ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const char *configId, IDocsumFieldWriterFactory& docsum_field_writer_factory) +ResultConfig::ReadConfig(const SummaryConfig &cfg, const char *configId, IDocsumFieldWriterFactory& docsum_field_writer_factory) { bool rc = true; Reset(); int maxclassID = 0x7fffffff; // avoid negative classids _defaultSummaryId = cfg.defaultsummaryid; - _useV8geoPositions = cfg.usev8geopositions; global_useV8geoPositions = cfg.usev8geopositions; for (uint32_t i = 0; rc && i < cfg.classes.size(); i++) { @@ -156,9 +145,7 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const } } } - if (rc) { - CreateEnumMaps(); // create mappings needed by TVM - } else { + if (!rc) { Reset(); // FAIL, discard all config } return rc; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h index 9aaf6fa65b3..810888fb637 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h @@ -3,9 +3,11 @@ #pragma once #include "res_type_utils.h" -#include <vespa/config-summary.h> -#include <vespa/searchlib/util/stringenum.h> +#include <vespa/vespalib/stllike/hash_map.h> +namespace vespa::config::search::internal { + class InternalSummaryType; +} namespace search::docsummary { class IDocsumFieldWriterFactory; @@ -28,21 +30,16 @@ class ResultClass; class ResultConfig { private: - ResultConfig(const ResultConfig &); - ResultConfig& operator=(const ResultConfig &); - - typedef vespalib::hash_map<vespalib::string, uint32_t> NameMap; - typedef vespalib::hash_map<uint32_t, std::unique_ptr<ResultClass>> IdMap; + using NameMap = vespalib::hash_map<vespalib::string, uint32_t>; + using IdMap = vespalib::hash_map<uint32_t, std::unique_ptr<ResultClass>>; uint32_t _defaultSummaryId; - bool _useV8geoPositions; - search::util::StringEnum _fieldEnum; IdMap _classLookup; NameMap _nameLookup; // name -> class id void Clean(); public: - bool useV8geoPositions() const { return _useV8geoPositions; } + using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; class iterator { public: iterator(IdMap::iterator it) : _it(it) { } @@ -69,16 +66,18 @@ public: IdMap::const_iterator _it; }; - iterator begin() { return iterator(_classLookup.begin()); } - iterator end() { return iterator(_classLookup.end()); } - const_iterator begin() const { return const_iterator(_classLookup.begin()); } - const_iterator end() const { return const_iterator(_classLookup.end()); } + iterator begin() { return { _classLookup.begin() }; } + iterator end() { return { _classLookup.end() }; } + const_iterator begin() const { return { _classLookup.begin() }; } + const_iterator end() const { return { _classLookup.end() }; } /** * Constructor. Create an initially empty result configuration. * NOTE: This method simply calls the Init method. **/ ResultConfig(); + ResultConfig(const ResultConfig &) = delete; + ResultConfig& operator=(const ResultConfig &) = delete; /** * Destructor. Delete all internal structures. NOTE: This method @@ -145,39 +144,13 @@ public: **/ uint32_t LookupResultClassId(const vespalib::string &name) const; - - /** - * Obtain the number of result classes held by this result - * configuration. - * - * @return number of result classes. - **/ - uint32_t GetNumResultClasses() const { return _classLookup.size(); } - - - /** - * Obtain the string enumeration object that holds the mapping from - * field name to field name enumerated value. - * - * @return field name enumeration. - **/ - const search::util::StringEnum & GetFieldNameEnum() const { return _fieldEnum; } - - - /** - * This method calls the CreateEnumMap on all result classes held by - * this object. This is needed in order to look up fields by field - * name enumerated value. - **/ - void CreateEnumMaps(); - /** * Read config that has been fetched from configserver. * * @return true(success)/false(fail) * @param configId reference on server **/ - bool ReadConfig(const vespa::config::search::SummaryConfig &cfg, const char *configId, IDocsumFieldWriterFactory& docsum_field_writer_factory); + bool ReadConfig(const SummaryConfig &cfg, const char *configId, IDocsumFieldWriterFactory& docsum_field_writer_factory); }; } diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp index 4f9fd25098b..1a104727f43 100644 --- a/storage/src/tests/distributor/garbagecollectiontest.cpp +++ b/storage/src/tests/distributor/garbagecollectiontest.cpp @@ -26,7 +26,7 @@ struct GarbageCollectionOperationTest : Test, DistributorStripeTestUtil { spi::IdAndTimestamp _e2; spi::IdAndTimestamp _e3; spi::IdAndTimestamp _e4; - + spi::IdAndTimestamp _e5; GarbageCollectionOperationTest() : _bucket_id(16, 1), @@ -35,7 +35,8 @@ struct GarbageCollectionOperationTest : Test, DistributorStripeTestUtil { _e1(DocumentId("id:foo:bar::doc-1"), spi::Timestamp(100)), _e2(DocumentId("id:foo:bar::doc-2"), spi::Timestamp(200)), _e3(DocumentId("id:foo:bar::doc-3"), spi::Timestamp(300)), - _e4(DocumentId("id:foo:bar::doc-4"), spi::Timestamp(400)) + _e4(DocumentId("id:foo:bar::doc-4"), spi::Timestamp(400)), + _e5(DocumentId("id:foo:bar::doc-4"), spi::Timestamp(500)) // Same as e4 but with higher timestamp {} void SetUp() override { @@ -225,16 +226,16 @@ TEST_F(GarbageCollectionOperationTest, first_phase_sends_enumerate_only_remove_l } } -TEST_F(GarbageCollectionOperationTest, second_phase_sends_intersection_of_returned_entries_with_feed_pri) { +TEST_F(GarbageCollectionOperationTest, second_phase_sends_highest_timestamped_union_of_returned_entries_with_feed_pri) { enable_two_phase_gc(); auto op = create_op(); op->start(_sender, framework::MilliSecTime(0)); ASSERT_EQ(2, _sender.commands().size()); auto r1 = make_remove_location_reply(*_sender.command(0)); - r1->set_selection_matches({_e1, _e2, _e3}); + r1->set_selection_matches({_e1, _e2, _e3, _e5}); auto r2 = make_remove_location_reply(*_sender.command(1)); - r2->set_selection_matches({_e2, _e3, _e4}); // e2, e3 in common with r1 + r2->set_selection_matches({_e2, _e3, _e4}); _sender.commands().clear(); op->receive(_sender, r1); @@ -242,7 +243,8 @@ TEST_F(GarbageCollectionOperationTest, second_phase_sends_intersection_of_return op->receive(_sender, r2); ASSERT_EQ(2u, _sender.commands().size()); // Phase 2 sent - std::vector<spi::IdAndTimestamp> expected({_e2, _e3}); + // e5 is same doc as e4, but at a higher timestamp; only e5 entry should be included. + std::vector<spi::IdAndTimestamp> expected({_e1, _e2, _e3, _e5}); for (int i : {0, 1}) { auto cmd = as_remove_location_command(_sender.command(i)); EXPECT_FALSE(cmd->only_enumerate_docs()); @@ -267,47 +269,6 @@ TEST_F(GarbageCollectionOperationTest, no_second_phase_if_first_phase_has_no_res EXPECT_NO_FATAL_FAILURE(assert_gc_op_completed_ok_without_second_phase(*op)); } -TEST_F(GarbageCollectionOperationTest, no_second_phase_if_first_phase_has_results_but_intersection_is_empty) { - enable_two_phase_gc(); - auto op = create_op(); - op->start(_sender, framework::MilliSecTime(0)); - ASSERT_EQ(2, _sender.commands().size()); - - // No docs in common - auto r1 = make_remove_location_reply(*_sender.command(0)); - r1->set_selection_matches({_e1}); - auto r2 = make_remove_location_reply(*_sender.command(1)); - r2->set_selection_matches({_e2}); - - _sender.commands().clear(); - op->receive(_sender, r1); - op->receive(_sender, r2); - - EXPECT_NO_FATAL_FAILURE(assert_gc_op_completed_ok_without_second_phase(*op)); -} - -// We explicitly test the case where the first reply has an empty result set since we internally -// establish the baseline candidate set from the first reply. This test case leaks some internal -// implementation details, but such is life. -TEST_F(GarbageCollectionOperationTest, no_second_phase_if_first_phase_intersection_empty_first_reply_is_empty_case) { - enable_two_phase_gc(); - auto op = create_op(); - op->start(_sender, framework::MilliSecTime(0)); - ASSERT_EQ(2, _sender.commands().size()); - - auto r1 = make_remove_location_reply(*_sender.command(0)); - r1->set_selection_matches({}); - auto r2 = make_remove_location_reply(*_sender.command(1)); - r2->set_selection_matches({_e1, _e2, _e3, _e4}); - - _sender.commands().clear(); - op->receive(_sender, r1); - op->receive(_sender, r2); - - EXPECT_NO_FATAL_FAILURE(assert_gc_op_completed_ok_without_second_phase(*op)); -} - - TEST_F(GarbageCollectionOperationTest, db_metrics_and_timestamp_are_updated_on_second_phase_completion) { enable_two_phase_gc(); auto op = create_op(); @@ -317,7 +278,7 @@ TEST_F(GarbageCollectionOperationTest, db_metrics_and_timestamp_are_updated_on_s auto r1 = make_remove_location_reply(*_sender.command(0)); r1->set_selection_matches({_e1, _e2, _e3}); auto r2 = make_remove_location_reply(*_sender.command(1)); - r2->set_selection_matches({_e2, _e3, _e4}); // e2, e3 in common with r1 + r2->set_selection_matches({_e2, _e3, _e4}); _sender.commands().clear(); op->receive(_sender, r1); @@ -421,6 +382,7 @@ TEST_F(GarbageCollectionOperationTest, document_level_write_locks_are_checked_an _sender.commands().clear(); op->receive(_sender, r1); op->receive(_sender, r2); + ASSERT_EQ(2, _sender.commands().size()); // Locks on e1 and e3 are held while GC removes are sent auto e1_lock = _operation_sequencer.try_acquire(FixedBucketSpaces::default_space(), _e1.id); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index 3e82b6b10a7..d3e4e49c193 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -7,7 +7,7 @@ #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/node_supported_features_repo.h> #include <vespa/storageapi/message/removelocation.h> -#include <vespa/vespalib/stllike/hash_set.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <algorithm> #include <vespa/log/log.h> @@ -20,8 +20,7 @@ GarbageCollectionOperation::GarbageCollectionOperation(const ClusterContext& clu _tracker(cluster_ctx), _phase(Phase::NotStarted), _cluster_state_version_at_phase1_start_time(0), - _phase1_replies_received(0), - _remove_candidate_set(), + _remove_candidates(), _replica_info(), _max_documents_removed(0), _is_done(false) @@ -50,7 +49,11 @@ bool GarbageCollectionOperation::all_involved_nodes_support_two_phase_gc() const } std::vector<spi::IdAndTimestamp> GarbageCollectionOperation::compile_phase_two_send_set() const { - std::vector<spi::IdAndTimestamp> docs_to_remove(_remove_candidate_set.begin(), _remove_candidate_set.end()); + std::vector<spi::IdAndTimestamp> docs_to_remove; + docs_to_remove.reserve(_remove_candidates.size()); + for (const auto& cand : _remove_candidates) { + docs_to_remove.emplace_back(cand.first, cand.second); + } // Use timestamp order to provide test determinism and allow for backend linear merging (if needed). // Tie-break on GID upon collisions (which technically should never happen...!) auto ts_then_gid_order = [](const spi::IdAndTimestamp& lhs, const spi::IdAndTimestamp& rhs) noexcept { @@ -155,36 +158,24 @@ void GarbageCollectionOperation::handle_ok_legacy_reply(uint16_t from_node, cons update_replica_response_info_from_reply(from_node, reply); } -GarbageCollectionOperation::RemoveCandidateSet -GarbageCollectionOperation::steal_selection_matches_as_set(api::RemoveLocationReply& reply) { +GarbageCollectionOperation::RemoveCandidates +GarbageCollectionOperation::steal_selection_matches_as_candidates(api::RemoveLocationReply& reply) { auto candidates = reply.steal_selection_matches(); - RemoveCandidateSet set; - set.resize(candidates.size()); + RemoveCandidates as_map; + as_map.resize(candidates.size()); for (auto& cand : candidates) { - set.insert(std::move(cand)); + as_map.insert(std::make_pair(std::move(cand.id), cand.timestamp)); } - return set; + return as_map; } void GarbageCollectionOperation::handle_ok_phase1_reply(api::RemoveLocationReply& reply) { assert(reply.documents_removed() == 0); - if (_phase1_replies_received == 0) { - // Establish baseline candidate set. Since we require an intersection between all - // sets, the number of candidates can never be _greater_ than that of the first reply. - _remove_candidate_set = steal_selection_matches_as_set(reply); - } else if (!_remove_candidate_set.empty()) { - auto their_set = steal_selection_matches_as_set(reply); - std::vector<spi::IdAndTimestamp> to_remove; - for (auto& our_cand : _remove_candidate_set) { - if (!their_set.contains(our_cand)) { - to_remove.emplace_back(our_cand); - } - } - for (auto& rm_entry : to_remove) { - _remove_candidate_set.erase(rm_entry); - } + auto their_matches = steal_selection_matches_as_candidates(reply); + for (auto& new_cand : their_matches) { + auto& maybe_existing_ts = _remove_candidates[new_cand.first]; + maybe_existing_ts = std::max(new_cand.second, maybe_existing_ts); } - ++_phase1_replies_received; } void GarbageCollectionOperation::handle_ok_phase2_reply(uint16_t from_node, const api::RemoveLocationReply& reply) { @@ -220,30 +211,30 @@ void GarbageCollectionOperation::on_metadata_read_phase_done(DistributorStripeMe mark_operation_complete(); return; } - std::vector<spi::IdAndTimestamp> already_pending_write; - for (auto& cand : _remove_candidate_set) { - auto maybe_seq_token = sender.operation_sequencer().try_acquire(getBucket().getBucketSpace(), cand.id); + std::vector<document::DocumentId> already_pending_write; + for (auto& cand : _remove_candidates) { + auto maybe_seq_token = sender.operation_sequencer().try_acquire(getBucket().getBucketSpace(), cand.first); if (maybe_seq_token.valid()) { _gc_write_locks.emplace_back(std::move(maybe_seq_token)); LOG(spam, "GC(%s): acquired write lock for '%s'; adding to GC set", - getBucket().toString().c_str(), cand.id.toString().c_str()); + getBucket().toString().c_str(), cand.first.toString().c_str()); } else { - already_pending_write.emplace_back(cand); + already_pending_write.emplace_back(cand.first); LOG(spam, "GC(%s): failed to acquire write lock for '%s'; not including in GC set", - getBucket().toString().c_str(), cand.id.toString().c_str()); + getBucket().toString().c_str(), cand.first.toString().c_str()); } } for (auto& rm_entry : already_pending_write) { - _remove_candidate_set.erase(rm_entry); + _remove_candidates.erase(rm_entry); } - if (_remove_candidate_set.empty()) { + if (_remove_candidates.empty()) { update_last_gc_timestamp_in_db(); // Nothing to remove now, try again later. mark_operation_complete(); return; } LOG(debug, "GC(%s): Sending phase 2 GC with %zu entries (with acquired write locks). " "%zu documents had pending writes and could not be GCd at this time", - getBucket().toString().c_str(), _remove_candidate_set.size(), already_pending_write.size()); + getBucket().toString().c_str(), _remove_candidates.size(), already_pending_write.size()); transition_to(Phase::WriteRemovesPhase); send_current_phase_remove_locations(sender); } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h index adbbd210877..25308b0fb4b 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h @@ -2,11 +2,12 @@ #pragma once #include "idealstateoperation.h" +#include <vespa/document/base/documentid.h> #include <vespa/storage/bucketdb/bucketcopy.h> #include <vespa/storage/distributor/messagetracker.h> #include <vespa/storage/distributor/operation_sequencer.h> #include <vespa/persistence/spi/id_and_timestamp.h> -#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/hash_map.h> #include <vector> namespace storage::distributor { @@ -41,18 +42,22 @@ private: static const char* to_string(Phase phase) noexcept; - using RemoveCandidateSet = vespalib::hash_set<spi::IdAndTimestamp, spi::IdAndTimestamp::hash>; + struct DocIdHasher { + size_t operator()(const document::DocumentId& id) const noexcept { + return document::GlobalId::hash()(id.getGlobalId()); + } + }; + using RemoveCandidates = vespalib::hash_map<document::DocumentId, spi::Timestamp, DocIdHasher>; Phase _phase; uint32_t _cluster_state_version_at_phase1_start_time; - uint32_t _phase1_replies_received; - RemoveCandidateSet _remove_candidate_set; + RemoveCandidates _remove_candidates; std::vector<SequencingHandle> _gc_write_locks; std::vector<BucketCopy> _replica_info; uint32_t _max_documents_removed; bool _is_done; - static RemoveCandidateSet steal_selection_matches_as_set(api::RemoveLocationReply& reply); + static RemoveCandidates steal_selection_matches_as_candidates(api::RemoveLocationReply& reply); void send_current_phase_remove_locations(DistributorStripeMessageSender& sender); std::vector<spi::IdAndTimestamp> compile_phase_two_send_set() const; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp index 09018538f95..1ac452e11d4 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp @@ -2,6 +2,7 @@ #include "searchenvironment.h" #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.searchenvironment"); @@ -11,7 +12,7 @@ using vsm::VSMAdapter; namespace streaming { -__thread SearchEnvironment::EnvMap * SearchEnvironment::_localEnvMap=0; +__thread SearchEnvironment::EnvMap * SearchEnvironment::_localEnvMap = nullptr; SearchEnvironment::Env::Env(const vespalib::string & muffens, const config::ConfigUri & configUri, Fast_NormalizeWordFolder & wf) : _configId(configUri.getConfigId()), @@ -29,7 +30,6 @@ SearchEnvironment::Env::createKeySet(const vespalib::string & configId) config::ConfigKeySet set; set.add<vespa::config::search::vsm::VsmfieldsConfig, vespa::config::search::SummaryConfig, - vespa::config::search::SummarymapConfig, vespa::config::search::vsm::VsmsummaryConfig, vespa::config::search::summary::JuniperrcConfig, vespa::config::search::RankProfilesConfig>(configId); diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 9d3d390efe1..3a3f2279ad2 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -231,7 +231,7 @@ void DocsumFilter::init(const FieldMap & fieldMap, const FieldPathMapT & fieldPa const ResConfigEntry &entry = *resClass->GetEntry(i); const DocsumTools::FieldSpec & toolsSpec = inputSpecs[i]; _fields.push_back(DocsumFieldSpec(entry._type, toolsSpec.getCommand())); - LOG(debug, "About to prepare field spec for summary field '%s'", entry._bindname.c_str()); + LOG(debug, "About to prepare field spec for summary field '%s'", entry._name.c_str()); prepareFieldSpec(_fields.back(), toolsSpec, fieldMap, fieldPathMap); } assert(entryCnt == _fields.size()); diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index e220cf685e2..5b3ef733eec 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -5,6 +5,7 @@ #include "i_matching_elements_filler.h" #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchsummary/docsummary/keywordextractor.h> +#include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.vsm-adapter"); @@ -13,6 +14,8 @@ using search::docsummary::ResConfigEntry; using search::docsummary::KeywordExtractor; using search::MatchingElements; using config::ConfigSnapshot; +using vespa::config::search::SummaryConfig; +using vespa::config::search::summary::JuniperrcConfig; namespace vsm { @@ -37,12 +40,6 @@ void GetDocsumsStateCallback::FillRankFeatures(GetDocsumsState& state) } } -void GetDocsumsStateCallback::FillDocumentLocations(GetDocsumsState *state, IDocsumEnvironment * env) -{ - (void) state; - (void) env; -} - std::unique_ptr<MatchingElements> GetDocsumsStateCallback::fill_matching_elements(const search::MatchingElementsFields& fields) { @@ -91,16 +88,16 @@ DocsumTools::obtainFieldNames(const FastS_VsmsummaryHandle &cfg) { uint32_t defaultSummaryId = getResultConfig()->LookupResultClassId(cfg->outputclass); _resultClass = getResultConfig()->LookupResultClass(defaultSummaryId); - if (_resultClass != NULL) { + if (_resultClass != nullptr) { for (uint32_t i = 0; i < _resultClass->GetNumEntries(); ++i) { const ResConfigEntry * entry = _resultClass->GetEntry(i); - _fieldSpecs.push_back(FieldSpec()); - _fieldSpecs.back().setOutputName(entry->_bindname); + _fieldSpecs.emplace_back(); + _fieldSpecs.back().setOutputName(entry->_name); bool found = false; if (cfg) { // check if we have this summary field in the vsmsummary config for (uint32_t j = 0; j < cfg->fieldmap.size() && !found; ++j) { - if (entry->_bindname == cfg->fieldmap[j].summary.c_str()) { + if (entry->_name == cfg->fieldmap[j].summary.c_str()) { for (uint32_t k = 0; k < cfg->fieldmap[j].document.size(); ++k) { _fieldSpecs.back().getInputNames().push_back(cfg->fieldmap[j].document[k].field); } @@ -111,7 +108,7 @@ DocsumTools::obtainFieldNames(const FastS_VsmsummaryHandle &cfg) } if (!found) { // use yourself as input - _fieldSpecs.back().getInputNames().push_back(entry->_bindname); + _fieldSpecs.back().getInputNames().push_back(entry->_name); } } } else { @@ -127,7 +124,6 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) LOG(debug, "(re-)configure VSM (docsum tools)"); std::shared_ptr<SummaryConfig> summary(snapshot.getConfig<SummaryConfig>()); - std::shared_ptr<SummarymapConfig> summaryMap(snapshot.getConfig<SummarymapConfig>()); std::shared_ptr<VsmsummaryConfig> vsmSummary(snapshot.getConfig<VsmsummaryConfig>()); std::shared_ptr<JuniperrcConfig> juniperrc(snapshot.getConfig<JuniperrcConfig>()); @@ -137,7 +133,6 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) LOG(debug, "configureFields(): Size of cfg fieldspec: %zd", _fieldsCfg.get()->fieldspec.size()); // UlfC: debugging LOG(debug, "configureFields(): Size of cfg documenttype: %zd", _fieldsCfg.get()->documenttype.size()); // UlfC: debugging LOG(debug, "configureSummary(): Size of cfg classes: %zd", summary->classes.size()); // UlfC: debugging - LOG(debug, "configureSummaryMap(): Size of cfg override: %zd", summaryMap->override.size()); // UlfC: debugging LOG(debug, "configureVsmSummary(): Size of cfg fieldmap: %zd", vsmSummary->fieldmap.size()); // UlfC: debugging LOG(debug, "configureVsmSummary(): outputclass='%s'", vsmSummary->outputclass.c_str()); // UlfC: debugging diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h index d6c1f55d092..ca7ead1afff 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h @@ -5,7 +5,6 @@ #include <vespa/searchlib/query/base.h> #include <vespa/vsm/config/vsm-cfif.h> #include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> #include <vespa/searchlib/common/featureset.h> #include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> @@ -20,9 +19,6 @@ using search::docsummary::GetDocsumsState; using search::docsummary::IDocsumEnvironment; using search::docsummary::JuniperProperties; -using vespa::config::search::SummaryConfig; -using vespa::config::search::SummarymapConfig; -using vespa::config::search::summary::JuniperrcConfig; namespace config { class ConfigSnapshot; } namespace vsm { @@ -40,12 +36,11 @@ public: GetDocsumsStateCallback(); void FillSummaryFeatures(GetDocsumsState& state) override; void FillRankFeatures(GetDocsumsState& state) override; - virtual void FillDocumentLocations(GetDocsumsState * state, IDocsumEnvironment * env); - virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; + std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; } void setRankFeatures(const search::FeatureSet::SP & rf) { _rankFeatures = rf; } void set_matching_elements_filler(std::unique_ptr<IMatchingElementsFiller> matching_elements_filler); - ~GetDocsumsStateCallback(); + ~GetDocsumsStateCallback() override; }; class DocsumTools : public IDocsumEnvironment @@ -73,12 +68,12 @@ private: std::unique_ptr<juniper::Juniper> _juniper; const ResultClass * _resultClass; std::vector<FieldSpec> _fieldSpecs; - DocsumTools(const DocsumTools &); - DocsumTools &operator=(const DocsumTools &); public: DocsumTools(); - ~DocsumTools(); + DocsumTools(const DocsumTools &) = delete; + DocsumTools &operator=(const DocsumTools &) = delete; + ~DocsumTools() override; void set_writer(std::unique_ptr<DynamicDocsumWriter> writer); void setJuniper(std::unique_ptr<juniper::Juniper> juniper) { _juniper = std::move(juniper); } const ResultConfig *getResultConfig() const { return _writer->GetResultConfig(); } |