diff options
20 files changed, 182 insertions, 82 deletions
diff --git a/Code-map.md b/Code-map.md index fba22bdf8e5..6eae723f31d 100644 --- a/Code-map.md +++ b/Code-map.md @@ -1,3 +1,5 @@ +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + # A map to the Vespa code base You want to get familiar with the Vespa code base but don't know where to start? diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index e3c10c1f8ce..77fb740043e 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -100,7 +100,7 @@ public class QueryPacket extends Packet { byte[] stripped = new byte[encodedBody.length - (ignoreableSize + getSessionKeySkipLength()) + utf8Summary.length + 1]; System.arraycopy(encodedBody, 0, stripped, 0, ignoreableOffset); - stripped[1] = (byte)(stripped[1] & 0x7f); // Ignor sessionKey feature flag + stripped[1] = (byte)(stripped[1] & 0x7f); // Ignore sessionKey feature flag System.arraycopy(utf8Summary, 0, stripped, ignoreableOffset, utf8Summary.length); stripped[ignoreableOffset + utf8Summary.length] = 0; diff --git a/document/src/tests/gtest_runner.cpp b/document/src/tests/gtest_runner.cpp index efe108c2042..fed042acceb 100644 --- a/document/src/tests/gtest_runner.cpp +++ b/document/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("document_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/metrics/src/tests/gtest_runner.cpp b/metrics/src/tests/gtest_runner.cpp index 35475ba19a9..2f0e0705792 100644 --- a/metrics/src/tests/gtest_runner.cpp +++ b/metrics/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("metrics_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp index 24c38ad118b..e1441bbf8b4 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/config-indexschema.h> #include <vespa/config-attributes.h> +#include <vespa/config-summary.h> #include <vespa/config-summarymap.h> #include <vespa/log/log.h> @@ -18,6 +19,8 @@ using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespa::config::search::IndexschemaConfig; using vespa::config::search::IndexschemaConfigBuilder; +using vespa::config::search::SummaryConfig; +using vespa::config::search::SummaryConfigBuilder; using vespa::config::search::SummarymapConfig; using vespa::config::search::SummarymapConfigBuilder; using search::attribute::Config; @@ -125,6 +128,24 @@ SummarymapConfig::Override make_attribute_combiner_override(const vespalib::stri return override; } +SummaryConfig::Classes::Fields make_summary_field(const vespalib::string &name, const vespalib::string &type) +{ + SummaryConfig::Classes::Fields field; + field.name = name; + field.type = type; + return field; +} + +SummaryConfig sCfg(std::vector<SummaryConfig::Classes::Fields> fields) +{ + SummaryConfigBuilder result; + result.classes.resize(1); + result.classes.back().id = 0; + result.classes.back().name = "default"; + result.classes.back().fields = std::move(fields); + return result; +} + SummarymapConfig smCfg(std::vector<SummarymapConfig::Override> overrides) { SummarymapConfigBuilder result; @@ -175,10 +196,10 @@ public: _oldIndexSchema.indexfield.emplace_back(field); } void setup(const AttributesConfig &oldAttributesConfig, const SummarymapConfig &oldSummarymapConfig, - const AttributesConfig &newAttributesConfig, const SummarymapConfig &newSummarymapConfig) { + const AttributesConfig &newAttributesConfig, const SummaryConfig &newSummaryConfig, const SummarymapConfig &newSummarymapConfig) { IndexschemaInspector indexschemaInspector(_oldIndexSchema); _delayer.setup(oldAttributesConfig, oldSummarymapConfig, - newAttributesConfig, newSummarymapConfig, + newAttributesConfig, newSummaryConfig, newSummarymapConfig, indexschemaInspector, _inspector); } void assertAttributeConfig(const std::vector<AttributesConfig::Attribute> &exp) @@ -195,14 +216,14 @@ public: TEST_F("require that empty config is OK", Fixture) { - f.setup(attrCfg({}), smCfg({}), attrCfg({}), smCfg({})); + f.setup(attrCfg({}), smCfg({}), attrCfg({}), sCfg({}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } TEST_F("require that simple attribute config is OK", Fixture) { - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -210,7 +231,7 @@ TEST_F("require that simple attribute config is OK", Fixture) TEST_F("require that adding attribute aspect is delayed if field type is unchanged", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -218,14 +239,14 @@ TEST_F("require that adding attribute aspect is delayed if field type is unchang TEST_F("require that adding attribute aspect is delayed if field type is unchanged, geopos override", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), smCfg({make_geopos_override("a")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_geopos_override("a")})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({make_geopos_override("a")})); } TEST_F("require that adding attribute is not delayed if field type changed", Fixture) { - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -233,22 +254,30 @@ TEST_F("require that adding attribute is not delayed if field type changed", Fix TEST_F("require that removing attribute aspect is delayed if field type is unchanged", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "integer")}), smCfg({})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } +TEST_F("require that summary map override is removed when summary aspect is removed, even if removing attribute aspect is delayed", Fixture) +{ + f.addFields({"a"}); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({}), smCfg({})); + TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); + TEST_DO(f.assertSummarymapConfig({})); +} + TEST_F("require that removing attribute aspect is delayed if field type is unchanged, gepos override", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_geopos_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_geopos_override("a")}), attrCfg({}), sCfg({}), smCfg({})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({})); } TEST_F("require that removing attribute aspect is not delayed if field type changed", Fixture) { - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "integer")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -257,7 +286,7 @@ TEST_F("require that removing attribute aspect is not delayed if also indexed", { f.addFields({"a"}); f.addOldIndexField("a"); - f.setup(attrCfg({make_string_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_string_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "string")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -265,7 +294,7 @@ TEST_F("require that removing attribute aspect is not delayed if also indexed", TEST_F("require that removing attribute aspect is not delayed for tensor", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), attrCfg({}), sCfg({make_summary_field("a", "tensor")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -273,7 +302,7 @@ TEST_F("require that removing attribute aspect is not delayed for tensor", Fixtu TEST_F("require that removing attribute aspect is not delayed for predicate", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_predicate_cfg(4)}), smCfg({}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_predicate_cfg(4)}), smCfg({}), attrCfg({}), sCfg({make_summary_field("a", "string")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -281,7 +310,7 @@ TEST_F("require that removing attribute aspect is not delayed for predicate", Fi TEST_F("require that removing attribute aspect is not delayed for reference", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_reference_cfg()}), smCfg({}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_reference_cfg()}), smCfg({}), attrCfg({}), sCfg({make_summary_field("a", "longstring")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -289,7 +318,7 @@ TEST_F("require that removing attribute aspect is not delayed for reference", Fi TEST_F("require that fast access flag change is delayed, false->true edge", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_int32_sv_cfg())}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_int32_sv_cfg())}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -297,7 +326,7 @@ TEST_F("require that fast access flag change is delayed, false->true edge", Fixt TEST_F("require that fast access flag change is delayed, true->false edge", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_fa(make_int32_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_fa(make_int32_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_fa(make_int32_sv_cfg())})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -305,7 +334,7 @@ TEST_F("require that fast access flag change is delayed, true->false edge", Fixt TEST_F("require that fast access flag change is delayed, false->true edge, tensor attr", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")}), attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), sCfg({make_summary_field("a", "tensor")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_tensor_cfg("tensor(x[10])")})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -313,7 +342,7 @@ TEST_F("require that fast access flag change is delayed, false->true edge, tenso TEST_F("require that fast access flag change is not delayed, true->false edge, tensor attr", Fixture) { f.addFields({"a"}); - f.setup(attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), smCfg({make_attribute_override("a")}), attrCfg({make_tensor_cfg("tensor(x[10])")}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), smCfg({make_attribute_override("a")}), attrCfg({make_tensor_cfg("tensor(x[10])")}), sCfg({make_summary_field("a", "tensor")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_tensor_cfg("tensor(x[10])")})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } @@ -322,14 +351,14 @@ TEST_F("require that fast access flag change is not delayed, true->false edge, s { f.addFields({"a"}); f.addOldIndexField("a"); - f.setup(attrCfg({make_fa(make_string_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_string_sv_cfg()}), smCfg({make_attribute_override("a")})); + f.setup(attrCfg({make_fa(make_string_sv_cfg())}), smCfg({make_attribute_override("a")}), attrCfg({make_string_sv_cfg()}), sCfg({make_summary_field("a", "tensor")}), smCfg({make_attribute_override("a")})); TEST_DO(f.assertAttributeConfig({make_string_sv_cfg()})); TEST_DO(f.assertSummarymapConfig({make_attribute_override("a")})); } TEST_F("require that adding attribute aspect to struct field is not delayed if field type is changed", Fixture) { - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring")}), smCfg({make_attribute_combiner_override("array")})); TEST_DO(f.assertAttributeConfig({make_int32_sv_cfg("array.a")})); TEST_DO(f.assertSummarymapConfig({make_attribute_combiner_override("array")})); } @@ -337,7 +366,7 @@ TEST_F("require that adding attribute aspect to struct field is not delayed if f TEST_F("require that adding attribute aspect to struct field is delayed if field type is unchanged", Fixture) { f.addFields({"array.a"}); - f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")})); + f.setup(attrCfg({}), smCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring")}), smCfg({make_attribute_combiner_override("array")})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } @@ -345,7 +374,7 @@ TEST_F("require that adding attribute aspect to struct field is delayed if field TEST_F("require that removing attribute aspect from struct field is not delayed", Fixture) { f.addFields({"array.a"}); - f.setup(attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")}), attrCfg({}), smCfg({})); + f.setup(attrCfg({make_int32_sv_cfg("array.a")}), smCfg({make_attribute_combiner_override("array")}), attrCfg({}), sCfg({make_summary_field("array", "jsonstring")}), smCfg({})); TEST_DO(f.assertAttributeConfig({})); TEST_DO(f.assertSummarymapConfig({})); } diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index df01ae0fc02..a2b824b88ba 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/server/documentdbconfig.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/config-summary.h> #include <vespa/config-summarymap.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> @@ -86,6 +87,19 @@ public: _builder.attributes(make_shared<AttributesConfig>(builder)); return *this; } + MyConfigBuilder &addSummary(bool hasField) { + SummaryConfigBuilder builder; + builder.classes.resize(1); + builder.classes.back().id = 0; + builder.classes.back().name = "default"; + if (hasField) { + builder.classes.back().fields.resize(1); + builder.classes.back().fields.back().name = "my_attribute"; + builder.classes.back().fields.back().type = "integer"; + } + _builder.summary(make_shared<SummaryConfig>(builder)); + return *this; + } MyConfigBuilder &addSummarymap() { SummarymapConfigBuilder builder; builder.override.resize(1); @@ -114,11 +128,12 @@ struct Fixture { replayCfg(), nullCfg() { - basicCfg = MyConfigBuilder(4, schema, repo).addAttribute().build(); + basicCfg = MyConfigBuilder(4, schema, repo).addAttribute().addSummary(true).build(); fullCfg = MyConfigBuilder(4, schema, repo).addAttribute(). addRankProfile(). addRankingConstant(). addImportedField(). + addSummary(true). addSummarymap(). build(); replayCfg = DocumentDBConfig::makeReplayConfig(fullCfg); @@ -152,11 +167,13 @@ struct DelayAttributeAspectFixture { addRankProfile(). addRankingConstant(). addImportedField(). + addSummary(true). addSummarymap(). build(); noAttrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(hasDocField)).addRankProfile(). addRankingConstant(). addImportedField(). + addSummary(hasDocField). build(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp index cf803ec0368..4cf2df97fd0 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/common/config_hash.hpp> #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/config-attributes.h> +#include <vespa/config-summary.h> #include <vespa/config-summarymap.h> using search::attribute::ConfigConverter; @@ -14,6 +15,7 @@ using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespa::config::search::SummarymapConfig; using vespa::config::search::SummarymapConfigBuilder; +using vespa::config::search::SummaryConfig; using search::attribute::BasicType; namespace proton { @@ -40,6 +42,30 @@ bool willTriggerReprocessOnAttributeAspectRemoval(const search::attribute::Confi return fastPartialUpdateAttribute(cfg) && !indexschemaInspector.isStringIndex(name) && !isStructFieldAttribute(name); } +class KnownSummaryFields +{ + vespalib::hash_set<vespalib::string> _fields; + +public: + KnownSummaryFields(const SummaryConfig &summaryConfig); + ~KnownSummaryFields(); + + bool known(const vespalib::string &fieldName) const { + return _fields.find(fieldName) != _fields.end(); + } +}; + +KnownSummaryFields::KnownSummaryFields(const SummaryConfig &summaryConfig) + : _fields() +{ + for (const auto &summaryClass : summaryConfig.classes) { + for (const auto &summaryField : summaryClass.fields) { + _fields.insert(summaryField.name); + } + } +} + +KnownSummaryFields::~KnownSummaryFields() = default; } @@ -135,12 +161,14 @@ void handleOldAttributes(const AttributesConfig &oldAttributesConfig, const AttributesConfig &newAttributesConfig, const SummarymapConfig &oldSummarymapConfig, + const SummaryConfig &newSummaryConfig, const IIndexschemaInspector &oldIndexschemaInspector, const IDocumentTypeInspector &inspector, AttributesConfigBuilder &attributesConfig, SummarymapConfigBuilder &summarymapConfig) { vespalib::hash_set<vespalib::string> delayed; + KnownSummaryFields knownSummaryFields(newSummaryConfig); AttributesConfigHash newAttrs(newAttributesConfig.attribute); for (const auto &oldAttr : oldAttributesConfig.attribute) { search::attribute::Config oldCfg = ConfigConverter::convert(oldAttr); @@ -159,7 +187,7 @@ handleOldAttributes(const AttributesConfig &oldAttributesConfig, for (const auto &override : oldSummarymapConfig.override) { if (override.command == "attribute") { auto itr = delayed.find(override.field); - if (itr != delayed.end()) { + if (itr != delayed.end() && knownSummaryFields.known(override.field)) { summarymapConfig.override.emplace_back(override); } } @@ -172,6 +200,7 @@ void AttributeAspectDelayer::setup(const AttributesConfig &oldAttributesConfig, const SummarymapConfig &oldSummarymapConfig, const AttributesConfig &newAttributesConfig, + const SummaryConfig &newSummaryConfig, const SummarymapConfig &newSummarymapConfig, const IIndexschemaInspector &oldIndexschemaInspector, const IDocumentTypeInspector &inspector) @@ -181,7 +210,7 @@ AttributeAspectDelayer::setup(const AttributesConfig &oldAttributesConfig, oldIndexschemaInspector, inspector, *_attributesConfig, *_summarymapConfig); handleOldAttributes(oldAttributesConfig, newAttributesConfig, - oldSummarymapConfig, + oldSummarymapConfig, newSummaryConfig, oldIndexschemaInspector, inspector, *_attributesConfig, *_summarymapConfig); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h index 3a2d1b5e0b1..abd3829f5dd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.h @@ -9,6 +9,7 @@ namespace vespa { namespace config { namespace search { namespace internal { class InternalAttributesType; class InternalIndexschemaType; class InternalSummarymapType; +class InternalSummaryType; } } } } namespace proton { @@ -28,6 +29,7 @@ class AttributeAspectDelayer using IndexschemaConfig = const vespa::config::search::internal::InternalIndexschemaType; using SummarymapConfigBuilder = vespa::config::search::internal::InternalSummarymapType; using SummarymapConfig = const vespa::config::search::internal::InternalSummarymapType; + using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; std::shared_ptr<AttributesConfigBuilder> _attributesConfig; std::shared_ptr<SummarymapConfigBuilder> _summarymapConfig; @@ -43,6 +45,7 @@ public: void setup(const AttributesConfig &oldAttributesConfig, const SummarymapConfig &oldSummarymapConfig, const AttributesConfig &newAttributesConfig, + const SummaryConfig &newSummaryConfig, const SummarymapConfig &newSummarymapConfig, const IIndexschemaInspector &oldIndexschemaInspector, const IDocumentTypeInspector &inspector); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index fd1f9f1155d..712bc553d08 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -265,7 +265,7 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum DocumentTypeInspector inspector(*oldCfg.getDocumentType(), *n.getDocumentType()); IndexschemaInspector oldIndexschemaInspector(oldCfg.getIndexschemaConfig()); attributeAspectDelayer.setup(oldCfg.getAttributesConfig(), oldCfg.getSummarymapConfig(), - n.getAttributesConfig(), n.getSummarymapConfig(), + n.getAttributesConfig(), n.getSummaryConfig(), n.getSummarymapConfig(), oldIndexschemaInspector, inspector); bool delayedAttributeAspects = (n.getAttributesConfig() != *attributeAspectDelayer.getAttributesConfig()) || (n.getSummarymapConfig() != *attributeAspectDelayer.getSummarymapConfig()); diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index 2de71dc3e1b..4a515cf3b19 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -58,6 +58,10 @@ public: _importedFields = importedFields_in; return *this; } + DocumentDBConfigBuilder &summary(const DocumentDBConfig::SummaryConfigSP &summary_in) { + _summary = summary_in; + return *this; + } DocumentDBConfigBuilder &summarymap(const DocumentDBConfig::SummarymapConfigSP &summarymap_in) { _summarymap = summarymap_in; return *this; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java index 88aefe42a14..ede11c48de4 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java @@ -22,6 +22,7 @@ import static com.yahoo.yolean.Exceptions.uncheck; * @author hakonhall */ public class StateV1HealthClient implements AutoCloseable { + private static final long MAX_CONTENT_LENGTH = 1L << 20; // 1 MB private static final ObjectMapper MAPPER = new ObjectMapper(); private static final Logger logger = Logger.getLogger(StateV1HealthClient.class.getName()); @@ -71,4 +72,5 @@ public class StateV1HealthClient implements AutoCloseable { logger.log(LogLevel.WARNING, "Failed to close CloseableHttpClient", e); } } + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java index 7a6494e0122..b28ceeddae0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java @@ -13,6 +13,7 @@ import java.time.Duration; * @author hakon */ class StateV1HealthMonitor implements HealthMonitor { + private final StateV1HealthUpdater updater; private final Cancellable periodicExecution; @@ -30,4 +31,5 @@ class StateV1HealthMonitor implements HealthMonitor { public void close() { periodicExecution.cancel(); } + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java index 6a6768aa78b..985685ebb8d 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java @@ -13,6 +13,7 @@ import java.util.Optional; * @author hakonhall */ class StateV1HealthUpdater implements HealthUpdater { + private final String endpoint; private final StateV1HealthClient healthClient; @@ -56,4 +57,5 @@ class StateV1HealthUpdater implements HealthUpdater { public void close() { healthClient.close(); } + } diff --git a/staging_vespalib/src/vespa/vespalib/gtest/gtest.h b/staging_vespalib/src/vespa/vespalib/gtest/gtest.h new file mode 100644 index 00000000000..ed4e65b71cb --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/gtest/gtest.h @@ -0,0 +1,14 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <gtest/gtest.h> + +/** + * Macro for creating a main function that runs all gtests. + */ +#define GTEST_MAIN_RUN_ALL_TESTS \ +int \ +main(int argc, char* argv[]) \ +{ \ + ::testing::InitGoogleTest(&argc, argv); \ + return RUN_ALL_TESTS(); \ +} diff --git a/storage/src/tests/gtest_runner.cpp b/storage/src/tests/gtest_runner.cpp index f3400187ff4..762f6dba6db 100644 --- a/storage/src/tests/gtest_runner.cpp +++ b/storage/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("storage_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/storageapi/src/tests/gtest_runner.cpp b/storageapi/src/tests/gtest_runner.cpp index c37be7231ac..d499a54af50 100644 --- a/storageapi/src/tests/gtest_runner.cpp +++ b/storageapi/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("storageapi_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/storageframework/src/tests/gtest_runner.cpp b/storageframework/src/tests/gtest_runner.cpp index ce0310d7fa1..e9a1c9b1ed8 100644 --- a/storageframework/src/tests/gtest_runner.cpp +++ b/storageframework/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("storageframework_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/vdslib/src/tests/gtest_runner.cpp b/vdslib/src/tests/gtest_runner.cpp index d10f7182c07..e200813ef61 100644 --- a/vdslib/src/tests/gtest_runner.cpp +++ b/vdslib/src/tests/gtest_runner.cpp @@ -1,14 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("vdslib_gtest_runner"); -int -main(int argc, char* argv[]) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - +GTEST_MAIN_RUN_ALL_TESTS diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialsProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialsProvider.java index 28f028832b4..bd2f76bac52 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialsProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialsProvider.java @@ -15,8 +15,7 @@ import javax.net.ssl.SSLContext; import java.net.URI; import java.time.Duration; import java.time.Instant; -import java.util.Objects; -import java.util.logging.Logger; +import java.util.Optional; /** * Implementation of AWSCredentialsProvider using com.yahoo.vespa.athenz.client.zts.ZtsClient @@ -25,8 +24,6 @@ import java.util.logging.Logger; */ public class AwsCredentialsProvider implements AWSCredentialsProvider { - private static final Logger logger = Logger.getLogger(AwsCredentialsProvider.class.getName()); - private final static Duration MIN_EXPIRY = Duration.ofMinutes(5); private final AthenzDomain athenzDomain; private final AwsRole awsRole; @@ -72,8 +69,8 @@ public class AwsCredentialsProvider implements AWSCredentialsProvider { /* * Checks credential expiration, returns true if it will expipre in the next MIN_EXPIRY minutes */ - private static boolean shouldRefresh(AwsTemporaryCredentials credentials) { - Instant expiration = credentials.expiration(); - return Objects.isNull(expiration) || expiration.minus(MIN_EXPIRY).isAfter(Instant.now()); + static boolean shouldRefresh(AwsTemporaryCredentials credentials) { + Instant expiration = Optional.ofNullable(credentials).map(AwsTemporaryCredentials::expiration).orElse(Instant.EPOCH); + return Duration.between(Instant.now(), expiration).toMinutes() < MIN_EXPIRY.toMinutes(); } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialProviderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialProviderTest.java new file mode 100644 index 00000000000..d637dcae14c --- /dev/null +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/client/aws/AwsCredentialProviderTest.java @@ -0,0 +1,35 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.aws; + +import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; +import org.junit.Test; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class AwsCredentialProviderTest { + + @Test + public void refreshes_correctly() { + Clock clock = Clock.systemUTC(); + // Does not require refresh when expires in 10 minutes + assertFalse(AwsCredentialsProvider.shouldRefresh(getCredentials(clock.instant().plus(Duration.ofMinutes(10))))); + + // Requires refresh when expires in 3 minutes + assertTrue(AwsCredentialsProvider.shouldRefresh(getCredentials(clock.instant().plus(Duration.ofMinutes(3))))); + + // Requires refresh when expired + assertTrue(AwsCredentialsProvider.shouldRefresh(getCredentials(clock.instant().minus(Duration.ofMinutes(1))))); + + // Refreshes when no credentials provided + assertTrue(AwsCredentialsProvider.shouldRefresh(null)); + } + + private AwsTemporaryCredentials getCredentials(Instant expiration) { + return new AwsTemporaryCredentials("accesskey", "secretaccesskey", "sessionToken", expiration); + } +} |