diff options
author | Harald Musum <musum@yahooinc.com> | 2021-09-21 09:05:17 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2021-09-21 09:05:17 +0200 |
commit | c8d5e3b339c57b0135f52aa3060ddd568d61f58b (patch) | |
tree | a74750ee9ea650687f45397a6c40c6c305c081fa /config | |
parent | 1aa004804ae69e180542efeb54204e10a52b81dd (diff) |
Use xxhash64 instead of md5 for config payload
Diffstat (limited to 'config')
27 files changed, 102 insertions, 103 deletions
diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index 273a3abd1cd..679d118d0bc 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -240,7 +240,7 @@ GetConfig::Main() printf("defNamespace %s\n", rKey.getDefNamespace().c_str()); printf("configID %s\n", rKey.getConfigId().c_str()); - printf("configMD5 %s\n", rState.md5.c_str()); + printf("configXxhash64 %s\n", rState.xxhash64.c_str()); printf("generation %" PRId64 "\n", rState.generation); printf("trace %s\n", response->getTrace().toString().c_str()); diff --git a/config/src/tests/configagent/configagent.cpp b/config/src/tests/configagent/configagent.cpp index d6766cce822..e4dc94fc2de 100644 --- a/config/src/tests/configagent/configagent.cpp +++ b/config/src/tests/configagent/configagent.cpp @@ -31,12 +31,12 @@ class MyConfigResponse : public ConfigResponse { public: MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool valid, int64_t timestamp, - const vespalib::string & md5, const std::string & errorMsg, int errorC0de, bool iserror) + const vespalib::string & xxhash64, const std::string & errorMsg, int errorC0de, bool iserror) : _key(key), _value(value), _fillCalled(false), _valid(valid), - _state(md5, timestamp, false), + _state(xxhash64, timestamp, false), _errorMessage(errorMsg), _errorCode(errorC0de), _isError(iserror) @@ -64,9 +64,9 @@ public: Trace _trace; - static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value, uint64_t timestamp = 10, const vespalib::string & md5 = "a") + static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value, uint64_t timestamp = 10, const vespalib::string & xxhash64 = "a") { - return std::make_unique<MyConfigResponse>(key, value, true, timestamp, md5, "", 0, false); + return std::make_unique<MyConfigResponse>(key, value, true, timestamp, xxhash64, "", 0, false); } static ConfigResponse::UP createServerErrorResponse(const ConfigKey & key, const ConfigValue & value) @@ -114,11 +114,11 @@ private: }; -ConfigValue createValue(const std::string & myField, const std::string & md5) +ConfigValue createValue(const std::string & myField, const std::string & xxhash64) { std::vector< vespalib::string > lines; lines.push_back("myField \"" + myField + "\""); - return ConfigValue(lines, md5); + return ConfigValue(lines, xxhash64); } static TimingValues testTimingValues( @@ -139,7 +139,7 @@ TEST("require that agent returns correct values") { ASSERT_EQUAL(500u, handler.getTimeout()); ASSERT_EQUAL(0u, handler.getWaitTime()); ConfigState cs; - ASSERT_EQUAL(cs.md5, handler.getConfigState().md5); + ASSERT_EQUAL(cs.xxhash64, handler.getConfigState().xxhash64); ASSERT_EQUAL(cs.generation, handler.getConfigState().generation); ASSERT_EQUAL(cs.applyOnRestart, handler.getConfigState().applyOnRestart); } @@ -167,7 +167,7 @@ TEST("require that important(the change) request is delivered to holder even if FRTConfigAgent handler(latch, testTimingValues); handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue1, 1, testValue1.getMd5())); + MyConfigResponse::createOKResponse(testKey, testValue1, 1, testValue1.getXxhash64())); ASSERT_TRUE(latch->poll()); ConfigUpdate::UP update(latch->provide()); ASSERT_TRUE(update); @@ -176,9 +176,9 @@ TEST("require that important(the change) request is delivered to holder even if ASSERT_EQUAL("l33t", cfg.myField); handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue2, 2, testValue2.getMd5())); + MyConfigResponse::createOKResponse(testKey, testValue2, 2, testValue2.getXxhash64())); handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue2, 3, testValue2.getMd5())); + MyConfigResponse::createOKResponse(testKey, testValue2, 3, testValue2.getXxhash64())); ASSERT_TRUE(latch->poll()); update = latch->provide(); ASSERT_TRUE(update); diff --git a/config/src/tests/failover/failover.cpp b/config/src/tests/failover/failover.cpp index 2e039081716..cb8b7121b02 100644 --- a/config/src/tests/failover/failover.cpp +++ b/config/src/tests/failover/failover.cpp @@ -60,7 +60,7 @@ struct RPCServer : public FRT_Invokable { info.setString("uncompressedSize", "0"); root.setString(RESPONSE_CONFIGID, "myId"); root.setString(RESPONSE_CLIENT_HOSTNAME, "myhost"); - root.setString(RESPONSE_CONFIG_MD5, "md5"); + root.setString(RESPONSE_CONFIG_XXHASH64, "xxhash64"); root.setLong(RESPONSE_CONFIG_GENERATION, gen); root.setObject(RESPONSE_TRACE); Slime payload; diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index cb09b8f7254..5af02f22ed2 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -72,7 +72,7 @@ namespace { FRT_RPCRequest * createOKResponse(const vespalib::string & defName="", const vespalib::string & defMd5="", const vespalib::string & configId="", - const vespalib::string & configMd5="", + const vespalib::string & configXxhash64="", int changed=0, long generation=0, const std::vector<vespalib::string> & payload = std::vector<vespalib::string>(), @@ -85,7 +85,7 @@ namespace { ret.AddString(""); ret.AddString(defMd5.c_str()); ret.AddString(configId.c_str()); - ret.AddString(configMd5.c_str()); + ret.AddString(configXxhash64.c_str()); ret.AddInt32(changed); ret.AddInt64(generation); FRT_StringValue * payload_arr = ret.AddStringArray(payload.size()); @@ -268,16 +268,16 @@ TEST_FF("require that request is config task is scheduled", SourceFixture(), FRT TEST("require that v3 request is correctly initialized") { ConnectionMock conn; ConfigKey key = ConfigKey::create<MyConfig>("foobi"); - vespalib::string md5 = "mymd5"; + vespalib::string xxhash64 = "myxxhash64"; int64_t currentGeneration = 3; vespalib::string hostName = "myhost"; int64_t timeout = 3000; Trace traceIn(3); traceIn.trace(2, "Hei"); - FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, hostName, + FRTConfigRequestV3 v3req(&conn, key, xxhash64, currentGeneration, hostName, timeout, traceIn, VespaVersion::fromString("1.2.3"), CompressionType::LZ4); - ASSERT_TRUE(v3req.verifyState(ConfigState(md5, 3, false))); - ASSERT_FALSE(v3req.verifyState(ConfigState(md5, 2, false))); + ASSERT_TRUE(v3req.verifyState(ConfigState(xxhash64, 3, false))); + ASSERT_FALSE(v3req.verifyState(ConfigState(xxhash64, 2, false))); ASSERT_FALSE(v3req.verifyState(ConfigState("xxx", 3, false))); ASSERT_FALSE(v3req.verifyState(ConfigState("xxx", 2, false))); @@ -297,7 +297,7 @@ TEST("require that v3 request is correctly initialized") { EXPECT_EQUAL(key.getConfigId(), root[REQUEST_CLIENT_CONFIGID].asString().make_string()); EXPECT_EQUAL(hostName, root[REQUEST_CLIENT_HOSTNAME].asString().make_string()); EXPECT_EQUAL(currentGeneration, root[REQUEST_CURRENT_GENERATION].asLong()); - EXPECT_EQUAL(md5, root[REQUEST_CONFIG_MD5].asString().make_string()); + EXPECT_EQUAL(xxhash64, root[REQUEST_CONFIG_XXHASH64].asString().make_string()); EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); EXPECT_EQUAL("LZ4", root[REQUEST_COMPRESSION_TYPE].asString().make_string()); EXPECT_EQUAL(root[REQUEST_VESPA_VERSION].asString().make_string(), "1.2.3"); @@ -322,7 +322,7 @@ struct V3RequestFixture { Cursor & root; FRT_RPCRequest * req; ConfigKey key; - vespalib::string md5; + vespalib::string xxhash64; int64_t generation; vespalib::string hostname; Trace traceIn; @@ -333,7 +333,7 @@ struct V3RequestFixture { root(slime.setObject()), req(conn.allocRPCRequest()), key(ConfigKey::create<BarConfig>("foobi")), - md5("mymd5"), + xxhash64("myxxhash64"), generation(3), hostname("myhhost"), traceIn(3) @@ -345,7 +345,7 @@ struct V3RequestFixture { root.setString(RESPONSE_DEF_MD5, Memory(key.getDefMd5())); root.setString(RESPONSE_CONFIGID, Memory(key.getConfigId())); root.setString(RESPONSE_CLIENT_HOSTNAME, Memory(hostname)); - root.setString(RESPONSE_CONFIG_MD5, Memory(md5)); + root.setString(RESPONSE_CONFIG_XXHASH64, Memory(xxhash64)); root.setLong(RESPONSE_CONFIG_GENERATION, generation); traceIn.serialize(root.setObject(RESPONSE_TRACE)); } @@ -379,7 +379,7 @@ struct V3RequestFixture { EXPECT_EQUAL(key.getConfigId(), responseKey.getConfigId()); EXPECT_EQUAL(hostname, response.getHostName()); ConfigState state(response.getConfigState()); - EXPECT_EQUAL(md5, state.md5); + EXPECT_EQUAL(xxhash64, state.xxhash64); EXPECT_EQUAL(generation, state.generation); ConfigValue value(response.getValue()); BarConfig::UP config(value.newInstance<BarConfig>()); diff --git a/config/src/tests/misc/misc.cpp b/config/src/tests/misc/misc.cpp index 1a8b8a59ede..55040242732 100644 --- a/config/src/tests/misc/misc.cpp +++ b/config/src/tests/misc/misc.cpp @@ -15,11 +15,11 @@ TEST("requireThatConfigUpdateWorks") { std::vector<vespalib::string> lines; lines.push_back("foo"); - ConfigUpdate up(ConfigValue(lines, "mymd5"), true, 1337); + ConfigUpdate up(ConfigValue(lines, "myxxhash"), true, 1337); ASSERT_EQUAL(1337, up.getGeneration()); ASSERT_TRUE(up.hasChanged()); - ConfigUpdate up2(ConfigValue(lines, "mymd52"), false, 1338); + ConfigUpdate up2(ConfigValue(lines, "myxxhash2"), false, 1338); ASSERT_EQUAL(1338, up2.getGeneration()); ASSERT_FALSE(up2.hasChanged()); } @@ -27,22 +27,22 @@ TEST("requireThatConfigUpdateWorks") { TEST("requireThatConfigValueWorks") { std::vector<vespalib::string> lines; lines.push_back("myFooField \"bar\""); - ConfigValue v1(lines, calculateContentMd5(lines)); - ConfigValue v2(lines, calculateContentMd5(lines)); - ConfigValue v3(lines, calculateContentMd5(lines)); + ConfigValue v1(lines, calculateContentXxhash64(lines)); + ConfigValue v2(lines, calculateContentXxhash64(lines)); + ConfigValue v3(lines, calculateContentXxhash64(lines)); lines.push_back("myFooField \"bar2\""); - ConfigValue v4(lines, calculateContentMd5(lines)); + ConfigValue v4(lines, calculateContentXxhash64(lines)); ASSERT_TRUE(v1 == v2); ASSERT_TRUE(v1 == v3); } TEST("requireThatConfigKeyWorks") { - ConfigKey key1("id1", "def1", "namespace1", "md51"); - ConfigKey key2("id1", "def1", "namespace1", "md51"); - ConfigKey key3("id2", "def1", "namespace1", "md51"); - ConfigKey key4("id1", "def2", "namespace1", "md51"); - ConfigKey key5("id1", "def1", "namespace2", "md51"); - ConfigKey key6("id1", "def1", "namespace1", "md52"); // Special case. Md5 does not matter, so should be qual to key1 and key2 + ConfigKey key1("id1", "def1", "namespace1", "xxhash1"); + ConfigKey key2("id1", "def1", "namespace1", "xxhash1"); + ConfigKey key3("id2", "def1", "namespace1", "xxhash1"); + ConfigKey key4("id1", "def2", "namespace1", "xxhash1"); + ConfigKey key5("id1", "def1", "namespace2", "xxhash1"); + ConfigKey key6("id1", "def1", "namespace1", "xxhash2"); // Special case. xxhash64 does not matter, so should be qual to key1 and key2 ASSERT_TRUE(key1 == key2); @@ -111,7 +111,7 @@ TEST("require that config key initializes schema") std::vector<vespalib::string> schema; schema.push_back("foo"); schema.push_back("bar"); - ConfigKey key("id1", "def1", "namespace1", "md51", schema); + ConfigKey key("id1", "def1", "namespace1", "xxhash1", schema); const std::vector<vespalib::string> &vref(key.getDefSchema()); for (size_t i = 0; i < schema.size(); i++) { ASSERT_EQUAL(schema[i], vref[i]); @@ -131,6 +131,7 @@ TEST("require that error codes are correctly translated to strings") { ASSERT_CONFIG(ILLEGAL_CONFIGID); ASSERT_CONFIG(ILLEGAL_DEF_MD5); ASSERT_CONFIG(ILLEGAL_CONFIG_MD5); + ASSERT_CONFIG(ILLEGAL_CONFIG_MD5); ASSERT_CONFIG(ILLEGAL_TIMEOUT); ASSERT_CONFIG(ILLEGAL_TIMESTAMP); ASSERT_CONFIG(ILLEGAL_NAME_SPACE); diff --git a/config/src/tests/subscriber/subscriber.cpp b/config/src/tests/subscriber/subscriber.cpp index d58699f26e0..5c5714a867a 100644 --- a/config/src/tests/subscriber/subscriber.cpp +++ b/config/src/tests/subscriber/subscriber.cpp @@ -20,7 +20,7 @@ namespace { { std::vector< vespalib::string > lines; lines.push_back(value); - return ConfigValue(lines, calculateContentMd5(lines)); + return ConfigValue(lines, calculateContentXxhash64(lines)); } ConfigValue createFooValue(const std::string & value) diff --git a/config/src/vespa/config/common/configstate.h b/config/src/vespa/config/common/configstate.h index 2dbea3cc30f..143d77e4ab9 100644 --- a/config/src/vespa/config/common/configstate.h +++ b/config/src/vespa/config/common/configstate.h @@ -13,17 +13,17 @@ struct ConfigState { public: ConfigState() - : md5(""), + : xxhash64(""), generation(0), applyOnRestart(false) { } - ConfigState(const vespalib::string & md5sum, int64_t gen, bool _applyOnRestart) - : md5(md5sum), + ConfigState(const vespalib::string & xxhash, int64_t gen, bool _applyOnRestart) + : xxhash64(xxhash), generation(gen), applyOnRestart(_applyOnRestart) { } - vespalib::string md5; + vespalib::string xxhash64; int64_t generation; bool applyOnRestart; @@ -32,7 +32,7 @@ public: } bool hasDifferentPayloadFrom(const ConfigState & other) const { - return (md5.compare(other.md5) != 0); + return (xxhash64.compare(other.xxhash64) != 0); } }; diff --git a/config/src/vespa/config/common/configvalue.cpp b/config/src/vespa/config/common/configvalue.cpp index d5c0c2047df..ce6fd3f20da 100644 --- a/config/src/vespa/config/common/configvalue.cpp +++ b/config/src/vespa/config/common/configvalue.cpp @@ -6,22 +6,22 @@ namespace config { -ConfigValue::ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & md5sum) +ConfigValue::ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & xxhash) : _payload(), _lines(lines), - _md5sum(md5sum) + _xxhash64(xxhash) { } ConfigValue::ConfigValue() : _payload(), _lines(), - _md5sum() + _xxhash64() { } -ConfigValue::ConfigValue(PayloadPtr payload, const vespalib::string & md5) +ConfigValue::ConfigValue(PayloadPtr payload, const vespalib::string & xxhash) : _payload(std::move(payload)), _lines(), - _md5sum(md5) + _xxhash64(xxhash) { } ConfigValue::ConfigValue(const ConfigValue &) = default; @@ -32,7 +32,7 @@ ConfigValue::~ConfigValue() = default; int ConfigValue::operator==(const ConfigValue & rhs) const { - return (_md5sum.compare(rhs._md5sum) == 0); + return (_xxhash64.compare(rhs._xxhash64) == 0); } int diff --git a/config/src/vespa/config/common/configvalue.h b/config/src/vespa/config/common/configvalue.h index a0450328f30..f5bfae00c19 100644 --- a/config/src/vespa/config/common/configvalue.h +++ b/config/src/vespa/config/common/configvalue.h @@ -21,8 +21,8 @@ typedef std::shared_ptr<const protocol::Payload> PayloadPtr; class ConfigValue { public: typedef std::unique_ptr<ConfigValue> UP; - ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & md5sum); - ConfigValue(PayloadPtr data, const vespalib::string & md5sum); + ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & xxhash); + ConfigValue(PayloadPtr data, const vespalib::string & xxhash); ConfigValue(); ConfigValue(const ConfigValue &); ConfigValue & operator = (const ConfigValue &); @@ -36,7 +36,7 @@ public: const std::vector<vespalib::string> & getLines() const { return _lines; } std::vector<vespalib::string> getLegacyFormat() const; const vespalib::string asJson() const; - const vespalib::string getMd5() const { return _md5sum; } + const vespalib::string getXxhash64() const { return _xxhash64; } void serializeV1(::vespalib::slime::Cursor & cursor) const; void serializeV2(::vespalib::slime::Cursor & cursor) const; @@ -47,7 +47,7 @@ public: private: PayloadPtr _payload; std::vector<vespalib::string> _lines; - vespalib::string _md5sum; + vespalib::string _xxhash64; }; } //namespace config diff --git a/config/src/vespa/config/common/misc.cpp b/config/src/vespa/config/common/misc.cpp index 1040962e25c..e1b4390caf5 100644 --- a/config/src/vespa/config/common/misc.cpp +++ b/config/src/vespa/config/common/misc.cpp @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "misc.h" -#include <vespa/vespalib/util/md5.h> +#include <iostream> +#include <sstream> +#include <xxhash.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> @@ -12,12 +14,12 @@ using vespalib::Memory; namespace config { vespalib::string -calculateContentMd5(const std::vector<vespalib::string> & fileContents) +calculateContentXxhash64(const std::vector<vespalib::string> & fileContents) { vespalib::string normalizedLines; - int compact_md5size = 16; - unsigned char md5sum[compact_md5size]; + XXH64_hash_t xxhash64; vespalib::asciistream s; + std::stringstream ss; // remove comments, trailing spaces and empty lines // TODO: Remove multiple spaces and space before comma, like in Java @@ -30,16 +32,11 @@ calculateContentMd5(const std::vector<vespalib::string> & fileContents) normalizedLines += line; } } - fastc_md5sum((const unsigned char*)normalizedLines.c_str(), normalizedLines.size(), md5sum); + xxhash64 = XXH64((const unsigned char*)normalizedLines.c_str(), normalizedLines.size(), 0); - // convert to 32 character hex string - for (int i = 0; i < compact_md5size; i++) { - if (md5sum[i] < 16) { - s << "0"; - } - s << vespalib::hex << (int)md5sum[i]; - } - return s.str(); + ss << std::hex << xxhash64; + ss << std::endl; + return ss.str(); } bool diff --git a/config/src/vespa/config/common/misc.h b/config/src/vespa/config/common/misc.h index 0299ef001f1..089fd890224 100644 --- a/config/src/vespa/config/common/misc.h +++ b/config/src/vespa/config/common/misc.h @@ -19,7 +19,7 @@ namespace config { /** * Miscellaneous utility functions specific to config. */ -vespalib::string calculateContentMd5(const std::vector<vespalib::string> & fileContents); +vespalib::string calculateContentXxhash64(const std::vector<vespalib::string> & fileContents); bool isGenerationNewer(int64_t newGen, int64_t oldGen); diff --git a/config/src/vespa/config/file/filesource.cpp b/config/src/vespa/config/file/filesource.cpp index fcc69f68066..8e25d5ccffc 100644 --- a/config/src/vespa/config/file/filesource.cpp +++ b/config/src/vespa/config/file/filesource.cpp @@ -27,10 +27,10 @@ FileSource::getConfig() int64_t last = getLast(_fileName); if (last > _lastLoaded) { - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentMd5(lines)), true, _generation))); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentXxhash64(lines)), true, _generation))); _lastLoaded = last; } else { - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentMd5(lines)), false, _generation))); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentXxhash64(lines)), false, _generation))); } } diff --git a/config/src/vespa/config/frt/frtconfigagent.cpp b/config/src/vespa/config/frt/frtconfigagent.cpp index 0e5ee4f6b05..30112b677e1 100644 --- a/config/src/vespa/config/frt/frtconfigagent.cpp +++ b/config/src/vespa/config/frt/frtconfigagent.cpp @@ -27,7 +27,7 @@ FRTConfigAgent::handleResponse(const ConfigRequest & request, ConfigResponse::UP { if (LOG_WOULD_LOG(spam)) { const ConfigKey & key(request.getKey()); - LOG(spam, "current state for %s: generation %" PRId64 " md5 %s", key.toString().c_str(), _configState.generation, _configState.md5.c_str()); + LOG(spam, "current state for %s: generation %" PRId64 " xxhash64 %s", key.toString().c_str(), _configState.generation, _configState.xxhash64.c_str()); } if (response->validateResponse() && !response->isError()) { handleOKResponse(request, std::move(response)); @@ -57,12 +57,12 @@ void FRTConfigAgent::handleUpdatedGeneration(const ConfigKey & key, const ConfigState & newState, const ConfigValue & configValue) { if (LOG_WOULD_LOG(spam)) { - LOG(spam, "new generation %" PRId64 " md5:%s for key %s", newState.generation, newState.md5.c_str(), key.toString().c_str()); - LOG(spam, "Old config: md5:%s \n%s", _latest.getMd5().c_str(), _latest.asJson().c_str()); - LOG(spam, "New config: md5:%s \n%s", configValue.getMd5().c_str(), configValue.asJson().c_str()); + LOG(spam, "new generation %" PRId64 " xxhash64:%s for key %s", newState.generation, newState.xxhash64.c_str(), key.toString().c_str()); + LOG(spam, "Old config: xxhash64:%s \n%s", _latest.getXxhash64().c_str(), _latest.asJson().c_str()); + LOG(spam, "New config: xxhash64:%s \n%s", configValue.getXxhash64().c_str(), configValue.asJson().c_str()); } bool changed = false; - if (_latest.getMd5() != configValue.getMd5()) { + if (_latest.getXxhash64() != configValue.getXxhash64()) { _latest = configValue; changed = true; } diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp index fbc13556d14..3573b8c9cfe 100644 --- a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp @@ -24,7 +24,7 @@ FRTConfigRequest::UP FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, const ConfigState & state, int64_t serverTimeout) const { - return make_unique<FRTConfigRequestV3>(connection, key, state.md5, state.generation, _hostName, + return make_unique<FRTConfigRequestV3>(connection, key, state.xxhash64, state.generation, _hostName, serverTimeout, Trace(_traceLevel), _vespaVersion, _compressionType); } diff --git a/config/src/vespa/config/frt/frtconfigresponsev3.cpp b/config/src/vespa/config/frt/frtconfigresponsev3.cpp index 80cdf88a79a..351f0fc8136 100644 --- a/config/src/vespa/config/frt/frtconfigresponsev3.cpp +++ b/config/src/vespa/config/frt/frtconfigresponsev3.cpp @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtconfigresponsev3.h" #include "compressioninfo.h" #include <vespa/fnet/frt/values.h> @@ -53,7 +53,7 @@ FRTConfigResponseV3::getResponseTypes() const ConfigValue FRTConfigResponseV3::readConfigValue() const { - vespalib::string md5(_data->get()[RESPONSE_CONFIG_MD5].asString().make_string()); + vespalib::string xxhash64(_data->get()[RESPONSE_CONFIG_XXHASH64].asString().make_string()); CompressionInfo info; info.deserialize(_data->get()[RESPONSE_COMPRESSION_INFO]); auto slime = std::make_unique<Slime>(); @@ -67,9 +67,9 @@ FRTConfigResponseV3::readConfigValue() const } } if (LOG_WOULD_LOG(spam)) { - LOG(spam, "read config value md5(%s), payload size: %lu", md5.c_str(), data.memRef.size); + LOG(spam, "read config value xxhash64(%s), payload size: %lu", xxhash64.c_str(), data.memRef.size); } - return ConfigValue(std::make_shared<V3Payload>(std::move(slime)), md5); + return ConfigValue(std::make_shared<V3Payload>(std::move(slime)), xxhash64); } } // namespace config diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp index 4a1b0a6ddef..4236af45dad 100644 --- a/config/src/vespa/config/frt/protocol.cpp +++ b/config/src/vespa/config/frt/protocol.cpp @@ -25,6 +25,7 @@ const Memory REQUEST_DEF_CONTENT = "defContent"; const Memory REQUEST_CLIENT_CONFIGID = "configId"; const Memory REQUEST_CLIENT_HOSTNAME = "clientHostname"; const Memory REQUEST_CONFIG_MD5 = "configMD5"; +const Memory REQUEST_CONFIG_XXHASH64 = "configXxhash64"; const Memory REQUEST_CURRENT_GENERATION = "currentGeneration"; const Memory REQUEST_TIMEOUT = "timeout"; const Memory REQUEST_TRACE = "trace"; @@ -36,7 +37,7 @@ const Memory RESPONSE_DEF_NAMESPACE = "defNamespace"; const Memory RESPONSE_DEF_MD5 = "defMD5"; const Memory RESPONSE_CONFIGID = "configId"; const Memory RESPONSE_CLIENT_HOSTNAME = "clientHostname"; -const Memory RESPONSE_CONFIG_MD5 = "configMD5"; +const Memory RESPONSE_CONFIG_XXHASH64 = "confighash64"; const Memory RESPONSE_CONFIG_GENERATION = "generation"; const Memory RESPONSE_PAYLOAD = "payload"; const Memory RESPONSE_TRACE = "trace"; diff --git a/config/src/vespa/config/frt/protocol.h b/config/src/vespa/config/frt/protocol.h index 0ec16952701..ce09217b619 100644 --- a/config/src/vespa/config/frt/protocol.h +++ b/config/src/vespa/config/frt/protocol.h @@ -36,7 +36,7 @@ extern const vespalib::Memory REQUEST_DEF_MD5; extern const vespalib::Memory REQUEST_DEF_CONTENT; extern const vespalib::Memory REQUEST_CLIENT_CONFIGID; extern const vespalib::Memory REQUEST_CLIENT_HOSTNAME; -extern const vespalib::Memory REQUEST_CONFIG_MD5; +extern const vespalib::Memory REQUEST_CONFIG_XXHASH64; extern const vespalib::Memory REQUEST_CURRENT_GENERATION; extern const vespalib::Memory REQUEST_TIMEOUT; extern const vespalib::Memory REQUEST_TRACE; @@ -48,7 +48,7 @@ extern const vespalib::Memory RESPONSE_DEF_NAMESPACE; extern const vespalib::Memory RESPONSE_DEF_MD5; extern const vespalib::Memory RESPONSE_CONFIGID; extern const vespalib::Memory RESPONSE_CLIENT_HOSTNAME; -extern const vespalib::Memory RESPONSE_CONFIG_MD5; +extern const vespalib::Memory RESPONSE_CONFIG_XXHASH64; extern const vespalib::Memory RESPONSE_CONFIG_GENERATION; extern const vespalib::Memory RESPONSE_PAYLOAD; extern const vespalib::Memory RESPONSE_TRACE; diff --git a/config/src/vespa/config/frt/slimeconfigrequest.cpp b/config/src/vespa/config/frt/slimeconfigrequest.cpp index 8a6706974f6..3573f8c07b9 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.cpp +++ b/config/src/vespa/config/frt/slimeconfigrequest.cpp @@ -23,7 +23,7 @@ const vespalib::string SlimeConfigRequest::REQUEST_TYPES = "s"; SlimeConfigRequest::SlimeConfigRequest(Connection * connection, const ConfigKey & key, - const vespalib::string & configMd5, + const vespalib::string & configXxhash64, int64_t currentGeneration, const vespalib::string & hostName, int64_t serverTimeout, @@ -35,7 +35,7 @@ SlimeConfigRequest::SlimeConfigRequest(Connection * connection, : FRTConfigRequest(connection, key), _data() { - populateSlimeRequest(key, configMd5, currentGeneration, hostName, serverTimeout, trace, vespaVersion, protocolVersion, compressionType); + populateSlimeRequest(key, configXxhash64, currentGeneration, hostName, serverTimeout, trace, vespaVersion, protocolVersion, compressionType); _request->SetMethodName(methodName.c_str()); _parameters.AddString(createJsonFromSlime(_data).c_str()); } @@ -43,13 +43,13 @@ SlimeConfigRequest::SlimeConfigRequest(Connection * connection, bool SlimeConfigRequest::verifyState(const ConfigState & state) const { - return (state.md5.compare(_data[REQUEST_CONFIG_MD5].asString().make_stringref()) == 0 && + return (state.xxhash64.compare(_data[REQUEST_CONFIG_XXHASH64].asString().make_stringref()) == 0 && state.generation == _data[REQUEST_CURRENT_GENERATION].asLong()); } void SlimeConfigRequest::populateSlimeRequest(const ConfigKey & key, - const vespalib::string & configMd5, + const vespalib::string & configXxhash64, int64_t currentGeneration, const vespalib::string & hostName, int64_t serverTimeout, @@ -67,7 +67,7 @@ SlimeConfigRequest::populateSlimeRequest(const ConfigKey & key, def.serialize(root.setArray(REQUEST_DEF_CONTENT)); root.setString(REQUEST_CLIENT_CONFIGID, Memory(key.getConfigId())); root.setString(REQUEST_CLIENT_HOSTNAME, Memory(hostName)); - root.setString(REQUEST_CONFIG_MD5, Memory(configMd5)); + root.setString(REQUEST_CONFIG_XXHASH64, Memory(configXxhash64)); root.setLong(REQUEST_CURRENT_GENERATION, currentGeneration); root.setLong(REQUEST_TIMEOUT, serverTimeout); trace.serialize(root.setObject(REQUEST_TRACE)); diff --git a/config/src/vespa/config/frt/slimeconfigrequest.h b/config/src/vespa/config/frt/slimeconfigrequest.h index 6f2f42c98d5..d25530f815c 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.h +++ b/config/src/vespa/config/frt/slimeconfigrequest.h @@ -19,7 +19,7 @@ class SlimeConfigRequest : public FRTConfigRequest { public: SlimeConfigRequest(Connection * connection, const ConfigKey & key, - const vespalib::string & configMd5, + const vespalib::string & configXxhash64, int64_t currentGeneration, const vespalib::string & hostName, int64_t serverTimeout, @@ -33,7 +33,7 @@ public: virtual ConfigResponse::UP createResponse(FRT_RPCRequest * request) const override = 0; private: void populateSlimeRequest(const ConfigKey & key, - const vespalib::string & configMd5, + const vespalib::string & configXxhash64, int64_t currentGeneration, const vespalib::string & hostName, int64_t serverTimeout, diff --git a/config/src/vespa/config/frt/slimeconfigresponse.cpp b/config/src/vespa/config/frt/slimeconfigresponse.cpp index af224008d01..c1f3df7f674 100644 --- a/config/src/vespa/config/frt/slimeconfigresponse.cpp +++ b/config/src/vespa/config/frt/slimeconfigresponse.cpp @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "slimeconfigresponse.h" #include <vespa/config/common/misc.h> #include <vespa/fnet/frt/values.h> @@ -66,7 +66,7 @@ ConfigState SlimeConfigResponse::readState() const { const Slime & data(*_data); - return ConfigState(data.get()[RESPONSE_CONFIG_MD5].asString().make_string(), + return ConfigState(data.get()[RESPONSE_CONFIG_XXHASH64].asString().make_string(), data.get()[RESPONSE_CONFIG_GENERATION].asLong(), data.get()[RESPONSE_APPLY_ON_RESTART].asBool()); } diff --git a/config/src/vespa/config/print/asciiconfigreader.hpp b/config/src/vespa/config/print/asciiconfigreader.hpp index b4a25f4fbbd..9d142e83c00 100644 --- a/config/src/vespa/config/print/asciiconfigreader.hpp +++ b/config/src/vespa/config/print/asciiconfigreader.hpp @@ -27,7 +27,7 @@ AsciiConfigReader<ConfigType>::read() while (getline(_is, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); } } // namespace config diff --git a/config/src/vespa/config/print/fileconfigreader.hpp b/config/src/vespa/config/print/fileconfigreader.hpp index bfebb4b89b9..87a4e20866c 100644 --- a/config/src/vespa/config/print/fileconfigreader.hpp +++ b/config/src/vespa/config/print/fileconfigreader.hpp @@ -44,7 +44,7 @@ FileConfigReader<ConfigType>::read() for (std::getline(f, line); f; std::getline(f, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); } } // namespace config diff --git a/config/src/vespa/config/print/istreamconfigreader.hpp b/config/src/vespa/config/print/istreamconfigreader.hpp index 70f7b7e5f28..3a501d26245 100644 --- a/config/src/vespa/config/print/istreamconfigreader.hpp +++ b/config/src/vespa/config/print/istreamconfigreader.hpp @@ -31,7 +31,7 @@ IstreamConfigReader<ConfigType>::read() while (getline(_is, line)) { lines.push_back(line); } - return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentXxhash64(lines)))); } } // namespace config diff --git a/config/src/vespa/config/raw/rawsource.cpp b/config/src/vespa/config/raw/rawsource.cpp index 2b38cff214c..a6c4da0b4dd 100644 --- a/config/src/vespa/config/raw/rawsource.cpp +++ b/config/src/vespa/config/raw/rawsource.cpp @@ -16,7 +16,7 @@ void RawSource::getConfig() { auto lines(readConfig()); - ConfigValue value(lines, calculateContentMd5(lines)); + ConfigValue value(lines, calculateContentXxhash64(lines)); _holder->handle(ConfigUpdate::UP(new ConfigUpdate(value, true, 1))); } diff --git a/config/src/vespa/config/retriever/configsnapshot.cpp b/config/src/vespa/config/retriever/configsnapshot.cpp index 7aaf4ffd6f8..523247b7b62 100644 --- a/config/src/vespa/config/retriever/configsnapshot.cpp +++ b/config/src/vespa/config/retriever/configsnapshot.cpp @@ -150,7 +150,7 @@ void ConfigSnapshot::serializeValueV2(Cursor & cursor, const Value & value) const { cursor.setDouble("lastChanged", value.first); - cursor.setString("md5", Memory(value.second.getMd5())); + cursor.setString("xxhash64", Memory(value.second.getXxhash64())); value.second.serializeV2(cursor.setObject("payload")); } @@ -225,7 +225,7 @@ ConfigSnapshot::deserializeValueV1(Inspector & inspector) const for (size_t i = 0; i < s.children(); i++) { payload.push_back(s[i].asString().make_string()); } - return Value(lastChanged, ConfigValue(payload, calculateContentMd5(payload))); + return Value(lastChanged, ConfigValue(payload, calculateContentXxhash64(payload))); } namespace { diff --git a/config/src/vespa/config/set/configinstancesourcefactory.cpp b/config/src/vespa/config/set/configinstancesourcefactory.cpp index 7f963847997..5ebffd4c4e9 100644 --- a/config/src/vespa/config/set/configinstancesourcefactory.cpp +++ b/config/src/vespa/config/set/configinstancesourcefactory.cpp @@ -14,8 +14,8 @@ public: void close() override { } void getConfig() override { std::vector<vespalib::string> lines(_buffer.getlines()); - std::string currentMd5(config::calculateContentMd5(lines)); - _holder->handle(config::ConfigUpdate::UP(new config::ConfigUpdate(config::ConfigValue(lines, currentMd5), true, _generation))); + std::string currentXxhash64(config::calculateContentXxhash64(lines)); + _holder->handle(config::ConfigUpdate::UP(new config::ConfigUpdate(config::ConfigValue(lines, currentXxhash64), true, _generation))); } void reload(int64_t generation) override { _generation = generation; } diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp index e6ab890a9df..36c7d70e0d3 100644 --- a/config/src/vespa/config/set/configsetsource.cpp +++ b/config/src/vespa/config/set/configsetsource.cpp @@ -30,16 +30,16 @@ ConfigSetSource::getConfig() AsciiConfigWriter writer(ss); writer.write(*instance); std::vector<vespalib::string> lines(ss.getlines()); - std::string currentMd5(calculateContentMd5(lines)); + std::string currentXxhash64(calculateContentXxhash64(lines)); - if (isGenerationNewer(_generation, _lastState.generation) && currentMd5.compare(_lastState.md5) != 0) { + if (isGenerationNewer(_generation, _lastState.generation) && currentXxhash64.compare(_lastState.xxhash64) != 0) { LOG(debug, "New generation, updating"); - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentMd5), true, _generation))); - _lastState.md5 = currentMd5; + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentXxhash64), true, _generation))); + _lastState.xxhash64 = currentXxhash64; _lastState.generation = _generation; } else { LOG(debug, "Sending timestamp update"); - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentMd5), false, _generation))); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentXxhash64), false, _generation))); _lastState.generation = _generation; } } |