diff options
author | Harald Musum <musum@yahooinc.com> | 2021-09-09 09:29:53 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2021-09-09 09:29:53 +0200 |
commit | c8d830e1600da93112ae36e4f464ac9a9a9c8fc5 (patch) | |
tree | 8719d0e7231b75de0528e71d4d2406739cab8a2a /config | |
parent | ee747ac8a83f1a7be8862635d4b451d913f51fff (diff) |
Use both xxhash64 and md5 for config payload in requests and responses
Chacking of equal config with hasEqualConfig() still uses only md5
Diffstat (limited to 'config')
26 files changed, 404 insertions, 158 deletions
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index 53a9f3f9f94..b1939ac23c6 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -11,6 +11,7 @@ import com.yahoo.config.subscription.FileSource; import com.yahoo.config.subscription.JarSource; import com.yahoo.config.subscription.RawSource; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.DefContent; @@ -40,31 +41,31 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { private final T config; private final Long generation; private final boolean applyOnRestart; - private final PayloadChecksum payloadChecksum; + private final PayloadChecksums payloadChecksums; private ConfigState(boolean generationChanged, Long generation, boolean applyOnRestart, boolean configChanged, T config, - PayloadChecksum payloadChecksum) { + PayloadChecksums payloadChecksums) { this.generationChanged = generationChanged; this.generation = generation; this.applyOnRestart = applyOnRestart; this.configChanged = configChanged; this.config = config; - this.payloadChecksum = payloadChecksum; + this.payloadChecksums = payloadChecksums; } - private ConfigState(Long generation, T config, PayloadChecksum payloadChecksum) { - this(false, generation, false, false, config, payloadChecksum); + private ConfigState(Long generation, T config, PayloadChecksums payloadChecksums) { + this(false, generation, false, false, config, payloadChecksums); } private ConfigState() { - this(false, 0L, false, false, null, PayloadChecksum.empty()); + this(false, 0L, false, false, null, PayloadChecksums.empty()); } - private ConfigState<T> createUnchanged() { return new ConfigState<>(generation, config, payloadChecksum); } + private ConfigState<T> createUnchanged() { return new ConfigState<>(generation, config, payloadChecksums); } public boolean isConfigChanged() { return configChanged; } @@ -76,7 +77,7 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { public T getConfig() { return config; } - public PayloadChecksum getChecksum() { return payloadChecksum; } + public PayloadChecksums getChecksums() { return payloadChecksums; } } @@ -195,8 +196,8 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { return !prev.getGeneration().equals(requiredGen) || prev.isConfigChanged(); } - void setConfig(Long generation, boolean applyOnRestart, T config, PayloadChecksum payloadChecksum) { - this.config.set(new ConfigState<>(true, generation, applyOnRestart, true, config, payloadChecksum)); + void setConfig(Long generation, boolean applyOnRestart, T config, PayloadChecksums payloadChecksums) { + this.config.set(new ConfigState<>(true, generation, applyOnRestart, true, config, payloadChecksums)); } /** @@ -204,22 +205,22 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { */ protected void setConfigIncGen(T config) { ConfigState<T> prev = this.config.get(); - this.config.set(new ConfigState<>(true, prev.getGeneration() + 1, prev.applyOnRestart(), true, config, prev.payloadChecksum)); + this.config.set(new ConfigState<>(true, prev.getGeneration() + 1, prev.applyOnRestart(), true, config, prev.payloadChecksums)); } protected void setConfigIfChanged(T config) { ConfigState<T> prev = this.config.get(); - this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.applyOnRestart(), !config.equals(prev.getConfig()), config, prev.payloadChecksum)); + this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.applyOnRestart(), !config.equals(prev.getConfig()), config, prev.payloadChecksums)); } void setGeneration(Long generation) { ConfigState<T> prev = config.get(); - this.config.set(new ConfigState<>(true, generation, prev.applyOnRestart(), prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksum)); + this.config.set(new ConfigState<>(true, generation, prev.applyOnRestart(), prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksums)); } void setApplyOnRestart(boolean applyOnRestart) { ConfigState<T> prev = config.get(); - this.config.set(new ConfigState<>(prev.isGenerationChanged(), prev.getGeneration(), applyOnRestart, prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksum)); + this.config.set(new ConfigState<>(prev.isGenerationChanged(), prev.getGeneration(), applyOnRestart, prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksums)); } /** diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index 05d4a33c02a..e9a40539bf0 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -33,7 +33,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfi @Override protected void setNewConfig(JRTClientConfigRequest jrtReq) { RawConfig rawConfig = RawConfig.createFromResponseParameters(jrtReq); - setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), rawConfig, new PayloadChecksum(jrtReq.getNewConfigMd5())); + setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), rawConfig, jrtReq.getNewChecksums()); log.log(FINE, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index b06f986555c..bb1a154d5d0 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -111,7 +111,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } catch (IllegalArgumentException e) { badConfigE = e; } - setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), configInstance, new PayloadChecksum(jrtReq.getNewConfigMd5())); + setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), configInstance, jrtReq.getNewChecksums()); if (badConfigE != null) { throw new IllegalArgumentException("Bad config from jrt", badConfigE); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java index b7198b0c694..4a55c046f13 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java @@ -9,6 +9,7 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.PayloadChecksums; import java.io.IOException; import java.io.InputStreamReader; @@ -63,7 +64,7 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } catch (IOException e) { throw new ConfigurationRuntimeException(e); } - setConfig(0L, false, config, PayloadChecksum.empty()); + setConfig(0L, false, config, PayloadChecksums.empty()); try { jarFile.close(); } catch (IOException e) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java index bed7a0fa3c4..e9e7f3e7bce 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -7,9 +7,9 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Payload; -import com.yahoo.vespa.config.util.ConfigUtils; /** * For unit testing @@ -96,7 +96,7 @@ public class MockConnection implements ConnectionPool, Connection { JRTServerConfigRequestV3 jrtReq = JRTServerConfigRequestV3.createFromRequest(request); Payload payload = Payload.from(ConfigPayload.empty()); long generation = 1; - jrtReq.addOkResponse(payload, generation, false, ConfigUtils.getMd5(payload.getData())); + jrtReq.addOkResponse(payload, generation, false, PayloadChecksums.fromPayload(payload)); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java index 91b674da3d2..8d5e7839086 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java @@ -7,6 +7,7 @@ import com.yahoo.config.subscription.ConfigInterruptedException; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.PayloadChecksums; import java.util.Arrays; @@ -35,7 +36,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc if (payload == null) { payload = inputPayload; ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n"))); - setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()), PayloadChecksum.empty()); + setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()), PayloadChecksums.empty()); return true; } try { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java index 93b85aaabd0..17f486e15d1 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java +++ b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.subscription.impl; +package com.yahoo.vespa.config; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -17,13 +18,13 @@ public class PayloadChecksum { private final String checksum; private final Type type; - public PayloadChecksum(String checksum) { + public PayloadChecksum(String checksum, Type type) { this.checksum = checksum; - this.type = Type.MD5; + this.type = type; } - public static PayloadChecksum empty() { - return new PayloadChecksum(""); + public static PayloadChecksum empty(Type type) { + return new PayloadChecksum("", type); } public String asString() { return checksum; } @@ -45,4 +46,21 @@ public class PayloadChecksum { return m.matches(); } + @Override + public int hashCode() { + return Objects.hash(checksum, type); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PayloadChecksum that = (PayloadChecksum) o; + return Objects.equals(checksum, that.checksum) && type == that.type; + } + + @Override + public String toString() { + return type.name() + ":" + checksum; + } } diff --git a/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java new file mode 100644 index 00000000000..1558771bd58 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.vespa.config.protocol.Payload; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + +/** + * Checksums for config payload, typically 1 for each PayloadChecksum type (md5 and xxhash64). + * Initialized with empty checksum for each existing type. + * + * @author hmusum + */ +public class PayloadChecksums { + + private final Map<PayloadChecksum.Type, PayloadChecksum> checksums = new LinkedHashMap<>(); + + private PayloadChecksums() { + Arrays.stream(PayloadChecksum.Type.values()).forEach(type -> checksums.put(type, PayloadChecksum.empty(type))); + } + + public static PayloadChecksums empty() { return new PayloadChecksums(); } + + public static PayloadChecksums from(PayloadChecksum... checksums) { + PayloadChecksums payloadChecksums = new PayloadChecksums(); + Arrays.stream(checksums).forEach(payloadChecksums::add); + return payloadChecksums; + } + + public static PayloadChecksums from(String configMd5, String configXxhash64) { + return new PayloadChecksums() + .add(new PayloadChecksum(configMd5, MD5)) + .add(new PayloadChecksum(configXxhash64, XXHASH64)); + } + + public static PayloadChecksums fromPayload(Payload payload) { + return new PayloadChecksums() + .add(new PayloadChecksum(ConfigUtils.getMd5(payload.getData()), MD5)) + .add(new PayloadChecksum(ConfigUtils.getXxhash64(payload.getData()), XXHASH64)); + } + + private PayloadChecksums add(PayloadChecksum checksum) { + checksums.put(checksum.type(), checksum); + return this; + } + + public PayloadChecksum getForType(PayloadChecksum.Type type) { + return checksums.get(type); + } + + public boolean valid() { + return checksums.values().stream().allMatch(PayloadChecksum::valid); + } + + @Override + public String toString() { + return checksums.values().stream() + .map(checksum -> checksum.type().name() + ":" + checksum.asString()) + .collect(Collectors.joining(",")); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PayloadChecksums that = (PayloadChecksums) o; + return Objects.equals(checksums, that.checksums); + } + + @Override + public int hashCode() { + return Objects.hash(checksums); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java index 028e84e4c29..52f2e0798f6 100755 --- a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java +++ b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; import com.yahoo.config.ConfigInstance; @@ -28,7 +28,7 @@ public class RawConfig extends ConfigInstance { private final List<String> defContent; private final Payload payload; private final int errorCode; - private final String configMd5; + private final PayloadChecksums payloadChecksums; private final Optional<VespaVersion> vespaVersion; private long generation; private boolean applyOnRestart; @@ -40,29 +40,28 @@ public class RawConfig extends ConfigInstance { * @param defMd5 The md5 sum of the .def-file. */ public RawConfig(ConfigKey<?> key, String defMd5) { - this(key, defMd5, null, "", 0L, false, 0, Collections.emptyList(), Optional.empty()); + this(key, defMd5, null, PayloadChecksums.empty(), 0L, false, 0, Collections.emptyList(), Optional.empty()); } - public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation, - boolean applyOnRestart, List<String> defContent, - Optional<VespaVersion> vespaVersion) { - this(key, defMd5, payload, configMd5, generation, applyOnRestart, 0, defContent, vespaVersion); + public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, PayloadChecksums payloadChecksums, long generation, + boolean applyOnRestart, List<String> defContent, Optional<VespaVersion> vespaVersion) { + this(key, defMd5, payload, payloadChecksums, generation, applyOnRestart, 0, defContent, vespaVersion); } /** Copy constructor */ public RawConfig(RawConfig rawConfig) { - this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.configMd5, + this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.payloadChecksums, rawConfig.generation, rawConfig.applyOnRestart, rawConfig.errorCode, rawConfig.defContent, rawConfig.getVespaVersion()); } - public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation, + public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, PayloadChecksums payloadChecksums, long generation, boolean applyOnRestart, int errorCode, List<String> defContent, Optional<VespaVersion> vespaVersion) { this.key = key; this.defMd5 = ConfigUtils.getDefMd5FromRequest(defMd5, defContent); this.payload = payload; - this.configMd5 = configMd5; + this.payloadChecksums = payloadChecksums; this.generation = generation; this.applyOnRestart = applyOnRestart; this.errorCode = errorCode; @@ -79,7 +78,7 @@ public class RawConfig extends ConfigInstance { return new RawConfig(req.getConfigKey(), ConfigUtils.getDefMd5(req.getDefContent().asList()), req.getNewPayload(), - req.getNewConfigMd5(), + req.getRequestConfigChecksums(), req.getNewGeneration(), req.responseIsApplyOnRestart(), 0, @@ -96,7 +95,7 @@ public class RawConfig extends ConfigInstance { return new RawConfig(req.getConfigKey(), ConfigUtils.getDefMd5(req.getDefContent().asList()), Payload.from(new Utf8String(""), CompressionInfo.uncompressed()), - req.getRequestConfigMd5(), + req.getRequestConfigChecksums(), req.getRequestGeneration(), req.applyOnRestart(), 0, @@ -113,7 +112,7 @@ public class RawConfig extends ConfigInstance { public String getConfigId() { return key.getConfigId(); } - public String getConfigMd5() { return configMd5; } + public String getConfigMd5() { return payloadChecksums.getForType(PayloadChecksum.Type.MD5).asString(); } public String getDefMd5() { return defMd5; } @@ -133,6 +132,8 @@ public class RawConfig extends ConfigInstance { public Optional<VespaVersion> getVespaVersion() { return vespaVersion; } + public PayloadChecksums getPayloadChecksums() { return payloadChecksums; } + /** * Returns true if this config is equal to the config (same payload md5) in the given request. * @@ -174,11 +175,7 @@ public class RawConfig extends ConfigInstance { // while non-zero and equal error codes means configs are equal. if (isError()) return true; if (generation != other.generation) return false; - if (configMd5 != null) { - return configMd5.equals(other.configMd5); - } else { - return (other.configMd5 == null); - } + return (payloadChecksums.equals(((RawConfig) o).payloadChecksums)); } @Override @@ -194,9 +191,7 @@ public class RawConfig extends ConfigInstance { if (! isError()) { // configMd5 and generation only matter when the RawConfig is not an error. hash = 31 * hash + (int)(generation ^(generation >>>32)); - if (configMd5 != null) { - hash = 31 * hash + configMd5.hashCode(); - } + hash = 31 * hash + payloadChecksums.hashCode(); } return hash; } @@ -210,7 +205,7 @@ public class RawConfig extends ConfigInstance { sb.append(","); sb.append(key.getConfigId()); sb.append(","); - sb.append(getConfigMd5()); + sb.append(payloadChecksums); sb.append(","); sb.append(getGeneration()); sb.append(","); diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java index adb27f37413..345118b5fd4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.benchmark; import com.yahoo.collections.Tuple2; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.io.IOUtils; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; @@ -256,7 +257,7 @@ public class LoadTester { final long serverTimeout = 1000; return JRTClientConfigRequestV3.createWithParams(fullKey, DefContent.fromList(List.of(defContent.second)), - ConfigUtils.getCanonicalHostName(), "", + ConfigUtils.getCanonicalHostName(), PayloadChecksums.empty(), 0, serverTimeout, Trace.createDummy(), compressionType, Optional.empty()); } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java index f6fce56c227..98fc7f7a50e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.text.AbstractUtf8Array; import java.io.IOException; @@ -36,4 +37,6 @@ public interface ConfigResponse { CompressionInfo getCompressionInfo(); + PayloadChecksums getPayloadChecksums(); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java index 8535cc23225..9d3b87574f3 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java @@ -1,6 +1,8 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; + /** * Interface for config requests used by clients. * @@ -57,13 +59,20 @@ public interface JRTClientConfigRequest extends JRTConfigRequest { boolean responseIsApplyOnRestart(); /** - * Get the config md5 of the config returned by the server. Return an empty string if no response has been returned. + * Gets the config md5 of the config returned by the server. Returns an empty string if no response has been returned. * * @return a config md5. */ String getNewConfigMd5(); /** + * Gets the config checksums of the config returned by the server. Returns an empty string if no response has been returned. + * + * @return a config checksum. + */ + PayloadChecksums getNewChecksums(); + + /** * Test whether or not the response contains an updated config or not. * False if no response has been returned. * diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java index f5b558550e4..a6271b159ef 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java @@ -1,9 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.impl.ConfigSubscription; import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; import com.yahoo.slime.JsonFormat; @@ -38,7 +39,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { protected JRTClientConfigRequestV3(ConfigKey<?> key, String hostname, DefContent defSchema, - String configMd5, + PayloadChecksums payloadChecksums, long generation, long timeout, Trace trace, @@ -47,7 +48,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { Slime data = SlimeRequestData.encodeRequest(key, hostname, defSchema, - configMd5, + payloadChecksums, generation, timeout, trace, @@ -97,7 +98,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { return new JRTClientConfigRequestV3(getConfigKey(), getClientHostName(), getDefContent(), - isError() ? getRequestConfigMd5() : newConfMd5(), + isError() ? getRequestConfigChecksums() : newConfigChecksums(), isError() ? getRequestGeneration() : newGen(), timeout, Trace.createNew(), @@ -113,7 +114,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { return createWithParams(sub.getKey(), sub.getDefContent(), ConfigUtils.getCanonicalHostName(), - configState.getChecksum().asString(), + configState.getChecksums(), configState.getGeneration(), sub.timingValues().getSubscribeTimeout(), trace, @@ -128,34 +129,34 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { Optional<VespaVersion> vespaVersion) { String hostname = ConfigUtils.getCanonicalHostName(); return createWithParams(config.getKey(), - DefContent.fromList(config.getDefContent()), - hostname, - config.getConfigMd5(), - config.getGeneration(), - serverTimeout, - trace, - compressionType, - vespaVersion); + DefContent.fromList(config.getDefContent()), + hostname, + config.getPayloadChecksums(), + config.getGeneration(), + serverTimeout, + trace, + compressionType, + vespaVersion); } public static JRTClientConfigRequest createWithParams(ConfigKey<?> reqKey, DefContent defContent, String hostname, - String configMd5, + PayloadChecksums payloadChecksums, long generation, long serverTimeout, Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { return new JRTClientConfigRequestV3(reqKey, - hostname, - defContent, - configMd5, - generation, - serverTimeout, - trace, - compressionType, - vespaVersion); + hostname, + defContent, + payloadChecksums, + generation, + serverTimeout, + trace, + compressionType, + vespaVersion); } @Override @@ -177,7 +178,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { .append(",").append(getTimeout()) .append(",").append(getVespaVersion().map(VespaVersion::toString).orElse("")) .append("'\n"); - sb.append("response='").append(getNewConfigMd5()) + sb.append("response='").append(getNewChecksums()) .append(",").append(getNewGeneration()) .append(",").append(responseIsApplyOnRestart()) .append("'\n"); @@ -221,6 +222,12 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { return requestData.getTimeout(); } + protected PayloadChecksums newConfigChecksums() { + PayloadChecksums newChecksum = getNewChecksums(); + if (PayloadChecksums.empty().equals(newChecksum)) return getRequestConfigChecksums(); + return newChecksum; + } + protected String newConfMd5() { String newMd5 = getNewConfigMd5(); if ("".equals(newMd5)) return getRequestConfigMd5(); @@ -264,6 +271,10 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { return requestData.getRequestDefMd5(); } + public PayloadChecksums getRequestConfigChecksums() { + return requestData.getRequestConfigChecksums(); + } + @Override public boolean validateResponse() { if (request.isError()) { @@ -285,7 +296,12 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { @Override public String getNewConfigMd5() { - return responseData.getResponseConfigMd5(); + return responseData.getResponseConfigMd5().asString(); + } + + @Override + public PayloadChecksums getNewChecksums() { + return responseData.getResponseConfigChecksums(); } @Override diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java index 5b8f040b8e3..0fc751dc49f 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; @@ -44,6 +45,14 @@ public interface JRTConfigRequest { /** * Returns the generation of the requested config. If none has been given, 0 should be returned. + * Returns the checksum of the config request. Return an empty string if no response has been returned. + * + * @return a config checksum. + */ + PayloadChecksums getRequestConfigChecksums(); + + /** + * Returns the generation of the requested config. If none has been given, 0 should be returned. * * @return the generation in the request. */ diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java index abc2b0b4473..41106e138b7 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.GetConfigRequest; /** @@ -34,9 +35,9 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque * @param generation The config generation of the given payload. * @param applyOnRestart true if this config should only be applied on the next restart, * false if it should be applied right away - * @param configMd5 The md5sum of the given payload. + * @param payloadChecksums checksums of the given payload. */ - void addOkResponse(Payload payload, long generation, boolean applyOnRestart, String configMd5); + void addOkResponse(Payload payload, long generation, boolean applyOnRestart, PayloadChecksums payloadChecksums); /** * Get the current config md5 of the client config. diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index e0a5b23a6d4..fbb52e81679 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.protocol; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.DataValue; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; @@ -18,6 +19,9 @@ import java.nio.ByteBuffer; import java.util.Optional; import java.util.logging.Logger; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + /** * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields: * @@ -68,9 +72,9 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { } @Override - public void addOkResponse(Payload payload, long generation, boolean applyOnRestart, String configMd5) { + public void addOkResponse(Payload payload, long generation, boolean applyOnRestart, PayloadChecksums payloadChecksums) { this.applyOnRestart = applyOnRestart; - boolean changedConfig = !configMd5.equals(getRequestConfigMd5()); + boolean changedConfig = !payloadChecksums.equals(getRequestConfigChecksums()); boolean changedConfigAndNewGeneration = changedConfig && ConfigUtils.isGenerationNewer(generation, getRequestGeneration()); Payload responsePayload = payload.withCompression(getCompressionType()); ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(4096); @@ -78,7 +82,8 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { JsonGenerator jsonGenerator = createJsonGenerator(byteArrayOutputStream); jsonGenerator.writeStartObject(); addCommonReturnValues(jsonGenerator); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, configMd5); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, payloadChecksums.getForType(MD5).asString()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_XXHASH64, payloadChecksums.getForType(XXHASH64).asString()); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_GENERATION, generation); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_APPLY_ON_RESTART, applyOnRestart); jsonGenerator.writeObjectFieldStart(SlimeResponseData.RESPONSE_COMPRESSION_INFO); @@ -194,6 +199,8 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { @Override public String getRequestDefMd5() { return requestData.getRequestDefMd5(); } + public PayloadChecksums getRequestConfigChecksums() { return requestData.getRequestConfigChecksums(); } + private void addErrorResponse(int errorCode) { addErrorResponse(errorCode, ErrorCode.getName(errorCode)); } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index 7db15844e8b..9cd59798ecd 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; -import com.yahoo.config.subscription.impl.PayloadChecksum; +import com.yahoo.vespa.config.PayloadChecksum; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ErrorCode; @@ -9,6 +9,7 @@ import com.yahoo.vespa.config.ErrorCode; import java.util.logging.Logger; import java.util.regex.Matcher; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; import static java.util.logging.Level.INFO; /** @@ -29,12 +30,12 @@ public class RequestValidation { log.log(INFO, "Illegal name space '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_NAME_SPACE; } - if (!(new PayloadChecksum(request.getRequestDefMd5()).valid())) { + if (!(new PayloadChecksum(request.getRequestDefMd5(), MD5).valid())) { log.log(INFO, "Illegal checksum '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_DEF_MD5; // TODO: Use ILLEGAL_DEF_CHECKSUM } - if (!new PayloadChecksum(request.getRequestConfigMd5()).valid()) { - log.log(INFO, "Illegal config checksum '" + request.getRequestConfigMd5() + "'"); + if (! request.getRequestConfigChecksums().valid()) { + log.log(INFO, "Illegal config checksum '" + request.getRequestConfigChecksums() + "'"); return ErrorCode.ILLEGAL_CONFIG_MD5; // TODO: Use ILLEGAL_CONFIG_CHECKSUM } if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java index 1ccf6e367fc..8d08717942b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.text.AbstractUtf8Array; import com.yahoo.vespa.config.ConfigPayload; @@ -8,6 +9,8 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; + /** * Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface. * @@ -19,25 +22,42 @@ public class SlimeConfigResponse implements ConfigResponse { private final CompressionInfo compressionInfo; private final long generation; private final boolean applyOnRestart; - private final String configMd5; + private final PayloadChecksums payloadChecksums; + + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, + long generation, + boolean applyOnRestart, + PayloadChecksums payloadChecksums) { + AbstractUtf8Array data = payload.toUtf8Array(true); + return new SlimeConfigResponse(data, + generation, + applyOnRestart, + payloadChecksums, + CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); + } - public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, long generation, - boolean applyOnRestart, String configMd5) { + // TODO: Legacy method, remove when not used anymore + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, + long generation, + boolean applyOnRestart, + String configMd5) { AbstractUtf8Array data = payload.toUtf8Array(true); - return new SlimeConfigResponse(data, generation, applyOnRestart, - configMd5, + return new SlimeConfigResponse(data, + generation, + applyOnRestart, + PayloadChecksums.from(configMd5, ""), CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); } public SlimeConfigResponse(AbstractUtf8Array payload, long generation, boolean applyOnRestart, - String configMd5, + PayloadChecksums payloadChecksums, CompressionInfo compressionInfo) { this.payload = payload; this.generation = generation; this.applyOnRestart = applyOnRestart; - this.configMd5 = configMd5; + this.payloadChecksums = payloadChecksums; this.compressionInfo = compressionInfo; } @@ -56,7 +76,7 @@ public class SlimeConfigResponse implements ConfigResponse { @Override public String getConfigMd5() { - return configMd5; + return payloadChecksums.getForType(MD5).asString(); } @Override @@ -68,11 +88,13 @@ public class SlimeConfigResponse implements ConfigResponse { @Override public String toString() { return "generation=" + generation + "\n" + - "configmd5=" + configMd5 + "\n" + + "checksums=" + payloadChecksums + "\n" + Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED); } @Override public CompressionInfo getCompressionInfo() { return compressionInfo; } + @Override + public PayloadChecksums getPayloadChecksums() { return payloadChecksums; } } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java index b885623a78b..679298bac73 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksum; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -11,6 +13,9 @@ import com.yahoo.vespa.config.util.ConfigUtils; import java.util.Optional; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + /** * Contains slime request data objects. Provides methods for reading various fields from slime request data. * All data is read lazily. @@ -27,6 +32,7 @@ class SlimeRequestData { private static final String REQUEST_CLIENT_HOSTNAME = "clientHostname"; private static final String REQUEST_CURRENT_GENERATION = "currentGeneration"; private static final String REQUEST_CONFIG_MD5 = "configMD5"; + private static final String REQUEST_CONFIG_XXHASH64 = "configXxhash64"; private static final String REQUEST_TRACE = "trace"; private static final String REQUEST_TIMEOUT = "timeout"; private static final String REQUEST_DEF_MD5 = "defMD5"; @@ -79,6 +85,17 @@ class SlimeRequestData { String getRequestDefMd5() { return getRequestField(REQUEST_DEF_MD5).asString(); } + PayloadChecksum getRequestConfigXxhash64() { + Inspector xxhash64Field = getRequestField(REQUEST_CONFIG_XXHASH64); + return xxhash64Field.valid() + ? new PayloadChecksum(xxhash64Field.asString(), XXHASH64) + : PayloadChecksum.empty(XXHASH64); + } + + PayloadChecksums getRequestConfigChecksums() { + return PayloadChecksums.from(getRequestConfigMd5(), getRequestConfigXxhash64().asString()); + } + long getRequestGeneration() { return getRequestField(REQUEST_CURRENT_GENERATION).asLong(); } @@ -86,7 +103,7 @@ class SlimeRequestData { static Slime encodeRequest(ConfigKey<?> key, String hostname, DefContent defSchema, - String configMd5, + PayloadChecksums payloadChecksums, long generation, long timeout, Trace trace, @@ -102,7 +119,8 @@ class SlimeRequestData { request.setString(REQUEST_CLIENT_CONFIGID, key.getConfigId()); request.setString(REQUEST_CLIENT_HOSTNAME, hostname); defSchema.serialize(request.setArray(REQUEST_DEF_CONTENT)); - request.setString(REQUEST_CONFIG_MD5, configMd5); + request.setString(REQUEST_CONFIG_MD5, payloadChecksums.getForType(MD5).asString()); + request.setString(REQUEST_CONFIG_XXHASH64, payloadChecksums.getForType(XXHASH64).asString()); request.setLong(REQUEST_CURRENT_GENERATION, generation); request.setLong(REQUEST_TIMEOUT, timeout); request.setString(REQUEST_COMPRESSION_TYPE, compressionType.name()); diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java index cc98587456c..965622adaa5 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksum; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + /** * Contains response data for a slime response and methods for decoding the response data that * are common to all {@link Slime} based config requests. @@ -22,6 +27,7 @@ class SlimeResponseData { static final String RESPONSE_CLIENT_HOSTNAME = "clientHostname"; static final String RESPONSE_TRACE = "trace"; static final String RESPONSE_CONFIG_MD5 = "configMD5"; + static final String RESPONSE_CONFIG_XXHASH64 = "configXxhash64"; static final String RESPONSE_CONFIG_GENERATION = "generation"; static final String RESPONSE_APPLY_ON_RESTART = "applyOnRestart"; static final String RESPONSE_COMPRESSION_INFO = "compressionInfo"; @@ -58,9 +64,25 @@ class SlimeResponseData { return trace.valid() ? Trace.fromSlime(trace) : Trace.createDummy(); } - String getResponseConfigMd5() { - Inspector inspector = getResponseField(RESPONSE_CONFIG_MD5); - return inspector.valid() ? inspector.asString() : ""; + PayloadChecksum getResponseConfigMd5() { + Inspector md5Field = getResponseField(RESPONSE_CONFIG_MD5); + return md5Field.valid() + ? new PayloadChecksum(md5Field.asString(), MD5) + : PayloadChecksum.empty(MD5); + } + + PayloadChecksum getResponseConfigXxhash64() { + Inspector xxhash64Field = getResponseField(RESPONSE_CONFIG_XXHASH64); + return xxhash64Field.valid() + ? new PayloadChecksum(xxhash64Field.asString(), XXHASH64) + : PayloadChecksum.empty(XXHASH64); + } + + + PayloadChecksums getResponseConfigChecksums() { + PayloadChecksum responseConfigMd5 = getResponseConfigMd5(); + System.out.println(responseConfigMd5); + return PayloadChecksums.from(responseConfigMd5, getResponseConfigXxhash64()); } CompressionInfo getCompressionInfo() { diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java index a7fc8afcad9..329661bf7ae 100644 --- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -10,6 +10,8 @@ import com.yahoo.text.AbstractUtf8Array; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; import java.io.ByteArrayOutputStream; import java.io.File; @@ -94,6 +96,15 @@ public class ConfigUtils { } } + public static String getXxhash64(AbstractUtf8Array input) { + return getXxhash64(input.wrap()); + } + + public static String getXxhash64(ByteBuffer input) { + XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + return Long.toHexString(hasher.hash(input, 0)).toLowerCase(); + } + /** * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; * diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java index 7bdaeb7d367..919155a3944 100644 --- a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.ErrorType; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import org.junit.Test; @@ -139,7 +140,7 @@ public class JRTConfigRequesterTest { ConfigSubscriber subscriber = new ConfigSubscriber(); final TimingValues timingValues = getTestTimingValues(); JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); - sub.setConfig(1L, false, config(), PayloadChecksum.empty()); + sub.setConfig(1L, false, config(), PayloadChecksums.empty()); final MockConnection connection = new MockConnection(new ErrorResponseHandler()); JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); @@ -165,7 +166,7 @@ public class JRTConfigRequesterTest { ConfigSubscriber subscriber = new ConfigSubscriber(); final TimingValues timingValues = getTestTimingValues(); JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); - sub.setConfig(1L, false, config(), PayloadChecksum.empty()); + sub.setConfig(1L, false, config(), PayloadChecksums.empty()); final MockConnection connection = new MockConnection(new ErrorResponseHandler(com.yahoo.jrt.ErrorCode.TIMEOUT)); JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); @@ -179,7 +180,7 @@ public class JRTConfigRequesterTest { ConfigSubscriber subscriber = new ConfigSubscriber(); final TimingValues timingValues = getTestTimingValues(); JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); - sub.setConfig(1L, false, config(), PayloadChecksum.empty()); + sub.setConfig(1L, false, config(), PayloadChecksums.empty()); final MockConnection connection = new MockConnection(new ErrorResponseHandler(ErrorCode.UNKNOWN_DEFINITION)); JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java index e1d11f82eea..96a2d976f51 100644 --- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java @@ -1,8 +1,9 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; import com.yahoo.text.Utf8String; -import com.yahoo.vespa.config.protocol.*; +import com.yahoo.vespa.config.protocol.CompressionInfo; +import com.yahoo.vespa.config.protocol.Payload; import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.util.ConfigUtils; import org.junit.Test; @@ -11,6 +12,8 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; @@ -26,11 +29,11 @@ import static org.junit.Assert.assertThat; public class RawConfigTest { private static final ConfigKey<?> key = new ConfigKey<>("foo", "id", "bar"); - private static List<String> defContent = Arrays.asList("version=1", "anInt int"); + private static final List<String> defContent = Arrays.asList("version=1", "anInt int"); private static final String defMd5 = ConfigUtils.getDefMd5FromRequest("", defContent); - private static final String configMd5 = "012345"; - private static Payload payload = Payload.from(new Utf8String("anInt 1"), CompressionInfo.uncompressed()); - private static long generation = 1L; + private static final PayloadChecksums payloadChecksums = PayloadChecksums.from("012345", ""); + private static final Payload payload = Payload.from(new Utf8String("anInt 1"), CompressionInfo.uncompressed()); + private static final long generation = 1L; @Test public void basic() { @@ -47,7 +50,7 @@ public class RawConfigTest { RawConfig copiedConfig = new RawConfig(config); assertEquals(config, copiedConfig); - assertEquals("bar.foo," + defMd5 + ",id,,0,null", config.toString()); + assertEquals("bar.foo," + defMd5 + ",id,MD5:,XXHASH64:,0,null", config.toString()); assertEquals(Optional.empty(), config.getVespaVersion()); } @@ -61,14 +64,14 @@ public class RawConfigTest { assertThat(config.hashCode(), is(not(new RawConfig(key, "a").hashCode()))); // different def md5 // different generation - config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty()); - RawConfig config2 = new RawConfig(key, defMd5, payload, configMd5, 2L, false, defContent, Optional.empty()); + config = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, defContent, Optional.empty()); + RawConfig config2 = new RawConfig(key, defMd5, payload, payloadChecksums, 2L, false, defContent, Optional.empty()); assertThat(config, is(not(config2))); assertThat(config.hashCode(), is(not(config2.hashCode()))); // different config md5 and with vespa version final VespaVersion vespaVersion = VespaVersion.fromString("5.37.38"); - RawConfig config3 = new RawConfig(key, defMd5, payload, "9999", generation, false, defContent, Optional.of(vespaVersion)); + RawConfig config3 = new RawConfig(key, defMd5, payload, PayloadChecksums.from("9999", ""), generation, false, defContent, Optional.of(vespaVersion)); assertThat(config, is(not(config3))); assertThat(config.hashCode(), is(not(config3.hashCode()))); // Check that vespa version is set correctly @@ -82,42 +85,43 @@ public class RawConfigTest { assertNotEquals(config, key); // errors - RawConfig errorConfig1 = new RawConfig(key, defMd5, payload, configMd5, generation, false, 1, defContent, Optional.empty()); + RawConfig errorConfig1 = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, 1, defContent, Optional.empty()); assertThat(errorConfig1, is(errorConfig1)); assertThat(config, is(not(errorConfig1))); assertThat(config.hashCode(), is(not(errorConfig1.hashCode()))); assertThat(errorConfig1, is(errorConfig1)); - RawConfig errorConfig2 = new RawConfig(key, defMd5, payload, configMd5, generation, false, 2, defContent, Optional.empty()); + RawConfig errorConfig2 = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, 2, defContent, Optional.empty()); assertThat(errorConfig1, is(not(errorConfig2))); assertThat(errorConfig1.hashCode(), is(not(errorConfig2.hashCode()))); } @Test public void payload() { - RawConfig config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty()); - assertThat(config.getPayload(), is(payload)); - assertThat(config.getConfigMd5(), is(configMd5)); - assertThat(config.getGeneration(), is(generation)); - assertThat(config.getDefContent(), is(defContent)); + RawConfig config = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, defContent, Optional.empty()); + assertEquals(config.getPayload(), payload); + assertEquals(config.getConfigMd5(), payloadChecksums.getForType(MD5).asString()); + assertEquals(config.getPayloadChecksums().getForType(XXHASH64), payloadChecksums.getForType(XXHASH64)); + assertEquals(config.getGeneration(), generation); + assertEquals(config.getDefContent(), defContent); } @Test public void require_correct_defmd5() { final String defMd5ForEmptyDefContent = "d41d8cd98f00b204e9800998ecf8427e"; - RawConfig config = new RawConfig(key, null, payload, configMd5, generation, false, defContent, Optional.empty()); + RawConfig config = new RawConfig(key, null, payload, payloadChecksums, generation, false, defContent, Optional.empty()); assertThat(config.getDefMd5(), is(defMd5)); - config = new RawConfig(key, "", payload, configMd5, generation, false, defContent, Optional.empty()); + config = new RawConfig(key, "", payload, payloadChecksums, generation, false, defContent, Optional.empty()); assertThat(config.getDefMd5(), is(defMd5)); - config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty()); + config = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, defContent, Optional.empty()); assertThat(config.getDefMd5(), is(defMd5)); - config = new RawConfig(key, null, payload, configMd5, generation, false, null, Optional.empty()); + config = new RawConfig(key, null, payload, payloadChecksums, generation, false, null, Optional.empty()); assertNull(config.getDefMd5()); - config = new RawConfig(key, null, payload, configMd5, generation, false, List.of(""), Optional.empty()); + config = new RawConfig(key, null, payload, payloadChecksums, generation, false, List.of(""), Optional.empty()); assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); - config = new RawConfig(key, "", payload, configMd5, generation, false, null, Optional.empty()); + config = new RawConfig(key, "", payload, payloadChecksums, generation, false, null, Optional.empty()); assertThat(config.getDefMd5(), is("")); - config = new RawConfig(key, "", payload, configMd5, generation, false, List.of(""), Optional.empty()); + config = new RawConfig(key, "", payload, payloadChecksums, generation, false, List.of(""), Optional.empty()); assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); } diff --git a/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java b/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java index 8c11db15f7c..2b7fd9c350b 100644 --- a/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java @@ -1,10 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; -import com.yahoo.config.subscription.impl.PayloadChecksum; import com.yahoo.vespa.config.protocol.RequestValidation; import org.junit.Test; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -21,11 +22,12 @@ public class RequestValidationTest { @Test public void testVerifyDefMd5() { - assertTrue(PayloadChecksum.empty().valid()); - assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d3f62d7ff777fce6b").valid()); - assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d3f62d7ff777fce6B").valid()); - assertFalse(new PayloadChecksum("aaaaaaaaaaaaaaaaaa").valid()); - assertFalse(new PayloadChecksum("-8f0c01c7c3dcb8d3f62d7ff777fce6b").valid()); + assertTrue(PayloadChecksum.empty(MD5).valid()); + assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d3f62d7ff777fce6b", MD5).valid()); + assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d3f62d7ff777fce6B", MD5).valid()); + assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d", XXHASH64).valid()); + assertFalse(new PayloadChecksum("aaaaaaaaaaaaaaaaaa", MD5).valid()); + assertFalse(new PayloadChecksum("-8f0c01c7c3dcb8d3f62d7ff777fce6b", MD5).valid()); } @Test diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java index c53a6b5c73d..f00e95ccea2 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java @@ -1,6 +1,7 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.foo.SimpletypesConfig; import com.yahoo.text.AbstractUtf8Array; import com.yahoo.text.Utf8Array; @@ -12,6 +13,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -23,14 +26,20 @@ public class ConfigResponseTest { @Test public void require_that_slime_response_is_initialized() throws IOException { ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, 3, false, "mymd5"); + PayloadChecksums payloadChecksums = PayloadChecksums.fromPayload(Payload.from(configPayload)); + ConfigResponse response = + SlimeConfigResponse.fromConfigPayload(configPayload, + 3, + false, + payloadChecksums); ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.serialize(baos, CompressionType.UNCOMPRESSED); String payload = baos.toString(StandardCharsets.UTF_8); assertNotNull(payload); - assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", payload.toString()); - assertEquals(response.getGeneration(), 3L); - assertEquals(response.getConfigMd5(), "mymd5"); + assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", payload); + assertEquals(3L, response.getGeneration()); + assertEquals(payloadChecksums.getForType(MD5), response.getPayloadChecksums().getForType(MD5)); + assertEquals(payloadChecksums.getForType(XXHASH64), response.getPayloadChecksums().getForType(XXHASH64)); baos = new ByteArrayOutputStream(); response.serialize(baos, CompressionType.UNCOMPRESSED); @@ -42,7 +51,7 @@ public class ConfigResponseTest { ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); AbstractUtf8Array data = configPayload.toUtf8Array(true); Utf8Array bytes = new Utf8Array(new LZ4PayloadCompressor().compress(data.wrap())); - ConfigResponse response = new SlimeConfigResponse(bytes, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); + ConfigResponse response = new SlimeConfigResponse(bytes, 3, false, PayloadChecksums.empty(), CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.serialize(baos, CompressionType.UNCOMPRESSED); String payload = baos.toString(StandardCharsets.UTF_8); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java index d6ce246aa1f..5a3110c9221 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java @@ -8,6 +8,7 @@ import com.yahoo.config.subscription.impl.GenericConfigSubscriber; import com.yahoo.config.subscription.impl.JRTConfigRequester; import com.yahoo.config.subscription.impl.JRTConfigSubscription; import com.yahoo.config.subscription.impl.MockConnection; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.foo.SimpletypesConfig; import com.yahoo.jrt.Request; import com.yahoo.slime.Inspector; @@ -27,6 +28,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -53,7 +56,7 @@ public class JRTConfigRequestV3Test { private final long currentGeneration = 3; private final long timeout = 5000; private Trace trace ; - private final String configMd5 = ConfigUtils.getMd5(createPayload().getData()); + private final PayloadChecksums payloadChecksums = PayloadChecksums.fromPayload(createPayload()); private JRTClientConfigRequest clientReq; private JRTServerConfigRequest serverReq; @@ -79,8 +82,12 @@ public class JRTConfigRequestV3Test { @Test public void emptypayload() { ConfigPayload payload = ConfigPayload.empty(); - SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, 0, false, ConfigUtils.getMd5(payload)); - serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), false, response.getConfigMd5()); + PayloadChecksums payloadChecksums = PayloadChecksums.fromPayload(Payload.from(payload)); + SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, + 0, + false, + payloadChecksums); + serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), false, payloadChecksums); assertTrue(clientReq.validateResponse()); assertTrue(clientReq.hasUpdatedGeneration()); assertEquals("{}", clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString()); @@ -97,7 +104,8 @@ public class JRTConfigRequestV3Test { @Test public void next_request_when_error_is_correct() { - serverReq.addOkResponse(createPayload(), 999999, false, "newmd5"); + Payload payload = createPayload(); + serverReq.addOkResponse(payload, 999999, false, PayloadChecksums.fromPayload(payload)); serverReq.addErrorResponse(ErrorCode.OUTDATED_CONFIG, "error message"); JRTClientConfigRequest next = clientReq.nextRequest(6); // Should use config md5 and generation from the request, not the response @@ -111,7 +119,7 @@ public class JRTConfigRequestV3Test { Payload payload = createPayload("vale"); String md5 = ConfigUtils.getMd5(payload.getData()); long generation = 4L; - serverReq.addOkResponse(payload, generation, false, md5); + serverReq.addOkResponse(payload, generation, false, PayloadChecksums.fromPayload(payload)); assertTrue(clientReq.validateResponse()); assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is(payload.getData().toString())); assertThat(clientReq.getNewGeneration(), is(4L)); @@ -137,7 +145,7 @@ public class JRTConfigRequestV3Test { @Test public void generation_only_is_updated() { Payload payload = createPayload(); - serverReq.addOkResponse(payload, 4L, false, ConfigUtils.getMd5(payload.getData())); + serverReq.addOkResponse(payload, 4L, false, PayloadChecksums.fromPayload(payload)); boolean value = clientReq.validateResponse(); assertTrue(clientReq.errorMessage(), value); assertFalse(clientReq.hasUpdatedConfig()); @@ -147,7 +155,7 @@ public class JRTConfigRequestV3Test { @Test public void nothing_is_updated() { Payload payload = createPayload(); - serverReq.addOkResponse(payload, currentGeneration, false, configMd5); + serverReq.addOkResponse(payload, currentGeneration, false, payloadChecksums); assertTrue(clientReq.validateResponse()); assertFalse(clientReq.hasUpdatedConfig()); assertFalse(clientReq.hasUpdatedGeneration()); @@ -158,7 +166,7 @@ public class JRTConfigRequestV3Test { Payload payload = Payload.from(ConfigPayload.empty()); clientReq = createReq(payload); serverReq = createReq(clientReq.getRequest()); - serverReq.addOkResponse(payload, currentGeneration, false, ConfigUtils.getMd5(payload.getData())); + serverReq.addOkResponse(payload, currentGeneration, false, PayloadChecksums.fromPayload(payload)); boolean val = clientReq.validateResponse(); assertTrue(clientReq.errorMessage(), val); assertFalse(clientReq.hasUpdatedConfig()); @@ -195,7 +203,7 @@ public class JRTConfigRequestV3Test { @Override public void createResponse() { JRTServerConfigRequest serverRequest = createReq(request); - serverRequest.addOkResponse(createPayload(), currentGeneration, false, configMd5); + serverRequest.addOkResponse(createPayload(), currentGeneration, false, payloadChecksums); } }); @@ -206,8 +214,10 @@ public class JRTConfigRequestV3Test { assertTrue(sub.nextConfig(120_0000)); sub.close(); JRTClientConfigRequest nextReq = createReq(sub, Trace.createNew()); - assertThat(nextReq.getRequestConfigMd5(), is(sub.getConfigState().getChecksum().asString())); - assertThat(nextReq.getRequestGeneration(), is(currentGeneration)); + assertEquals(nextReq.getRequestConfigMd5(), sub.getConfigState().getChecksums().getForType(MD5).asString()); + assertEquals(nextReq.getRequestConfigChecksums().getForType(MD5).asString(), sub.getConfigState().getChecksums().getForType(MD5).asString()); + assertEquals(nextReq.getRequestConfigChecksums().getForType(XXHASH64).asString(), sub.getConfigState().getChecksums().getForType(XXHASH64).asString()); + assertEquals(nextReq.getRequestGeneration(), currentGeneration); } @Test @@ -225,12 +235,12 @@ public class JRTConfigRequestV3Test { @Test public void parameters_are_validated() { assertTrue(serverReq.validateParameters()); - assertValidationFail(createReq("35#$#!$@#", defNamespace, hostname, configId, configMd5, currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, "abcd.o#$*(!&$", hostname, configId, configMd5, currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, hostname, configId, "34", currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, hostname, configId, configMd5, -34, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, hostname, configId, configMd5, currentGeneration, -23, trace)); - assertValidationFail(createReq(defName, defNamespace, "", configId, configMd5, currentGeneration, timeout, trace)); + assertValidationFail(createReq("35#$#!$@#", defNamespace, hostname, configId, payloadChecksums, currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, "abcd.o#$*(!&$", hostname, configId, payloadChecksums, currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, hostname, configId, PayloadChecksums.from("abcd", "1234"), currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, hostname, configId, payloadChecksums, -34, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, hostname, configId, payloadChecksums, currentGeneration, -23, trace)); + assertValidationFail(createReq(defName, defNamespace, "", configId, payloadChecksums, currentGeneration, timeout, trace)); } private void assertValidationFail(JRTClientConfigRequest req) { @@ -248,12 +258,12 @@ public class JRTConfigRequestV3Test { } private JRTClientConfigRequest createReq(String defName, String defNamespace, - String hostname, String configId, String configMd5, + String hostname, String configId, PayloadChecksums payloadChecksums, long currentGeneration, long timeout, Trace trace) { return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace), DefContent.fromList(List.of(configDefinition)), hostname, - configMd5, + payloadChecksums, currentGeneration, timeout, trace, @@ -276,13 +286,13 @@ public class JRTConfigRequestV3Test { private JRTClientConfigRequest createReq() { trace = Trace.createNew(3, new ManualClock()); trace.trace(1, "hei"); - return createReq(defName, defNamespace, hostname, configId, configMd5, currentGeneration, timeout, trace); + return createReq(defName, defNamespace, hostname, configId, payloadChecksums, currentGeneration, timeout, trace); } private JRTClientConfigRequest createReq(Payload payload) { trace = Trace.createNew(3, new ManualClock()); trace.trace(1, "hei"); - return createReq(defName, defNamespace, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace); + return createReq(defName, defNamespace, hostname, configId, PayloadChecksums.fromPayload(payload), currentGeneration, timeout, trace); } private void request_is_parsed_base() { @@ -294,7 +304,8 @@ public class JRTConfigRequestV3Test { assertThat(serverReq.getDefContent().asStringArray(), is(configDefinition)); assertFalse(serverReq.noCache()); assertTrue(serverReq.getRequestTrace().toString().contains("hi")); - assertThat(serverReq.getRequestConfigMd5(), is(configMd5)); + assertThat(serverReq.getRequestConfigChecksums().getForType(MD5), is(payloadChecksums.getForType(MD5))); + assertThat(serverReq.getRequestConfigChecksums().getForType(XXHASH64), is(payloadChecksums.getForType(XXHASH64))); assertThat(serverReq.getRequestGeneration(), is(currentGeneration)); } |