diff options
Diffstat (limited to 'config/src/main')
21 files changed, 316 insertions, 97 deletions
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index 53a9f3f9f94..b1939ac23c6 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -11,6 +11,7 @@ import com.yahoo.config.subscription.FileSource; import com.yahoo.config.subscription.JarSource; import com.yahoo.config.subscription.RawSource; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.DefContent; @@ -40,31 +41,31 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { private final T config; private final Long generation; private final boolean applyOnRestart; - private final PayloadChecksum payloadChecksum; + private final PayloadChecksums payloadChecksums; private ConfigState(boolean generationChanged, Long generation, boolean applyOnRestart, boolean configChanged, T config, - PayloadChecksum payloadChecksum) { + PayloadChecksums payloadChecksums) { this.generationChanged = generationChanged; this.generation = generation; this.applyOnRestart = applyOnRestart; this.configChanged = configChanged; this.config = config; - this.payloadChecksum = payloadChecksum; + this.payloadChecksums = payloadChecksums; } - private ConfigState(Long generation, T config, PayloadChecksum payloadChecksum) { - this(false, generation, false, false, config, payloadChecksum); + private ConfigState(Long generation, T config, PayloadChecksums payloadChecksums) { + this(false, generation, false, false, config, payloadChecksums); } private ConfigState() { - this(false, 0L, false, false, null, PayloadChecksum.empty()); + this(false, 0L, false, false, null, PayloadChecksums.empty()); } - private ConfigState<T> createUnchanged() { return new ConfigState<>(generation, config, payloadChecksum); } + private ConfigState<T> createUnchanged() { return new ConfigState<>(generation, config, payloadChecksums); } public boolean isConfigChanged() { return configChanged; } @@ -76,7 +77,7 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { public T getConfig() { return config; } - public PayloadChecksum getChecksum() { return payloadChecksum; } + public PayloadChecksums getChecksums() { return payloadChecksums; } } @@ -195,8 +196,8 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { return !prev.getGeneration().equals(requiredGen) || prev.isConfigChanged(); } - void setConfig(Long generation, boolean applyOnRestart, T config, PayloadChecksum payloadChecksum) { - this.config.set(new ConfigState<>(true, generation, applyOnRestart, true, config, payloadChecksum)); + void setConfig(Long generation, boolean applyOnRestart, T config, PayloadChecksums payloadChecksums) { + this.config.set(new ConfigState<>(true, generation, applyOnRestart, true, config, payloadChecksums)); } /** @@ -204,22 +205,22 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { */ protected void setConfigIncGen(T config) { ConfigState<T> prev = this.config.get(); - this.config.set(new ConfigState<>(true, prev.getGeneration() + 1, prev.applyOnRestart(), true, config, prev.payloadChecksum)); + this.config.set(new ConfigState<>(true, prev.getGeneration() + 1, prev.applyOnRestart(), true, config, prev.payloadChecksums)); } protected void setConfigIfChanged(T config) { ConfigState<T> prev = this.config.get(); - this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.applyOnRestart(), !config.equals(prev.getConfig()), config, prev.payloadChecksum)); + this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.applyOnRestart(), !config.equals(prev.getConfig()), config, prev.payloadChecksums)); } void setGeneration(Long generation) { ConfigState<T> prev = config.get(); - this.config.set(new ConfigState<>(true, generation, prev.applyOnRestart(), prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksum)); + this.config.set(new ConfigState<>(true, generation, prev.applyOnRestart(), prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksums)); } void setApplyOnRestart(boolean applyOnRestart) { ConfigState<T> prev = config.get(); - this.config.set(new ConfigState<>(prev.isGenerationChanged(), prev.getGeneration(), applyOnRestart, prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksum)); + this.config.set(new ConfigState<>(prev.isGenerationChanged(), prev.getGeneration(), applyOnRestart, prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksums)); } /** diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index 05d4a33c02a..e9a40539bf0 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -33,7 +33,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfi @Override protected void setNewConfig(JRTClientConfigRequest jrtReq) { RawConfig rawConfig = RawConfig.createFromResponseParameters(jrtReq); - setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), rawConfig, new PayloadChecksum(jrtReq.getNewConfigMd5())); + setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), rawConfig, jrtReq.getNewChecksums()); log.log(FINE, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index b06f986555c..bb1a154d5d0 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -111,7 +111,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } catch (IllegalArgumentException e) { badConfigE = e; } - setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), configInstance, new PayloadChecksum(jrtReq.getNewConfigMd5())); + setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), configInstance, jrtReq.getNewChecksums()); if (badConfigE != null) { throw new IllegalArgumentException("Bad config from jrt", badConfigE); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java index b7198b0c694..4a55c046f13 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java @@ -9,6 +9,7 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.PayloadChecksums; import java.io.IOException; import java.io.InputStreamReader; @@ -63,7 +64,7 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } catch (IOException e) { throw new ConfigurationRuntimeException(e); } - setConfig(0L, false, config, PayloadChecksum.empty()); + setConfig(0L, false, config, PayloadChecksums.empty()); try { jarFile.close(); } catch (IOException e) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java index bed7a0fa3c4..e9e7f3e7bce 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -7,9 +7,9 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Payload; -import com.yahoo.vespa.config.util.ConfigUtils; /** * For unit testing @@ -96,7 +96,7 @@ public class MockConnection implements ConnectionPool, Connection { JRTServerConfigRequestV3 jrtReq = JRTServerConfigRequestV3.createFromRequest(request); Payload payload = Payload.from(ConfigPayload.empty()); long generation = 1; - jrtReq.addOkResponse(payload, generation, false, ConfigUtils.getMd5(payload.getData())); + jrtReq.addOkResponse(payload, generation, false, PayloadChecksums.fromPayload(payload)); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java index 91b674da3d2..8d5e7839086 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java @@ -7,6 +7,7 @@ import com.yahoo.config.subscription.ConfigInterruptedException; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.PayloadChecksums; import java.util.Arrays; @@ -35,7 +36,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc if (payload == null) { payload = inputPayload; ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n"))); - setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()), PayloadChecksum.empty()); + setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()), PayloadChecksums.empty()); return true; } try { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java index 93b85aaabd0..17f486e15d1 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java +++ b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.subscription.impl; +package com.yahoo.vespa.config; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -17,13 +18,13 @@ public class PayloadChecksum { private final String checksum; private final Type type; - public PayloadChecksum(String checksum) { + public PayloadChecksum(String checksum, Type type) { this.checksum = checksum; - this.type = Type.MD5; + this.type = type; } - public static PayloadChecksum empty() { - return new PayloadChecksum(""); + public static PayloadChecksum empty(Type type) { + return new PayloadChecksum("", type); } public String asString() { return checksum; } @@ -45,4 +46,21 @@ public class PayloadChecksum { return m.matches(); } + @Override + public int hashCode() { + return Objects.hash(checksum, type); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PayloadChecksum that = (PayloadChecksum) o; + return Objects.equals(checksum, that.checksum) && type == that.type; + } + + @Override + public String toString() { + return type.name() + ":" + checksum; + } } diff --git a/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java new file mode 100644 index 00000000000..1558771bd58 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.vespa.config.protocol.Payload; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + +/** + * Checksums for config payload, typically 1 for each PayloadChecksum type (md5 and xxhash64). + * Initialized with empty checksum for each existing type. + * + * @author hmusum + */ +public class PayloadChecksums { + + private final Map<PayloadChecksum.Type, PayloadChecksum> checksums = new LinkedHashMap<>(); + + private PayloadChecksums() { + Arrays.stream(PayloadChecksum.Type.values()).forEach(type -> checksums.put(type, PayloadChecksum.empty(type))); + } + + public static PayloadChecksums empty() { return new PayloadChecksums(); } + + public static PayloadChecksums from(PayloadChecksum... checksums) { + PayloadChecksums payloadChecksums = new PayloadChecksums(); + Arrays.stream(checksums).forEach(payloadChecksums::add); + return payloadChecksums; + } + + public static PayloadChecksums from(String configMd5, String configXxhash64) { + return new PayloadChecksums() + .add(new PayloadChecksum(configMd5, MD5)) + .add(new PayloadChecksum(configXxhash64, XXHASH64)); + } + + public static PayloadChecksums fromPayload(Payload payload) { + return new PayloadChecksums() + .add(new PayloadChecksum(ConfigUtils.getMd5(payload.getData()), MD5)) + .add(new PayloadChecksum(ConfigUtils.getXxhash64(payload.getData()), XXHASH64)); + } + + private PayloadChecksums add(PayloadChecksum checksum) { + checksums.put(checksum.type(), checksum); + return this; + } + + public PayloadChecksum getForType(PayloadChecksum.Type type) { + return checksums.get(type); + } + + public boolean valid() { + return checksums.values().stream().allMatch(PayloadChecksum::valid); + } + + @Override + public String toString() { + return checksums.values().stream() + .map(checksum -> checksum.type().name() + ":" + checksum.asString()) + .collect(Collectors.joining(",")); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PayloadChecksums that = (PayloadChecksums) o; + return Objects.equals(checksums, that.checksums); + } + + @Override + public int hashCode() { + return Objects.hash(checksums); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java index 028e84e4c29..52f2e0798f6 100755 --- a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java +++ b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; import com.yahoo.config.ConfigInstance; @@ -28,7 +28,7 @@ public class RawConfig extends ConfigInstance { private final List<String> defContent; private final Payload payload; private final int errorCode; - private final String configMd5; + private final PayloadChecksums payloadChecksums; private final Optional<VespaVersion> vespaVersion; private long generation; private boolean applyOnRestart; @@ -40,29 +40,28 @@ public class RawConfig extends ConfigInstance { * @param defMd5 The md5 sum of the .def-file. */ public RawConfig(ConfigKey<?> key, String defMd5) { - this(key, defMd5, null, "", 0L, false, 0, Collections.emptyList(), Optional.empty()); + this(key, defMd5, null, PayloadChecksums.empty(), 0L, false, 0, Collections.emptyList(), Optional.empty()); } - public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation, - boolean applyOnRestart, List<String> defContent, - Optional<VespaVersion> vespaVersion) { - this(key, defMd5, payload, configMd5, generation, applyOnRestart, 0, defContent, vespaVersion); + public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, PayloadChecksums payloadChecksums, long generation, + boolean applyOnRestart, List<String> defContent, Optional<VespaVersion> vespaVersion) { + this(key, defMd5, payload, payloadChecksums, generation, applyOnRestart, 0, defContent, vespaVersion); } /** Copy constructor */ public RawConfig(RawConfig rawConfig) { - this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.configMd5, + this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.payloadChecksums, rawConfig.generation, rawConfig.applyOnRestart, rawConfig.errorCode, rawConfig.defContent, rawConfig.getVespaVersion()); } - public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation, + public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, PayloadChecksums payloadChecksums, long generation, boolean applyOnRestart, int errorCode, List<String> defContent, Optional<VespaVersion> vespaVersion) { this.key = key; this.defMd5 = ConfigUtils.getDefMd5FromRequest(defMd5, defContent); this.payload = payload; - this.configMd5 = configMd5; + this.payloadChecksums = payloadChecksums; this.generation = generation; this.applyOnRestart = applyOnRestart; this.errorCode = errorCode; @@ -79,7 +78,7 @@ public class RawConfig extends ConfigInstance { return new RawConfig(req.getConfigKey(), ConfigUtils.getDefMd5(req.getDefContent().asList()), req.getNewPayload(), - req.getNewConfigMd5(), + req.getRequestConfigChecksums(), req.getNewGeneration(), req.responseIsApplyOnRestart(), 0, @@ -96,7 +95,7 @@ public class RawConfig extends ConfigInstance { return new RawConfig(req.getConfigKey(), ConfigUtils.getDefMd5(req.getDefContent().asList()), Payload.from(new Utf8String(""), CompressionInfo.uncompressed()), - req.getRequestConfigMd5(), + req.getRequestConfigChecksums(), req.getRequestGeneration(), req.applyOnRestart(), 0, @@ -113,7 +112,7 @@ public class RawConfig extends ConfigInstance { public String getConfigId() { return key.getConfigId(); } - public String getConfigMd5() { return configMd5; } + public String getConfigMd5() { return payloadChecksums.getForType(PayloadChecksum.Type.MD5).asString(); } public String getDefMd5() { return defMd5; } @@ -133,6 +132,8 @@ public class RawConfig extends ConfigInstance { public Optional<VespaVersion> getVespaVersion() { return vespaVersion; } + public PayloadChecksums getPayloadChecksums() { return payloadChecksums; } + /** * Returns true if this config is equal to the config (same payload md5) in the given request. * @@ -174,11 +175,7 @@ public class RawConfig extends ConfigInstance { // while non-zero and equal error codes means configs are equal. if (isError()) return true; if (generation != other.generation) return false; - if (configMd5 != null) { - return configMd5.equals(other.configMd5); - } else { - return (other.configMd5 == null); - } + return (payloadChecksums.equals(((RawConfig) o).payloadChecksums)); } @Override @@ -194,9 +191,7 @@ public class RawConfig extends ConfigInstance { if (! isError()) { // configMd5 and generation only matter when the RawConfig is not an error. hash = 31 * hash + (int)(generation ^(generation >>>32)); - if (configMd5 != null) { - hash = 31 * hash + configMd5.hashCode(); - } + hash = 31 * hash + payloadChecksums.hashCode(); } return hash; } @@ -210,7 +205,7 @@ public class RawConfig extends ConfigInstance { sb.append(","); sb.append(key.getConfigId()); sb.append(","); - sb.append(getConfigMd5()); + sb.append(payloadChecksums); sb.append(","); sb.append(getGeneration()); sb.append(","); diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java index adb27f37413..345118b5fd4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.benchmark; import com.yahoo.collections.Tuple2; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.io.IOUtils; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; @@ -256,7 +257,7 @@ public class LoadTester { final long serverTimeout = 1000; return JRTClientConfigRequestV3.createWithParams(fullKey, DefContent.fromList(List.of(defContent.second)), - ConfigUtils.getCanonicalHostName(), "", + ConfigUtils.getCanonicalHostName(), PayloadChecksums.empty(), 0, serverTimeout, Trace.createDummy(), compressionType, Optional.empty()); } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java index f6fce56c227..98fc7f7a50e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.text.AbstractUtf8Array; import java.io.IOException; @@ -36,4 +37,6 @@ public interface ConfigResponse { CompressionInfo getCompressionInfo(); + PayloadChecksums getPayloadChecksums(); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java index 8535cc23225..9d3b87574f3 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java @@ -1,6 +1,8 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; + /** * Interface for config requests used by clients. * @@ -57,13 +59,20 @@ public interface JRTClientConfigRequest extends JRTConfigRequest { boolean responseIsApplyOnRestart(); /** - * Get the config md5 of the config returned by the server. Return an empty string if no response has been returned. + * Gets the config md5 of the config returned by the server. Returns an empty string if no response has been returned. * * @return a config md5. */ String getNewConfigMd5(); /** + * Gets the config checksums of the config returned by the server. Returns an empty string if no response has been returned. + * + * @return a config checksum. + */ + PayloadChecksums getNewChecksums(); + + /** * Test whether or not the response contains an updated config or not. * False if no response has been returned. * diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java index f5b558550e4..a6271b159ef 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java @@ -1,9 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.impl.ConfigSubscription; import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; import com.yahoo.slime.JsonFormat; @@ -38,7 +39,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { protected JRTClientConfigRequestV3(ConfigKey<?> key, String hostname, DefContent defSchema, - String configMd5, + PayloadChecksums payloadChecksums, long generation, long timeout, Trace trace, @@ -47,7 +48,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { Slime data = SlimeRequestData.encodeRequest(key, hostname, defSchema, - configMd5, + payloadChecksums, generation, timeout, trace, @@ -97,7 +98,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { return new JRTClientConfigRequestV3(getConfigKey(), getClientHostName(), getDefContent(), - isError() ? getRequestConfigMd5() : newConfMd5(), + isError() ? getRequestConfigChecksums() : newConfigChecksums(), isError() ? getRequestGeneration() : newGen(), timeout, Trace.createNew(), @@ -113,7 +114,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { return createWithParams(sub.getKey(), sub.getDefContent(), ConfigUtils.getCanonicalHostName(), - configState.getChecksum().asString(), + configState.getChecksums(), configState.getGeneration(), sub.timingValues().getSubscribeTimeout(), trace, @@ -128,34 +129,34 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { Optional<VespaVersion> vespaVersion) { String hostname = ConfigUtils.getCanonicalHostName(); return createWithParams(config.getKey(), - DefContent.fromList(config.getDefContent()), - hostname, - config.getConfigMd5(), - config.getGeneration(), - serverTimeout, - trace, - compressionType, - vespaVersion); + DefContent.fromList(config.getDefContent()), + hostname, + config.getPayloadChecksums(), + config.getGeneration(), + serverTimeout, + trace, + compressionType, + vespaVersion); } public static JRTClientConfigRequest createWithParams(ConfigKey<?> reqKey, DefContent defContent, String hostname, - String configMd5, + PayloadChecksums payloadChecksums, long generation, long serverTimeout, Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { return new JRTClientConfigRequestV3(reqKey, - hostname, - defContent, - configMd5, - generation, - serverTimeout, - trace, - compressionType, - vespaVersion); + hostname, + defContent, + payloadChecksums, + generation, + serverTimeout, + trace, + compressionType, + vespaVersion); } @Override @@ -177,7 +178,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { .append(",").append(getTimeout()) .append(",").append(getVespaVersion().map(VespaVersion::toString).orElse("")) .append("'\n"); - sb.append("response='").append(getNewConfigMd5()) + sb.append("response='").append(getNewChecksums()) .append(",").append(getNewGeneration()) .append(",").append(responseIsApplyOnRestart()) .append("'\n"); @@ -221,6 +222,12 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { return requestData.getTimeout(); } + protected PayloadChecksums newConfigChecksums() { + PayloadChecksums newChecksum = getNewChecksums(); + if (PayloadChecksums.empty().equals(newChecksum)) return getRequestConfigChecksums(); + return newChecksum; + } + protected String newConfMd5() { String newMd5 = getNewConfigMd5(); if ("".equals(newMd5)) return getRequestConfigMd5(); @@ -264,6 +271,10 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { return requestData.getRequestDefMd5(); } + public PayloadChecksums getRequestConfigChecksums() { + return requestData.getRequestConfigChecksums(); + } + @Override public boolean validateResponse() { if (request.isError()) { @@ -285,7 +296,12 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { @Override public String getNewConfigMd5() { - return responseData.getResponseConfigMd5(); + return responseData.getResponseConfigMd5().asString(); + } + + @Override + public PayloadChecksums getNewChecksums() { + return responseData.getResponseConfigChecksums(); } @Override diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java index 5b8f040b8e3..0fc751dc49f 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; @@ -44,6 +45,14 @@ public interface JRTConfigRequest { /** * Returns the generation of the requested config. If none has been given, 0 should be returned. + * Returns the checksum of the config request. Return an empty string if no response has been returned. + * + * @return a config checksum. + */ + PayloadChecksums getRequestConfigChecksums(); + + /** + * Returns the generation of the requested config. If none has been given, 0 should be returned. * * @return the generation in the request. */ diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java index abc2b0b4473..41106e138b7 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.GetConfigRequest; /** @@ -34,9 +35,9 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque * @param generation The config generation of the given payload. * @param applyOnRestart true if this config should only be applied on the next restart, * false if it should be applied right away - * @param configMd5 The md5sum of the given payload. + * @param payloadChecksums checksums of the given payload. */ - void addOkResponse(Payload payload, long generation, boolean applyOnRestart, String configMd5); + void addOkResponse(Payload payload, long generation, boolean applyOnRestart, PayloadChecksums payloadChecksums); /** * Get the current config md5 of the client config. diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index e0a5b23a6d4..fbb52e81679 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.protocol; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.DataValue; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; @@ -18,6 +19,9 @@ import java.nio.ByteBuffer; import java.util.Optional; import java.util.logging.Logger; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + /** * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields: * @@ -68,9 +72,9 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { } @Override - public void addOkResponse(Payload payload, long generation, boolean applyOnRestart, String configMd5) { + public void addOkResponse(Payload payload, long generation, boolean applyOnRestart, PayloadChecksums payloadChecksums) { this.applyOnRestart = applyOnRestart; - boolean changedConfig = !configMd5.equals(getRequestConfigMd5()); + boolean changedConfig = !payloadChecksums.equals(getRequestConfigChecksums()); boolean changedConfigAndNewGeneration = changedConfig && ConfigUtils.isGenerationNewer(generation, getRequestGeneration()); Payload responsePayload = payload.withCompression(getCompressionType()); ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(4096); @@ -78,7 +82,8 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { JsonGenerator jsonGenerator = createJsonGenerator(byteArrayOutputStream); jsonGenerator.writeStartObject(); addCommonReturnValues(jsonGenerator); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, configMd5); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, payloadChecksums.getForType(MD5).asString()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_XXHASH64, payloadChecksums.getForType(XXHASH64).asString()); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_GENERATION, generation); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_APPLY_ON_RESTART, applyOnRestart); jsonGenerator.writeObjectFieldStart(SlimeResponseData.RESPONSE_COMPRESSION_INFO); @@ -194,6 +199,8 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { @Override public String getRequestDefMd5() { return requestData.getRequestDefMd5(); } + public PayloadChecksums getRequestConfigChecksums() { return requestData.getRequestConfigChecksums(); } + private void addErrorResponse(int errorCode) { addErrorResponse(errorCode, ErrorCode.getName(errorCode)); } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index 7db15844e8b..9cd59798ecd 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; -import com.yahoo.config.subscription.impl.PayloadChecksum; +import com.yahoo.vespa.config.PayloadChecksum; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ErrorCode; @@ -9,6 +9,7 @@ import com.yahoo.vespa.config.ErrorCode; import java.util.logging.Logger; import java.util.regex.Matcher; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; import static java.util.logging.Level.INFO; /** @@ -29,12 +30,12 @@ public class RequestValidation { log.log(INFO, "Illegal name space '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_NAME_SPACE; } - if (!(new PayloadChecksum(request.getRequestDefMd5()).valid())) { + if (!(new PayloadChecksum(request.getRequestDefMd5(), MD5).valid())) { log.log(INFO, "Illegal checksum '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_DEF_MD5; // TODO: Use ILLEGAL_DEF_CHECKSUM } - if (!new PayloadChecksum(request.getRequestConfigMd5()).valid()) { - log.log(INFO, "Illegal config checksum '" + request.getRequestConfigMd5() + "'"); + if (! request.getRequestConfigChecksums().valid()) { + log.log(INFO, "Illegal config checksum '" + request.getRequestConfigChecksums() + "'"); return ErrorCode.ILLEGAL_CONFIG_MD5; // TODO: Use ILLEGAL_CONFIG_CHECKSUM } if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java index 1ccf6e367fc..8d08717942b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.text.AbstractUtf8Array; import com.yahoo.vespa.config.ConfigPayload; @@ -8,6 +9,8 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; + /** * Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface. * @@ -19,25 +22,42 @@ public class SlimeConfigResponse implements ConfigResponse { private final CompressionInfo compressionInfo; private final long generation; private final boolean applyOnRestart; - private final String configMd5; + private final PayloadChecksums payloadChecksums; + + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, + long generation, + boolean applyOnRestart, + PayloadChecksums payloadChecksums) { + AbstractUtf8Array data = payload.toUtf8Array(true); + return new SlimeConfigResponse(data, + generation, + applyOnRestart, + payloadChecksums, + CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); + } - public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, long generation, - boolean applyOnRestart, String configMd5) { + // TODO: Legacy method, remove when not used anymore + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, + long generation, + boolean applyOnRestart, + String configMd5) { AbstractUtf8Array data = payload.toUtf8Array(true); - return new SlimeConfigResponse(data, generation, applyOnRestart, - configMd5, + return new SlimeConfigResponse(data, + generation, + applyOnRestart, + PayloadChecksums.from(configMd5, ""), CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); } public SlimeConfigResponse(AbstractUtf8Array payload, long generation, boolean applyOnRestart, - String configMd5, + PayloadChecksums payloadChecksums, CompressionInfo compressionInfo) { this.payload = payload; this.generation = generation; this.applyOnRestart = applyOnRestart; - this.configMd5 = configMd5; + this.payloadChecksums = payloadChecksums; this.compressionInfo = compressionInfo; } @@ -56,7 +76,7 @@ public class SlimeConfigResponse implements ConfigResponse { @Override public String getConfigMd5() { - return configMd5; + return payloadChecksums.getForType(MD5).asString(); } @Override @@ -68,11 +88,13 @@ public class SlimeConfigResponse implements ConfigResponse { @Override public String toString() { return "generation=" + generation + "\n" + - "configmd5=" + configMd5 + "\n" + + "checksums=" + payloadChecksums + "\n" + Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED); } @Override public CompressionInfo getCompressionInfo() { return compressionInfo; } + @Override + public PayloadChecksums getPayloadChecksums() { return payloadChecksums; } } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java index b885623a78b..679298bac73 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksum; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -11,6 +13,9 @@ import com.yahoo.vespa.config.util.ConfigUtils; import java.util.Optional; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + /** * Contains slime request data objects. Provides methods for reading various fields from slime request data. * All data is read lazily. @@ -27,6 +32,7 @@ class SlimeRequestData { private static final String REQUEST_CLIENT_HOSTNAME = "clientHostname"; private static final String REQUEST_CURRENT_GENERATION = "currentGeneration"; private static final String REQUEST_CONFIG_MD5 = "configMD5"; + private static final String REQUEST_CONFIG_XXHASH64 = "configXxhash64"; private static final String REQUEST_TRACE = "trace"; private static final String REQUEST_TIMEOUT = "timeout"; private static final String REQUEST_DEF_MD5 = "defMD5"; @@ -79,6 +85,17 @@ class SlimeRequestData { String getRequestDefMd5() { return getRequestField(REQUEST_DEF_MD5).asString(); } + PayloadChecksum getRequestConfigXxhash64() { + Inspector xxhash64Field = getRequestField(REQUEST_CONFIG_XXHASH64); + return xxhash64Field.valid() + ? new PayloadChecksum(xxhash64Field.asString(), XXHASH64) + : PayloadChecksum.empty(XXHASH64); + } + + PayloadChecksums getRequestConfigChecksums() { + return PayloadChecksums.from(getRequestConfigMd5(), getRequestConfigXxhash64().asString()); + } + long getRequestGeneration() { return getRequestField(REQUEST_CURRENT_GENERATION).asLong(); } @@ -86,7 +103,7 @@ class SlimeRequestData { static Slime encodeRequest(ConfigKey<?> key, String hostname, DefContent defSchema, - String configMd5, + PayloadChecksums payloadChecksums, long generation, long timeout, Trace trace, @@ -102,7 +119,8 @@ class SlimeRequestData { request.setString(REQUEST_CLIENT_CONFIGID, key.getConfigId()); request.setString(REQUEST_CLIENT_HOSTNAME, hostname); defSchema.serialize(request.setArray(REQUEST_DEF_CONTENT)); - request.setString(REQUEST_CONFIG_MD5, configMd5); + request.setString(REQUEST_CONFIG_MD5, payloadChecksums.getForType(MD5).asString()); + request.setString(REQUEST_CONFIG_XXHASH64, payloadChecksums.getForType(XXHASH64).asString()); request.setLong(REQUEST_CURRENT_GENERATION, generation); request.setLong(REQUEST_TIMEOUT, timeout); request.setString(REQUEST_COMPRESSION_TYPE, compressionType.name()); diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java index cc98587456c..965622adaa5 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.vespa.config.PayloadChecksum; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + /** * Contains response data for a slime response and methods for decoding the response data that * are common to all {@link Slime} based config requests. @@ -22,6 +27,7 @@ class SlimeResponseData { static final String RESPONSE_CLIENT_HOSTNAME = "clientHostname"; static final String RESPONSE_TRACE = "trace"; static final String RESPONSE_CONFIG_MD5 = "configMD5"; + static final String RESPONSE_CONFIG_XXHASH64 = "configXxhash64"; static final String RESPONSE_CONFIG_GENERATION = "generation"; static final String RESPONSE_APPLY_ON_RESTART = "applyOnRestart"; static final String RESPONSE_COMPRESSION_INFO = "compressionInfo"; @@ -58,9 +64,25 @@ class SlimeResponseData { return trace.valid() ? Trace.fromSlime(trace) : Trace.createDummy(); } - String getResponseConfigMd5() { - Inspector inspector = getResponseField(RESPONSE_CONFIG_MD5); - return inspector.valid() ? inspector.asString() : ""; + PayloadChecksum getResponseConfigMd5() { + Inspector md5Field = getResponseField(RESPONSE_CONFIG_MD5); + return md5Field.valid() + ? new PayloadChecksum(md5Field.asString(), MD5) + : PayloadChecksum.empty(MD5); + } + + PayloadChecksum getResponseConfigXxhash64() { + Inspector xxhash64Field = getResponseField(RESPONSE_CONFIG_XXHASH64); + return xxhash64Field.valid() + ? new PayloadChecksum(xxhash64Field.asString(), XXHASH64) + : PayloadChecksum.empty(XXHASH64); + } + + + PayloadChecksums getResponseConfigChecksums() { + PayloadChecksum responseConfigMd5 = getResponseConfigMd5(); + System.out.println(responseConfigMd5); + return PayloadChecksums.from(responseConfigMd5, getResponseConfigXxhash64()); } CompressionInfo getCompressionInfo() { diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java index a7fc8afcad9..329661bf7ae 100644 --- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -10,6 +10,8 @@ import com.yahoo.text.AbstractUtf8Array; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; import java.io.ByteArrayOutputStream; import java.io.File; @@ -94,6 +96,15 @@ public class ConfigUtils { } } + public static String getXxhash64(AbstractUtf8Array input) { + return getXxhash64(input.wrap()); + } + + public static String getXxhash64(ByteBuffer input) { + XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + return Long.toHexString(hasher.hash(input, 0)).toLowerCase(); + } + /** * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; * |