summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2021-09-09 17:00:16 +0200
committerGitHub <noreply@github.com>2021-09-09 17:00:16 +0200
commit979cb96cdeb6739d7ea769c60a04bac174619be1 (patch)
tree398df7405a72f47f14f90d421cda829d385f010b
parent9cf8fa3a53f581dfa794a8c1610844c6a9d460e0 (diff)
Revert "Revert "Use both xxhash64 and md5 for config payload in requests and responses [run-systemtest]""
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java4
-rw-r--r--config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java16
-rw-r--r--config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java25
-rw-r--r--config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java6
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java29
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java3
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java4
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java3
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java (renamed from config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java)28
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java83
-rwxr-xr-xconfig/src/main/java/com/yahoo/vespa/config/RawConfig.java39
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java3
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java3
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java13
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java64
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java9
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java5
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java13
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java9
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java40
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java22
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java28
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java11
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java7
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java52
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java14
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java21
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java55
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java17
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java7
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java10
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java16
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java12
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java23
36 files changed, 481 insertions, 217 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
index 3700dee92e3..2d9f4bd7fd0 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
@@ -12,7 +12,6 @@ import com.yahoo.jrt.StringValue;
import com.yahoo.jrt.Supervisor;
import com.yahoo.jrt.Target;
import com.yahoo.jrt.TargetWatcher;
-import java.util.logging.Level;
import com.yahoo.vespa.config.JRTMethods;
import com.yahoo.vespa.config.RawConfig;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
@@ -23,6 +22,7 @@ import java.util.Iterator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
import java.util.logging.Logger;
/**
@@ -353,7 +353,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer
request.addOkResponse(config.getPayload(),
config.getGeneration(),
config.applyOnRestart(),
- config.getConfigMd5());
+ config.getPayloadChecksums());
log.log(Level.FINE, () -> "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() +
",generation=" + config.getGeneration());
log.log(Level.FINEST, () -> "Config payload in response for " + request.getShortDescription() + ":" + config.getPayload());
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java
index 1b009b80fc1..45f52479cbd 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java
@@ -5,6 +5,8 @@ import com.yahoo.jrt.Request;
import com.yahoo.slime.Slime;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ConfigPayload;
+import com.yahoo.vespa.config.PayloadChecksum;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.vespa.config.RawConfig;
import com.yahoo.vespa.config.protocol.CompressionType;
import com.yahoo.vespa.config.protocol.DefContent;
@@ -19,6 +21,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
+
/**
* @author bratseth
*/
@@ -42,7 +47,8 @@ public class ConfigTester {
long generation = 1;
String defMd5 = ConfigUtils.getDefMd5(defContent);
- String configMd5 = ConfigUtils.getMd5(fooConfigPayload);
+ PayloadChecksums configMd5 = PayloadChecksums.from(new PayloadChecksum(ConfigUtils.getMd5(fooConfigPayload), MD5),
+ PayloadChecksum.empty(XXHASH64));
fooConfig = new RawConfig(configKey, defMd5, fooPayload, configMd5,
generation, false, defContent, Optional.empty());
@@ -57,24 +63,24 @@ public class ConfigTester {
JRTServerConfigRequest createRequest(RawConfig config, long timeout) {
return createRequest(config.getName(), config.getConfigId(), config.getNamespace(),
- config.getConfigMd5(), config.getGeneration(), timeout);
+ config.getPayloadChecksums(), config.getGeneration(), timeout);
}
JRTServerConfigRequest createRequest(String configName, String configId, String namespace, long timeout) {
- return createRequest(configName, configId, namespace, null, 0, timeout);
+ return createRequest(configName, configId, namespace, PayloadChecksums.empty(), 0, timeout);
}
private JRTServerConfigRequest createRequest(String configName,
String configId,
String namespace,
- String md5,
+ PayloadChecksums payloadChecksums,
long generation,
long timeout) {
Request request = JRTClientConfigRequestV3.
createWithParams(new ConfigKey<>(configName, configId, namespace, null),
DefContent.fromList(defContent),
"fromHost",
- md5,
+ payloadChecksums,
generation,
timeout,
Trace.createDummy(),
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java
index 485a091d9ae..b47c0bcc5ce 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java
@@ -1,10 +1,11 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.proxy;
import com.yahoo.slime.Slime;
import com.yahoo.vespa.config.ConfigCacheKey;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ConfigPayload;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.vespa.config.RawConfig;
import com.yahoo.vespa.config.protocol.Payload;
import org.junit.Before;
@@ -18,24 +19,24 @@ import static org.junit.Assert.*;
/**
* @author hmusum
- * @since 5.1.9
*/
public class MemoryCacheTest {
- private String defName = "foo";
- private String configId = "id";
- private String namespace = "bar";
+
+ private final String defName = "foo";
+ private final String configId = "id";
+ private final String namespace = "bar";
private static final String defMd5 = "a";
- private long generation = 1L;
- private String defName2 = "baz-quux";
- private String namespace2 = "search.config";
+ private final long generation = 1L;
+ private final String defName2 = "baz-quux";
+ private final String namespace2 = "search.config";
// Test with a config id with / in it
- private String configId2 = "clients/gateways/gateway/component/com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation";
+ private final String configId2 = "clients/gateways/gateway/component/com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation";
private static final String defMd52 = "a2";
private static final String differentDefMd5 = "09ef";
- private static final String configMd5 = "b";
- private ConfigKey<?> configKey = new ConfigKey<>(defName, configId, namespace);
- private ConfigKey<?> configKey2 = new ConfigKey<>(defName2, configId2, namespace2);
+ private static final PayloadChecksums configMd5 = PayloadChecksums.from("b", "");
+ private final ConfigKey<?> configKey = new ConfigKey<>(defName, configId, namespace);
+ private final ConfigKey<?> configKey2 = new ConfigKey<>(defName2, configId2, namespace2);
private ConfigCacheKey cacheKey;
private ConfigCacheKey cacheKeyDifferentMd5;
private ConfigCacheKey cacheKey2;
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
index 87c1fa151f8..32e68c662e8 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
@@ -35,7 +35,7 @@ public class ProxyServerTest {
// errorConfig based on fooConfig
private static final ConfigKey<?> errorConfigKey = new ConfigKey<>("error", fooConfig.getConfigId(), fooConfig.getNamespace());
static final RawConfig errorConfig = new RawConfig(errorConfigKey, fooConfig.getDefMd5(), fooConfig.getPayload(),
- fooConfig.getConfigMd5(), fooConfig.getGeneration(), false,
+ fooConfig.getPayloadChecksums(), fooConfig.getGeneration(), false,
ErrorCode.UNKNOWN_DEFINITION, fooConfig.getDefContent(), Optional.empty());
@Rule
@@ -179,7 +179,7 @@ public class ProxyServerTest {
// Simulate an empty response
RawConfig emptyConfig = new RawConfig(fooConfig.getKey(), fooConfig.getDefMd5(), Payload.from("{}"),
- fooConfig.getConfigMd5(), 0, false,
+ fooConfig.getPayloadChecksums(), 0, false,
0, fooConfig.getDefContent(), Optional.empty());
source.put(fooConfig.getKey(), emptyConfig);
@@ -238,7 +238,7 @@ public class ProxyServerTest {
static RawConfig createConfigWithNextConfigGeneration(RawConfig config, int errorCode, Payload payload, long configGeneration) {
return new RawConfig(config.getKey(), config.getDefMd5(),
- payload, config.getConfigMd5(),
+ payload, config.getPayloadChecksums(),
configGeneration, false,
errorCode, config.getDefContent(), Optional.empty());
}
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
index 53a9f3f9f94..b1939ac23c6 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
@@ -11,6 +11,7 @@ import com.yahoo.config.subscription.FileSource;
import com.yahoo.config.subscription.JarSource;
import com.yahoo.config.subscription.RawSource;
import com.yahoo.vespa.config.ConfigKey;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.vespa.config.TimingValues;
import com.yahoo.vespa.config.protocol.DefContent;
@@ -40,31 +41,31 @@ public abstract class ConfigSubscription<T extends ConfigInstance> {
private final T config;
private final Long generation;
private final boolean applyOnRestart;
- private final PayloadChecksum payloadChecksum;
+ private final PayloadChecksums payloadChecksums;
private ConfigState(boolean generationChanged,
Long generation,
boolean applyOnRestart,
boolean configChanged,
T config,
- PayloadChecksum payloadChecksum) {
+ PayloadChecksums payloadChecksums) {
this.generationChanged = generationChanged;
this.generation = generation;
this.applyOnRestart = applyOnRestart;
this.configChanged = configChanged;
this.config = config;
- this.payloadChecksum = payloadChecksum;
+ this.payloadChecksums = payloadChecksums;
}
- private ConfigState(Long generation, T config, PayloadChecksum payloadChecksum) {
- this(false, generation, false, false, config, payloadChecksum);
+ private ConfigState(Long generation, T config, PayloadChecksums payloadChecksums) {
+ this(false, generation, false, false, config, payloadChecksums);
}
private ConfigState() {
- this(false, 0L, false, false, null, PayloadChecksum.empty());
+ this(false, 0L, false, false, null, PayloadChecksums.empty());
}
- private ConfigState<T> createUnchanged() { return new ConfigState<>(generation, config, payloadChecksum); }
+ private ConfigState<T> createUnchanged() { return new ConfigState<>(generation, config, payloadChecksums); }
public boolean isConfigChanged() { return configChanged; }
@@ -76,7 +77,7 @@ public abstract class ConfigSubscription<T extends ConfigInstance> {
public T getConfig() { return config; }
- public PayloadChecksum getChecksum() { return payloadChecksum; }
+ public PayloadChecksums getChecksums() { return payloadChecksums; }
}
@@ -195,8 +196,8 @@ public abstract class ConfigSubscription<T extends ConfigInstance> {
return !prev.getGeneration().equals(requiredGen) || prev.isConfigChanged();
}
- void setConfig(Long generation, boolean applyOnRestart, T config, PayloadChecksum payloadChecksum) {
- this.config.set(new ConfigState<>(true, generation, applyOnRestart, true, config, payloadChecksum));
+ void setConfig(Long generation, boolean applyOnRestart, T config, PayloadChecksums payloadChecksums) {
+ this.config.set(new ConfigState<>(true, generation, applyOnRestart, true, config, payloadChecksums));
}
/**
@@ -204,22 +205,22 @@ public abstract class ConfigSubscription<T extends ConfigInstance> {
*/
protected void setConfigIncGen(T config) {
ConfigState<T> prev = this.config.get();
- this.config.set(new ConfigState<>(true, prev.getGeneration() + 1, prev.applyOnRestart(), true, config, prev.payloadChecksum));
+ this.config.set(new ConfigState<>(true, prev.getGeneration() + 1, prev.applyOnRestart(), true, config, prev.payloadChecksums));
}
protected void setConfigIfChanged(T config) {
ConfigState<T> prev = this.config.get();
- this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.applyOnRestart(), !config.equals(prev.getConfig()), config, prev.payloadChecksum));
+ this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.applyOnRestart(), !config.equals(prev.getConfig()), config, prev.payloadChecksums));
}
void setGeneration(Long generation) {
ConfigState<T> prev = config.get();
- this.config.set(new ConfigState<>(true, generation, prev.applyOnRestart(), prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksum));
+ this.config.set(new ConfigState<>(true, generation, prev.applyOnRestart(), prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksums));
}
void setApplyOnRestart(boolean applyOnRestart) {
ConfigState<T> prev = config.get();
- this.config.set(new ConfigState<>(prev.isGenerationChanged(), prev.getGeneration(), applyOnRestart, prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksum));
+ this.config.set(new ConfigState<>(prev.isGenerationChanged(), prev.getGeneration(), applyOnRestart, prev.isConfigChanged(), prev.getConfig(), prev.payloadChecksums));
}
/**
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
index 05d4a33c02a..e9a40539bf0 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
@@ -33,7 +33,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfi
@Override
protected void setNewConfig(JRTClientConfigRequest jrtReq) {
RawConfig rawConfig = RawConfig.createFromResponseParameters(jrtReq);
- setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), rawConfig, new PayloadChecksum(jrtReq.getNewConfigMd5()));
+ setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), rawConfig, jrtReq.getNewChecksums());
log.log(FINE, () -> "in setNewConfig, config=" + this.getConfigState().getConfig());
}
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
index b06f986555c..bb1a154d5d0 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
@@ -111,7 +111,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
} catch (IllegalArgumentException e) {
badConfigE = e;
}
- setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), configInstance, new PayloadChecksum(jrtReq.getNewConfigMd5()));
+ setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), configInstance, jrtReq.getNewChecksums());
if (badConfigE != null) {
throw new IllegalArgumentException("Bad config from jrt", badConfigE);
}
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
index b7198b0c694..4a55c046f13 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
@@ -9,6 +9,7 @@ import com.yahoo.config.subscription.ConfigSubscriber;
import com.yahoo.io.IOUtils;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ConfigPayload;
+import com.yahoo.vespa.config.PayloadChecksums;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -63,7 +64,7 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
} catch (IOException e) {
throw new ConfigurationRuntimeException(e);
}
- setConfig(0L, false, config, PayloadChecksum.empty());
+ setConfig(0L, false, config, PayloadChecksums.empty());
try {
jarFile.close();
} catch (IOException e) {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
index bed7a0fa3c4..e9e7f3e7bce 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
@@ -7,9 +7,9 @@ import com.yahoo.jrt.Supervisor;
import com.yahoo.vespa.config.ConfigPayload;
import com.yahoo.vespa.config.Connection;
import com.yahoo.vespa.config.ConnectionPool;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
import com.yahoo.vespa.config.protocol.Payload;
-import com.yahoo.vespa.config.util.ConfigUtils;
/**
* For unit testing
@@ -96,7 +96,7 @@ public class MockConnection implements ConnectionPool, Connection {
JRTServerConfigRequestV3 jrtReq = JRTServerConfigRequestV3.createFromRequest(request);
Payload payload = Payload.from(ConfigPayload.empty());
long generation = 1;
- jrtReq.addOkResponse(payload, generation, false, ConfigUtils.getMd5(payload.getData()));
+ jrtReq.addOkResponse(payload, generation, false, PayloadChecksums.fromPayload(payload));
}
}
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
index 91b674da3d2..8d5e7839086 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
@@ -7,6 +7,7 @@ import com.yahoo.config.subscription.ConfigInterruptedException;
import com.yahoo.config.subscription.ConfigSubscriber;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ConfigPayload;
+import com.yahoo.vespa.config.PayloadChecksums;
import java.util.Arrays;
@@ -35,7 +36,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
if (payload == null) {
payload = inputPayload;
ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n")));
- setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()), PayloadChecksum.empty());
+ setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()), PayloadChecksums.empty());
return true;
}
try {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java
index 93b85aaabd0..17f486e15d1 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java
+++ b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.config.subscription.impl;
+package com.yahoo.vespa.config;
+import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -17,13 +18,13 @@ public class PayloadChecksum {
private final String checksum;
private final Type type;
- public PayloadChecksum(String checksum) {
+ public PayloadChecksum(String checksum, Type type) {
this.checksum = checksum;
- this.type = Type.MD5;
+ this.type = type;
}
- public static PayloadChecksum empty() {
- return new PayloadChecksum("");
+ public static PayloadChecksum empty(Type type) {
+ return new PayloadChecksum("", type);
}
public String asString() { return checksum; }
@@ -45,4 +46,21 @@ public class PayloadChecksum {
return m.matches();
}
+ @Override
+ public int hashCode() {
+ return Objects.hash(checksum, type);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ PayloadChecksum that = (PayloadChecksum) o;
+ return Objects.equals(checksum, that.checksum) && type == that.type;
+ }
+
+ @Override
+ public String toString() {
+ return type.name() + ":" + checksum;
+ }
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java
new file mode 100644
index 00000000000..1558771bd58
--- /dev/null
+++ b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksums.java
@@ -0,0 +1,83 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config;
+
+import com.yahoo.vespa.config.protocol.Payload;
+import com.yahoo.vespa.config.util.ConfigUtils;
+
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
+
+/**
+ * Checksums for config payload, typically 1 for each PayloadChecksum type (md5 and xxhash64).
+ * Initialized with empty checksum for each existing type.
+ *
+ * @author hmusum
+ */
+public class PayloadChecksums {
+
+ private final Map<PayloadChecksum.Type, PayloadChecksum> checksums = new LinkedHashMap<>();
+
+ private PayloadChecksums() {
+ Arrays.stream(PayloadChecksum.Type.values()).forEach(type -> checksums.put(type, PayloadChecksum.empty(type)));
+ }
+
+ public static PayloadChecksums empty() { return new PayloadChecksums(); }
+
+ public static PayloadChecksums from(PayloadChecksum... checksums) {
+ PayloadChecksums payloadChecksums = new PayloadChecksums();
+ Arrays.stream(checksums).forEach(payloadChecksums::add);
+ return payloadChecksums;
+ }
+
+ public static PayloadChecksums from(String configMd5, String configXxhash64) {
+ return new PayloadChecksums()
+ .add(new PayloadChecksum(configMd5, MD5))
+ .add(new PayloadChecksum(configXxhash64, XXHASH64));
+ }
+
+ public static PayloadChecksums fromPayload(Payload payload) {
+ return new PayloadChecksums()
+ .add(new PayloadChecksum(ConfigUtils.getMd5(payload.getData()), MD5))
+ .add(new PayloadChecksum(ConfigUtils.getXxhash64(payload.getData()), XXHASH64));
+ }
+
+ private PayloadChecksums add(PayloadChecksum checksum) {
+ checksums.put(checksum.type(), checksum);
+ return this;
+ }
+
+ public PayloadChecksum getForType(PayloadChecksum.Type type) {
+ return checksums.get(type);
+ }
+
+ public boolean valid() {
+ return checksums.values().stream().allMatch(PayloadChecksum::valid);
+ }
+
+ @Override
+ public String toString() {
+ return checksums.values().stream()
+ .map(checksum -> checksum.type().name() + ":" + checksum.asString())
+ .collect(Collectors.joining(","));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ PayloadChecksums that = (PayloadChecksums) o;
+ return Objects.equals(checksums, that.checksums);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(checksums);
+ }
+
+}
diff --git a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java
index 028e84e4c29..78c3fefc936 100755
--- a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java
+++ b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config;
import com.yahoo.config.ConfigInstance;
@@ -28,7 +28,7 @@ public class RawConfig extends ConfigInstance {
private final List<String> defContent;
private final Payload payload;
private final int errorCode;
- private final String configMd5;
+ private final PayloadChecksums payloadChecksums;
private final Optional<VespaVersion> vespaVersion;
private long generation;
private boolean applyOnRestart;
@@ -40,29 +40,28 @@ public class RawConfig extends ConfigInstance {
* @param defMd5 The md5 sum of the .def-file.
*/
public RawConfig(ConfigKey<?> key, String defMd5) {
- this(key, defMd5, null, "", 0L, false, 0, Collections.emptyList(), Optional.empty());
+ this(key, defMd5, null, PayloadChecksums.empty(), 0L, false, 0, Collections.emptyList(), Optional.empty());
}
- public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation,
- boolean applyOnRestart, List<String> defContent,
- Optional<VespaVersion> vespaVersion) {
- this(key, defMd5, payload, configMd5, generation, applyOnRestart, 0, defContent, vespaVersion);
+ public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, PayloadChecksums payloadChecksums, long generation,
+ boolean applyOnRestart, List<String> defContent, Optional<VespaVersion> vespaVersion) {
+ this(key, defMd5, payload, payloadChecksums, generation, applyOnRestart, 0, defContent, vespaVersion);
}
/** Copy constructor */
public RawConfig(RawConfig rawConfig) {
- this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.configMd5,
+ this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.payloadChecksums,
rawConfig.generation, rawConfig.applyOnRestart,
rawConfig.errorCode, rawConfig.defContent, rawConfig.getVespaVersion());
}
- public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation,
+ public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, PayloadChecksums payloadChecksums, long generation,
boolean applyOnRestart, int errorCode, List<String> defContent,
Optional<VespaVersion> vespaVersion) {
this.key = key;
this.defMd5 = ConfigUtils.getDefMd5FromRequest(defMd5, defContent);
this.payload = payload;
- this.configMd5 = configMd5;
+ this.payloadChecksums = payloadChecksums;
this.generation = generation;
this.applyOnRestart = applyOnRestart;
this.errorCode = errorCode;
@@ -79,7 +78,7 @@ public class RawConfig extends ConfigInstance {
return new RawConfig(req.getConfigKey(),
ConfigUtils.getDefMd5(req.getDefContent().asList()),
req.getNewPayload(),
- req.getNewConfigMd5(),
+ req.getNewChecksums(),
req.getNewGeneration(),
req.responseIsApplyOnRestart(),
0,
@@ -96,7 +95,7 @@ public class RawConfig extends ConfigInstance {
return new RawConfig(req.getConfigKey(),
ConfigUtils.getDefMd5(req.getDefContent().asList()),
Payload.from(new Utf8String(""), CompressionInfo.uncompressed()),
- req.getRequestConfigMd5(),
+ req.getRequestConfigChecksums(),
req.getRequestGeneration(),
req.applyOnRestart(),
0,
@@ -113,7 +112,7 @@ public class RawConfig extends ConfigInstance {
public String getConfigId() { return key.getConfigId(); }
- public String getConfigMd5() { return configMd5; }
+ public String getConfigMd5() { return payloadChecksums.getForType(PayloadChecksum.Type.MD5).asString(); }
public String getDefMd5() { return defMd5; }
@@ -133,6 +132,8 @@ public class RawConfig extends ConfigInstance {
public Optional<VespaVersion> getVespaVersion() { return vespaVersion; }
+ public PayloadChecksums getPayloadChecksums() { return payloadChecksums; }
+
/**
* Returns true if this config is equal to the config (same payload md5) in the given request.
*
@@ -174,11 +175,7 @@ public class RawConfig extends ConfigInstance {
// while non-zero and equal error codes means configs are equal.
if (isError()) return true;
if (generation != other.generation) return false;
- if (configMd5 != null) {
- return configMd5.equals(other.configMd5);
- } else {
- return (other.configMd5 == null);
- }
+ return (payloadChecksums.equals(((RawConfig) o).payloadChecksums));
}
@Override
@@ -194,9 +191,7 @@ public class RawConfig extends ConfigInstance {
if (! isError()) {
// configMd5 and generation only matter when the RawConfig is not an error.
hash = 31 * hash + (int)(generation ^(generation >>>32));
- if (configMd5 != null) {
- hash = 31 * hash + configMd5.hashCode();
- }
+ hash = 31 * hash + payloadChecksums.hashCode();
}
return hash;
}
@@ -210,7 +205,7 @@ public class RawConfig extends ConfigInstance {
sb.append(",");
sb.append(key.getConfigId());
sb.append(",");
- sb.append(getConfigMd5());
+ sb.append(payloadChecksums);
sb.append(",");
sb.append(getGeneration());
sb.append(",");
diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java
index adb27f37413..345118b5fd4 100644
--- a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java
+++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.config.benchmark;
import com.yahoo.collections.Tuple2;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.io.IOUtils;
import com.yahoo.jrt.Spec;
import com.yahoo.jrt.Supervisor;
@@ -256,7 +257,7 @@ public class LoadTester {
final long serverTimeout = 1000;
return JRTClientConfigRequestV3.createWithParams(fullKey, DefContent.fromList(List.of(defContent.second)),
- ConfigUtils.getCanonicalHostName(), "",
+ ConfigUtils.getCanonicalHostName(), PayloadChecksums.empty(),
0, serverTimeout, Trace.createDummy(),
compressionType, Optional.empty());
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java
index f6fce56c227..98fc7f7a50e 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.text.AbstractUtf8Array;
import java.io.IOException;
@@ -36,4 +37,6 @@ public interface ConfigResponse {
CompressionInfo getCompressionInfo();
+ PayloadChecksums getPayloadChecksums();
+
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java
index 8535cc23225..9d3b87574f3 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java
@@ -1,6 +1,8 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
+import com.yahoo.vespa.config.PayloadChecksums;
+
/**
* Interface for config requests used by clients.
*
@@ -57,13 +59,20 @@ public interface JRTClientConfigRequest extends JRTConfigRequest {
boolean responseIsApplyOnRestart();
/**
- * Get the config md5 of the config returned by the server. Return an empty string if no response has been returned.
+ * Gets the config md5 of the config returned by the server. Returns an empty string if no response has been returned.
*
* @return a config md5.
*/
String getNewConfigMd5();
/**
+ * Gets the config checksums of the config returned by the server. Returns an empty string if no response has been returned.
+ *
+ * @return a config checksum.
+ */
+ PayloadChecksums getNewChecksums();
+
+ /**
* Test whether or not the response contains an updated config or not.
* False if no response has been returned.
*
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java
index f5b558550e4..a6271b159ef 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java
@@ -1,9 +1,10 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
import com.yahoo.config.ConfigInstance;
import com.yahoo.config.subscription.impl.ConfigSubscription;
import com.yahoo.config.subscription.impl.JRTConfigSubscription;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.Request;
import com.yahoo.jrt.StringValue;
import com.yahoo.slime.JsonFormat;
@@ -38,7 +39,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
protected JRTClientConfigRequestV3(ConfigKey<?> key,
String hostname,
DefContent defSchema,
- String configMd5,
+ PayloadChecksums payloadChecksums,
long generation,
long timeout,
Trace trace,
@@ -47,7 +48,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
Slime data = SlimeRequestData.encodeRequest(key,
hostname,
defSchema,
- configMd5,
+ payloadChecksums,
generation,
timeout,
trace,
@@ -97,7 +98,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
return new JRTClientConfigRequestV3(getConfigKey(),
getClientHostName(),
getDefContent(),
- isError() ? getRequestConfigMd5() : newConfMd5(),
+ isError() ? getRequestConfigChecksums() : newConfigChecksums(),
isError() ? getRequestGeneration() : newGen(),
timeout,
Trace.createNew(),
@@ -113,7 +114,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
return createWithParams(sub.getKey(),
sub.getDefContent(),
ConfigUtils.getCanonicalHostName(),
- configState.getChecksum().asString(),
+ configState.getChecksums(),
configState.getGeneration(),
sub.timingValues().getSubscribeTimeout(),
trace,
@@ -128,34 +129,34 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
Optional<VespaVersion> vespaVersion) {
String hostname = ConfigUtils.getCanonicalHostName();
return createWithParams(config.getKey(),
- DefContent.fromList(config.getDefContent()),
- hostname,
- config.getConfigMd5(),
- config.getGeneration(),
- serverTimeout,
- trace,
- compressionType,
- vespaVersion);
+ DefContent.fromList(config.getDefContent()),
+ hostname,
+ config.getPayloadChecksums(),
+ config.getGeneration(),
+ serverTimeout,
+ trace,
+ compressionType,
+ vespaVersion);
}
public static JRTClientConfigRequest createWithParams(ConfigKey<?> reqKey,
DefContent defContent,
String hostname,
- String configMd5,
+ PayloadChecksums payloadChecksums,
long generation,
long serverTimeout,
Trace trace,
CompressionType compressionType,
Optional<VespaVersion> vespaVersion) {
return new JRTClientConfigRequestV3(reqKey,
- hostname,
- defContent,
- configMd5,
- generation,
- serverTimeout,
- trace,
- compressionType,
- vespaVersion);
+ hostname,
+ defContent,
+ payloadChecksums,
+ generation,
+ serverTimeout,
+ trace,
+ compressionType,
+ vespaVersion);
}
@Override
@@ -177,7 +178,7 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
.append(",").append(getTimeout())
.append(",").append(getVespaVersion().map(VespaVersion::toString).orElse(""))
.append("'\n");
- sb.append("response='").append(getNewConfigMd5())
+ sb.append("response='").append(getNewChecksums())
.append(",").append(getNewGeneration())
.append(",").append(responseIsApplyOnRestart())
.append("'\n");
@@ -221,6 +222,12 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
return requestData.getTimeout();
}
+ protected PayloadChecksums newConfigChecksums() {
+ PayloadChecksums newChecksum = getNewChecksums();
+ if (PayloadChecksums.empty().equals(newChecksum)) return getRequestConfigChecksums();
+ return newChecksum;
+ }
+
protected String newConfMd5() {
String newMd5 = getNewConfigMd5();
if ("".equals(newMd5)) return getRequestConfigMd5();
@@ -264,6 +271,10 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
return requestData.getRequestDefMd5();
}
+ public PayloadChecksums getRequestConfigChecksums() {
+ return requestData.getRequestConfigChecksums();
+ }
+
@Override
public boolean validateResponse() {
if (request.isError()) {
@@ -285,7 +296,12 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
@Override
public String getNewConfigMd5() {
- return responseData.getResponseConfigMd5();
+ return responseData.getResponseConfigMd5().asString();
+ }
+
+ @Override
+ public PayloadChecksums getNewChecksums() {
+ return responseData.getResponseConfigChecksums();
}
@Override
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java
index 5b8f040b8e3..0fc751dc49f 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.Request;
import com.yahoo.vespa.config.ConfigKey;
@@ -44,6 +45,14 @@ public interface JRTConfigRequest {
/**
* Returns the generation of the requested config. If none has been given, 0 should be returned.
+ * Returns the checksum of the config request. Return an empty string if no response has been returned.
+ *
+ * @return a config checksum.
+ */
+ PayloadChecksums getRequestConfigChecksums();
+
+ /**
+ * Returns the generation of the requested config. If none has been given, 0 should be returned.
*
* @return the generation in the request.
*/
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java
index abc2b0b4473..41106e138b7 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.vespa.config.GetConfigRequest;
/**
@@ -34,9 +35,9 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque
* @param generation The config generation of the given payload.
* @param applyOnRestart true if this config should only be applied on the next restart,
* false if it should be applied right away
- * @param configMd5 The md5sum of the given payload.
+ * @param payloadChecksums checksums of the given payload.
*/
- void addOkResponse(Payload payload, long generation, boolean applyOnRestart, String configMd5);
+ void addOkResponse(Payload payload, long generation, boolean applyOnRestart, PayloadChecksums payloadChecksums);
/**
* Get the current config md5 of the client config.
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
index e0a5b23a6d4..fbb52e81679 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.config.protocol;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.DataValue;
import com.yahoo.jrt.Request;
import com.yahoo.jrt.StringValue;
@@ -18,6 +19,9 @@ import java.nio.ByteBuffer;
import java.util.Optional;
import java.util.logging.Logger;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
+
/**
* The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields:
*
@@ -68,9 +72,9 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest {
}
@Override
- public void addOkResponse(Payload payload, long generation, boolean applyOnRestart, String configMd5) {
+ public void addOkResponse(Payload payload, long generation, boolean applyOnRestart, PayloadChecksums payloadChecksums) {
this.applyOnRestart = applyOnRestart;
- boolean changedConfig = !configMd5.equals(getRequestConfigMd5());
+ boolean changedConfig = !payloadChecksums.equals(getRequestConfigChecksums());
boolean changedConfigAndNewGeneration = changedConfig && ConfigUtils.isGenerationNewer(generation, getRequestGeneration());
Payload responsePayload = payload.withCompression(getCompressionType());
ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(4096);
@@ -78,7 +82,8 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest {
JsonGenerator jsonGenerator = createJsonGenerator(byteArrayOutputStream);
jsonGenerator.writeStartObject();
addCommonReturnValues(jsonGenerator);
- setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, configMd5);
+ setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, payloadChecksums.getForType(MD5).asString());
+ setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_XXHASH64, payloadChecksums.getForType(XXHASH64).asString());
setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_GENERATION, generation);
setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_APPLY_ON_RESTART, applyOnRestart);
jsonGenerator.writeObjectFieldStart(SlimeResponseData.RESPONSE_COMPRESSION_INFO);
@@ -194,6 +199,8 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest {
@Override
public String getRequestDefMd5() { return requestData.getRequestDefMd5(); }
+ public PayloadChecksums getRequestConfigChecksums() { return requestData.getRequestConfigChecksums(); }
+
private void addErrorResponse(int errorCode) {
addErrorResponse(errorCode, ErrorCode.getName(errorCode));
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
index 7db15844e8b..9cd59798ecd 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
@@ -1,7 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
-import com.yahoo.config.subscription.impl.PayloadChecksum;
+import com.yahoo.vespa.config.PayloadChecksum;
import com.yahoo.vespa.config.ConfigDefinition;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ErrorCode;
@@ -9,6 +9,7 @@ import com.yahoo.vespa.config.ErrorCode;
import java.util.logging.Logger;
import java.util.regex.Matcher;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
import static java.util.logging.Level.INFO;
/**
@@ -29,12 +30,12 @@ public class RequestValidation {
log.log(INFO, "Illegal name space '" + key.getNamespace() + "'");
return ErrorCode.ILLEGAL_NAME_SPACE;
}
- if (!(new PayloadChecksum(request.getRequestDefMd5()).valid())) {
+ if (!(new PayloadChecksum(request.getRequestDefMd5(), MD5).valid())) {
log.log(INFO, "Illegal checksum '" + key.getNamespace() + "'");
return ErrorCode.ILLEGAL_DEF_MD5; // TODO: Use ILLEGAL_DEF_CHECKSUM
}
- if (!new PayloadChecksum(request.getRequestConfigMd5()).valid()) {
- log.log(INFO, "Illegal config checksum '" + request.getRequestConfigMd5() + "'");
+ if (! request.getRequestConfigChecksums().valid()) {
+ log.log(INFO, "Illegal config checksum '" + request.getRequestConfigChecksums() + "'");
return ErrorCode.ILLEGAL_CONFIG_MD5; // TODO: Use ILLEGAL_CONFIG_CHECKSUM
}
if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java
index 1ccf6e367fc..8d08717942b 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.text.AbstractUtf8Array;
import com.yahoo.vespa.config.ConfigPayload;
@@ -8,6 +9,8 @@ import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+
/**
* Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface.
*
@@ -19,25 +22,42 @@ public class SlimeConfigResponse implements ConfigResponse {
private final CompressionInfo compressionInfo;
private final long generation;
private final boolean applyOnRestart;
- private final String configMd5;
+ private final PayloadChecksums payloadChecksums;
+
+ public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload,
+ long generation,
+ boolean applyOnRestart,
+ PayloadChecksums payloadChecksums) {
+ AbstractUtf8Array data = payload.toUtf8Array(true);
+ return new SlimeConfigResponse(data,
+ generation,
+ applyOnRestart,
+ payloadChecksums,
+ CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength()));
+ }
- public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, long generation,
- boolean applyOnRestart, String configMd5) {
+ // TODO: Legacy method, remove when not used anymore
+ public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload,
+ long generation,
+ boolean applyOnRestart,
+ String configMd5) {
AbstractUtf8Array data = payload.toUtf8Array(true);
- return new SlimeConfigResponse(data, generation, applyOnRestart,
- configMd5,
+ return new SlimeConfigResponse(data,
+ generation,
+ applyOnRestart,
+ PayloadChecksums.from(configMd5, ""),
CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength()));
}
public SlimeConfigResponse(AbstractUtf8Array payload,
long generation,
boolean applyOnRestart,
- String configMd5,
+ PayloadChecksums payloadChecksums,
CompressionInfo compressionInfo) {
this.payload = payload;
this.generation = generation;
this.applyOnRestart = applyOnRestart;
- this.configMd5 = configMd5;
+ this.payloadChecksums = payloadChecksums;
this.compressionInfo = compressionInfo;
}
@@ -56,7 +76,7 @@ public class SlimeConfigResponse implements ConfigResponse {
@Override
public String getConfigMd5() {
- return configMd5;
+ return payloadChecksums.getForType(MD5).asString();
}
@Override
@@ -68,11 +88,13 @@ public class SlimeConfigResponse implements ConfigResponse {
@Override
public String toString() {
return "generation=" + generation + "\n" +
- "configmd5=" + configMd5 + "\n" +
+ "checksums=" + payloadChecksums + "\n" +
Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED);
}
@Override
public CompressionInfo getCompressionInfo() { return compressionInfo; }
+ @Override
+ public PayloadChecksums getPayloadChecksums() { return payloadChecksums; }
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java
index b885623a78b..679298bac73 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java
@@ -1,6 +1,8 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
+import com.yahoo.vespa.config.PayloadChecksum;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.Request;
import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
@@ -11,6 +13,9 @@ import com.yahoo.vespa.config.util.ConfigUtils;
import java.util.Optional;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
+
/**
* Contains slime request data objects. Provides methods for reading various fields from slime request data.
* All data is read lazily.
@@ -27,6 +32,7 @@ class SlimeRequestData {
private static final String REQUEST_CLIENT_HOSTNAME = "clientHostname";
private static final String REQUEST_CURRENT_GENERATION = "currentGeneration";
private static final String REQUEST_CONFIG_MD5 = "configMD5";
+ private static final String REQUEST_CONFIG_XXHASH64 = "configXxhash64";
private static final String REQUEST_TRACE = "trace";
private static final String REQUEST_TIMEOUT = "timeout";
private static final String REQUEST_DEF_MD5 = "defMD5";
@@ -79,6 +85,17 @@ class SlimeRequestData {
String getRequestDefMd5() { return getRequestField(REQUEST_DEF_MD5).asString(); }
+ PayloadChecksum getRequestConfigXxhash64() {
+ Inspector xxhash64Field = getRequestField(REQUEST_CONFIG_XXHASH64);
+ return xxhash64Field.valid()
+ ? new PayloadChecksum(xxhash64Field.asString(), XXHASH64)
+ : PayloadChecksum.empty(XXHASH64);
+ }
+
+ PayloadChecksums getRequestConfigChecksums() {
+ return PayloadChecksums.from(getRequestConfigMd5(), getRequestConfigXxhash64().asString());
+ }
+
long getRequestGeneration() {
return getRequestField(REQUEST_CURRENT_GENERATION).asLong();
}
@@ -86,7 +103,7 @@ class SlimeRequestData {
static Slime encodeRequest(ConfigKey<?> key,
String hostname,
DefContent defSchema,
- String configMd5,
+ PayloadChecksums payloadChecksums,
long generation,
long timeout,
Trace trace,
@@ -102,7 +119,8 @@ class SlimeRequestData {
request.setString(REQUEST_CLIENT_CONFIGID, key.getConfigId());
request.setString(REQUEST_CLIENT_HOSTNAME, hostname);
defSchema.serialize(request.setArray(REQUEST_DEF_CONTENT));
- request.setString(REQUEST_CONFIG_MD5, configMd5);
+ request.setString(REQUEST_CONFIG_MD5, payloadChecksums.getForType(MD5).asString());
+ request.setString(REQUEST_CONFIG_XXHASH64, payloadChecksums.getForType(XXHASH64).asString());
request.setLong(REQUEST_CURRENT_GENERATION, generation);
request.setLong(REQUEST_TIMEOUT, timeout);
request.setString(REQUEST_COMPRESSION_TYPE, compressionType.name());
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java
index cc98587456c..965622adaa5 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java
@@ -1,11 +1,16 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
+import com.yahoo.vespa.config.PayloadChecksum;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.Request;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
import com.yahoo.slime.SlimeUtils;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
+
/**
* Contains response data for a slime response and methods for decoding the response data that
* are common to all {@link Slime} based config requests.
@@ -22,6 +27,7 @@ class SlimeResponseData {
static final String RESPONSE_CLIENT_HOSTNAME = "clientHostname";
static final String RESPONSE_TRACE = "trace";
static final String RESPONSE_CONFIG_MD5 = "configMD5";
+ static final String RESPONSE_CONFIG_XXHASH64 = "configXxhash64";
static final String RESPONSE_CONFIG_GENERATION = "generation";
static final String RESPONSE_APPLY_ON_RESTART = "applyOnRestart";
static final String RESPONSE_COMPRESSION_INFO = "compressionInfo";
@@ -58,9 +64,25 @@ class SlimeResponseData {
return trace.valid() ? Trace.fromSlime(trace) : Trace.createDummy();
}
- String getResponseConfigMd5() {
- Inspector inspector = getResponseField(RESPONSE_CONFIG_MD5);
- return inspector.valid() ? inspector.asString() : "";
+ PayloadChecksum getResponseConfigMd5() {
+ Inspector md5Field = getResponseField(RESPONSE_CONFIG_MD5);
+ return md5Field.valid()
+ ? new PayloadChecksum(md5Field.asString(), MD5)
+ : PayloadChecksum.empty(MD5);
+ }
+
+ PayloadChecksum getResponseConfigXxhash64() {
+ Inspector xxhash64Field = getResponseField(RESPONSE_CONFIG_XXHASH64);
+ return xxhash64Field.valid()
+ ? new PayloadChecksum(xxhash64Field.asString(), XXHASH64)
+ : PayloadChecksum.empty(XXHASH64);
+ }
+
+
+ PayloadChecksums getResponseConfigChecksums() {
+ PayloadChecksum responseConfigMd5 = getResponseConfigMd5();
+ System.out.println(responseConfigMd5);
+ return PayloadChecksums.from(responseConfigMd5, getResponseConfigXxhash64());
}
CompressionInfo getCompressionInfo() {
diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
index a7fc8afcad9..329661bf7ae 100644
--- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
+++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
@@ -10,6 +10,8 @@ import com.yahoo.text.AbstractUtf8Array;
import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.config.ConfigPayload;
+import net.jpountz.xxhash.XXHash64;
+import net.jpountz.xxhash.XXHashFactory;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -94,6 +96,15 @@ public class ConfigUtils {
}
}
+ public static String getXxhash64(AbstractUtf8Array input) {
+ return getXxhash64(input.wrap());
+ }
+
+ public static String getXxhash64(ByteBuffer input) {
+ XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
+ return Long.toHexString(hasher.hash(input, 0)).toLowerCase();
+ }
+
/**
* Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing;
*
diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
index 7bdaeb7d367..919155a3944 100644
--- a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ConnectionPool;
import com.yahoo.vespa.config.ErrorCode;
import com.yahoo.vespa.config.ErrorType;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.vespa.config.TimingValues;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
import org.junit.Test;
@@ -139,7 +140,7 @@ public class JRTConfigRequesterTest {
ConfigSubscriber subscriber = new ConfigSubscriber();
final TimingValues timingValues = getTestTimingValues();
JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues);
- sub.setConfig(1L, false, config(), PayloadChecksum.empty());
+ sub.setConfig(1L, false, config(), PayloadChecksums.empty());
final MockConnection connection = new MockConnection(new ErrorResponseHandler());
JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues);
@@ -165,7 +166,7 @@ public class JRTConfigRequesterTest {
ConfigSubscriber subscriber = new ConfigSubscriber();
final TimingValues timingValues = getTestTimingValues();
JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues);
- sub.setConfig(1L, false, config(), PayloadChecksum.empty());
+ sub.setConfig(1L, false, config(), PayloadChecksums.empty());
final MockConnection connection = new MockConnection(new ErrorResponseHandler(com.yahoo.jrt.ErrorCode.TIMEOUT));
JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues);
@@ -179,7 +180,7 @@ public class JRTConfigRequesterTest {
ConfigSubscriber subscriber = new ConfigSubscriber();
final TimingValues timingValues = getTestTimingValues();
JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues);
- sub.setConfig(1L, false, config(), PayloadChecksum.empty());
+ sub.setConfig(1L, false, config(), PayloadChecksums.empty());
final MockConnection connection = new MockConnection(new ErrorResponseHandler(ErrorCode.UNKNOWN_DEFINITION));
JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues);
diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
index e1d11f82eea..96a2d976f51 100644
--- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
@@ -1,8 +1,9 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config;
import com.yahoo.text.Utf8String;
-import com.yahoo.vespa.config.protocol.*;
+import com.yahoo.vespa.config.protocol.CompressionInfo;
+import com.yahoo.vespa.config.protocol.Payload;
import com.yahoo.vespa.config.protocol.VespaVersion;
import com.yahoo.vespa.config.util.ConfigUtils;
import org.junit.Test;
@@ -11,6 +12,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.Optional;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.junit.Assert.assertEquals;
@@ -26,11 +29,11 @@ import static org.junit.Assert.assertThat;
public class RawConfigTest {
private static final ConfigKey<?> key = new ConfigKey<>("foo", "id", "bar");
- private static List<String> defContent = Arrays.asList("version=1", "anInt int");
+ private static final List<String> defContent = Arrays.asList("version=1", "anInt int");
private static final String defMd5 = ConfigUtils.getDefMd5FromRequest("", defContent);
- private static final String configMd5 = "012345";
- private static Payload payload = Payload.from(new Utf8String("anInt 1"), CompressionInfo.uncompressed());
- private static long generation = 1L;
+ private static final PayloadChecksums payloadChecksums = PayloadChecksums.from("012345", "");
+ private static final Payload payload = Payload.from(new Utf8String("anInt 1"), CompressionInfo.uncompressed());
+ private static final long generation = 1L;
@Test
public void basic() {
@@ -47,7 +50,7 @@ public class RawConfigTest {
RawConfig copiedConfig = new RawConfig(config);
assertEquals(config, copiedConfig);
- assertEquals("bar.foo," + defMd5 + ",id,,0,null", config.toString());
+ assertEquals("bar.foo," + defMd5 + ",id,MD5:,XXHASH64:,0,null", config.toString());
assertEquals(Optional.empty(), config.getVespaVersion());
}
@@ -61,14 +64,14 @@ public class RawConfigTest {
assertThat(config.hashCode(), is(not(new RawConfig(key, "a").hashCode()))); // different def md5
// different generation
- config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty());
- RawConfig config2 = new RawConfig(key, defMd5, payload, configMd5, 2L, false, defContent, Optional.empty());
+ config = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, defContent, Optional.empty());
+ RawConfig config2 = new RawConfig(key, defMd5, payload, payloadChecksums, 2L, false, defContent, Optional.empty());
assertThat(config, is(not(config2)));
assertThat(config.hashCode(), is(not(config2.hashCode())));
// different config md5 and with vespa version
final VespaVersion vespaVersion = VespaVersion.fromString("5.37.38");
- RawConfig config3 = new RawConfig(key, defMd5, payload, "9999", generation, false, defContent, Optional.of(vespaVersion));
+ RawConfig config3 = new RawConfig(key, defMd5, payload, PayloadChecksums.from("9999", ""), generation, false, defContent, Optional.of(vespaVersion));
assertThat(config, is(not(config3)));
assertThat(config.hashCode(), is(not(config3.hashCode())));
// Check that vespa version is set correctly
@@ -82,42 +85,43 @@ public class RawConfigTest {
assertNotEquals(config, key);
// errors
- RawConfig errorConfig1 = new RawConfig(key, defMd5, payload, configMd5, generation, false, 1, defContent, Optional.empty());
+ RawConfig errorConfig1 = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, 1, defContent, Optional.empty());
assertThat(errorConfig1, is(errorConfig1));
assertThat(config, is(not(errorConfig1)));
assertThat(config.hashCode(), is(not(errorConfig1.hashCode())));
assertThat(errorConfig1, is(errorConfig1));
- RawConfig errorConfig2 = new RawConfig(key, defMd5, payload, configMd5, generation, false, 2, defContent, Optional.empty());
+ RawConfig errorConfig2 = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, 2, defContent, Optional.empty());
assertThat(errorConfig1, is(not(errorConfig2)));
assertThat(errorConfig1.hashCode(), is(not(errorConfig2.hashCode())));
}
@Test
public void payload() {
- RawConfig config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty());
- assertThat(config.getPayload(), is(payload));
- assertThat(config.getConfigMd5(), is(configMd5));
- assertThat(config.getGeneration(), is(generation));
- assertThat(config.getDefContent(), is(defContent));
+ RawConfig config = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, defContent, Optional.empty());
+ assertEquals(config.getPayload(), payload);
+ assertEquals(config.getConfigMd5(), payloadChecksums.getForType(MD5).asString());
+ assertEquals(config.getPayloadChecksums().getForType(XXHASH64), payloadChecksums.getForType(XXHASH64));
+ assertEquals(config.getGeneration(), generation);
+ assertEquals(config.getDefContent(), defContent);
}
@Test
public void require_correct_defmd5() {
final String defMd5ForEmptyDefContent = "d41d8cd98f00b204e9800998ecf8427e";
- RawConfig config = new RawConfig(key, null, payload, configMd5, generation, false, defContent, Optional.empty());
+ RawConfig config = new RawConfig(key, null, payload, payloadChecksums, generation, false, defContent, Optional.empty());
assertThat(config.getDefMd5(), is(defMd5));
- config = new RawConfig(key, "", payload, configMd5, generation, false, defContent, Optional.empty());
+ config = new RawConfig(key, "", payload, payloadChecksums, generation, false, defContent, Optional.empty());
assertThat(config.getDefMd5(), is(defMd5));
- config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty());
+ config = new RawConfig(key, defMd5, payload, payloadChecksums, generation, false, defContent, Optional.empty());
assertThat(config.getDefMd5(), is(defMd5));
- config = new RawConfig(key, null, payload, configMd5, generation, false, null, Optional.empty());
+ config = new RawConfig(key, null, payload, payloadChecksums, generation, false, null, Optional.empty());
assertNull(config.getDefMd5());
- config = new RawConfig(key, null, payload, configMd5, generation, false, List.of(""), Optional.empty());
+ config = new RawConfig(key, null, payload, payloadChecksums, generation, false, List.of(""), Optional.empty());
assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent));
- config = new RawConfig(key, "", payload, configMd5, generation, false, null, Optional.empty());
+ config = new RawConfig(key, "", payload, payloadChecksums, generation, false, null, Optional.empty());
assertThat(config.getDefMd5(), is(""));
- config = new RawConfig(key, "", payload, configMd5, generation, false, List.of(""), Optional.empty());
+ config = new RawConfig(key, "", payload, payloadChecksums, generation, false, List.of(""), Optional.empty());
assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent));
}
diff --git a/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java b/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java
index 8c11db15f7c..2b7fd9c350b 100644
--- a/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java
@@ -1,10 +1,11 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config;
-import com.yahoo.config.subscription.impl.PayloadChecksum;
import com.yahoo.vespa.config.protocol.RequestValidation;
import org.junit.Test;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -21,11 +22,12 @@ public class RequestValidationTest {
@Test
public void testVerifyDefMd5() {
- assertTrue(PayloadChecksum.empty().valid());
- assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d3f62d7ff777fce6b").valid());
- assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d3f62d7ff777fce6B").valid());
- assertFalse(new PayloadChecksum("aaaaaaaaaaaaaaaaaa").valid());
- assertFalse(new PayloadChecksum("-8f0c01c7c3dcb8d3f62d7ff777fce6b").valid());
+ assertTrue(PayloadChecksum.empty(MD5).valid());
+ assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d3f62d7ff777fce6b", MD5).valid());
+ assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d3f62d7ff777fce6B", MD5).valid());
+ assertTrue(new PayloadChecksum("e8f0c01c7c3dcb8d", XXHASH64).valid());
+ assertFalse(new PayloadChecksum("aaaaaaaaaaaaaaaaaa", MD5).valid());
+ assertFalse(new PayloadChecksum("-8f0c01c7c3dcb8d3f62d7ff777fce6b", MD5).valid());
}
@Test
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java
index c53a6b5c73d..f00e95ccea2 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java
@@ -1,6 +1,7 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.protocol;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.foo.SimpletypesConfig;
import com.yahoo.text.AbstractUtf8Array;
import com.yahoo.text.Utf8Array;
@@ -12,6 +13,8 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -23,14 +26,20 @@ public class ConfigResponseTest {
@Test
public void require_that_slime_response_is_initialized() throws IOException {
ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
- ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, 3, false, "mymd5");
+ PayloadChecksums payloadChecksums = PayloadChecksums.fromPayload(Payload.from(configPayload));
+ ConfigResponse response =
+ SlimeConfigResponse.fromConfigPayload(configPayload,
+ 3,
+ false,
+ payloadChecksums);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
response.serialize(baos, CompressionType.UNCOMPRESSED);
String payload = baos.toString(StandardCharsets.UTF_8);
assertNotNull(payload);
- assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", payload.toString());
- assertEquals(response.getGeneration(), 3L);
- assertEquals(response.getConfigMd5(), "mymd5");
+ assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", payload);
+ assertEquals(3L, response.getGeneration());
+ assertEquals(payloadChecksums.getForType(MD5), response.getPayloadChecksums().getForType(MD5));
+ assertEquals(payloadChecksums.getForType(XXHASH64), response.getPayloadChecksums().getForType(XXHASH64));
baos = new ByteArrayOutputStream();
response.serialize(baos, CompressionType.UNCOMPRESSED);
@@ -42,7 +51,7 @@ public class ConfigResponseTest {
ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
AbstractUtf8Array data = configPayload.toUtf8Array(true);
Utf8Array bytes = new Utf8Array(new LZ4PayloadCompressor().compress(data.wrap()));
- ConfigResponse response = new SlimeConfigResponse(bytes, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength()));
+ ConfigResponse response = new SlimeConfigResponse(bytes, 3, false, PayloadChecksums.empty(), CompressionInfo.create(CompressionType.LZ4, data.getByteLength()));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
response.serialize(baos, CompressionType.UNCOMPRESSED);
String payload = baos.toString(StandardCharsets.UTF_8);
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
index d6ce246aa1f..5a3110c9221 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
@@ -8,6 +8,7 @@ import com.yahoo.config.subscription.impl.GenericConfigSubscriber;
import com.yahoo.config.subscription.impl.JRTConfigRequester;
import com.yahoo.config.subscription.impl.JRTConfigSubscription;
import com.yahoo.config.subscription.impl.MockConnection;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.foo.SimpletypesConfig;
import com.yahoo.jrt.Request;
import com.yahoo.slime.Inspector;
@@ -27,6 +28,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -53,7 +56,7 @@ public class JRTConfigRequestV3Test {
private final long currentGeneration = 3;
private final long timeout = 5000;
private Trace trace ;
- private final String configMd5 = ConfigUtils.getMd5(createPayload().getData());
+ private final PayloadChecksums payloadChecksums = PayloadChecksums.fromPayload(createPayload());
private JRTClientConfigRequest clientReq;
private JRTServerConfigRequest serverReq;
@@ -79,8 +82,12 @@ public class JRTConfigRequestV3Test {
@Test
public void emptypayload() {
ConfigPayload payload = ConfigPayload.empty();
- SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, 0, false, ConfigUtils.getMd5(payload));
- serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), false, response.getConfigMd5());
+ PayloadChecksums payloadChecksums = PayloadChecksums.fromPayload(Payload.from(payload));
+ SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload,
+ 0,
+ false,
+ payloadChecksums);
+ serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), false, payloadChecksums);
assertTrue(clientReq.validateResponse());
assertTrue(clientReq.hasUpdatedGeneration());
assertEquals("{}", clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString());
@@ -97,7 +104,8 @@ public class JRTConfigRequestV3Test {
@Test
public void next_request_when_error_is_correct() {
- serverReq.addOkResponse(createPayload(), 999999, false, "newmd5");
+ Payload payload = createPayload();
+ serverReq.addOkResponse(payload, 999999, false, PayloadChecksums.fromPayload(payload));
serverReq.addErrorResponse(ErrorCode.OUTDATED_CONFIG, "error message");
JRTClientConfigRequest next = clientReq.nextRequest(6);
// Should use config md5 and generation from the request, not the response
@@ -111,7 +119,7 @@ public class JRTConfigRequestV3Test {
Payload payload = createPayload("vale");
String md5 = ConfigUtils.getMd5(payload.getData());
long generation = 4L;
- serverReq.addOkResponse(payload, generation, false, md5);
+ serverReq.addOkResponse(payload, generation, false, PayloadChecksums.fromPayload(payload));
assertTrue(clientReq.validateResponse());
assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is(payload.getData().toString()));
assertThat(clientReq.getNewGeneration(), is(4L));
@@ -137,7 +145,7 @@ public class JRTConfigRequestV3Test {
@Test
public void generation_only_is_updated() {
Payload payload = createPayload();
- serverReq.addOkResponse(payload, 4L, false, ConfigUtils.getMd5(payload.getData()));
+ serverReq.addOkResponse(payload, 4L, false, PayloadChecksums.fromPayload(payload));
boolean value = clientReq.validateResponse();
assertTrue(clientReq.errorMessage(), value);
assertFalse(clientReq.hasUpdatedConfig());
@@ -147,7 +155,7 @@ public class JRTConfigRequestV3Test {
@Test
public void nothing_is_updated() {
Payload payload = createPayload();
- serverReq.addOkResponse(payload, currentGeneration, false, configMd5);
+ serverReq.addOkResponse(payload, currentGeneration, false, payloadChecksums);
assertTrue(clientReq.validateResponse());
assertFalse(clientReq.hasUpdatedConfig());
assertFalse(clientReq.hasUpdatedGeneration());
@@ -158,7 +166,7 @@ public class JRTConfigRequestV3Test {
Payload payload = Payload.from(ConfigPayload.empty());
clientReq = createReq(payload);
serverReq = createReq(clientReq.getRequest());
- serverReq.addOkResponse(payload, currentGeneration, false, ConfigUtils.getMd5(payload.getData()));
+ serverReq.addOkResponse(payload, currentGeneration, false, PayloadChecksums.fromPayload(payload));
boolean val = clientReq.validateResponse();
assertTrue(clientReq.errorMessage(), val);
assertFalse(clientReq.hasUpdatedConfig());
@@ -195,7 +203,7 @@ public class JRTConfigRequestV3Test {
@Override
public void createResponse() {
JRTServerConfigRequest serverRequest = createReq(request);
- serverRequest.addOkResponse(createPayload(), currentGeneration, false, configMd5);
+ serverRequest.addOkResponse(createPayload(), currentGeneration, false, payloadChecksums);
}
});
@@ -206,8 +214,10 @@ public class JRTConfigRequestV3Test {
assertTrue(sub.nextConfig(120_0000));
sub.close();
JRTClientConfigRequest nextReq = createReq(sub, Trace.createNew());
- assertThat(nextReq.getRequestConfigMd5(), is(sub.getConfigState().getChecksum().asString()));
- assertThat(nextReq.getRequestGeneration(), is(currentGeneration));
+ assertEquals(nextReq.getRequestConfigMd5(), sub.getConfigState().getChecksums().getForType(MD5).asString());
+ assertEquals(nextReq.getRequestConfigChecksums().getForType(MD5).asString(), sub.getConfigState().getChecksums().getForType(MD5).asString());
+ assertEquals(nextReq.getRequestConfigChecksums().getForType(XXHASH64).asString(), sub.getConfigState().getChecksums().getForType(XXHASH64).asString());
+ assertEquals(nextReq.getRequestGeneration(), currentGeneration);
}
@Test
@@ -225,12 +235,12 @@ public class JRTConfigRequestV3Test {
@Test
public void parameters_are_validated() {
assertTrue(serverReq.validateParameters());
- assertValidationFail(createReq("35#$#!$@#", defNamespace, hostname, configId, configMd5, currentGeneration, timeout, trace));
- assertValidationFail(createReq(defName, "abcd.o#$*(!&$", hostname, configId, configMd5, currentGeneration, timeout, trace));
- assertValidationFail(createReq(defName, defNamespace, hostname, configId, "34", currentGeneration, timeout, trace));
- assertValidationFail(createReq(defName, defNamespace, hostname, configId, configMd5, -34, timeout, trace));
- assertValidationFail(createReq(defName, defNamespace, hostname, configId, configMd5, currentGeneration, -23, trace));
- assertValidationFail(createReq(defName, defNamespace, "", configId, configMd5, currentGeneration, timeout, trace));
+ assertValidationFail(createReq("35#$#!$@#", defNamespace, hostname, configId, payloadChecksums, currentGeneration, timeout, trace));
+ assertValidationFail(createReq(defName, "abcd.o#$*(!&$", hostname, configId, payloadChecksums, currentGeneration, timeout, trace));
+ assertValidationFail(createReq(defName, defNamespace, hostname, configId, PayloadChecksums.from("abcd", "1234"), currentGeneration, timeout, trace));
+ assertValidationFail(createReq(defName, defNamespace, hostname, configId, payloadChecksums, -34, timeout, trace));
+ assertValidationFail(createReq(defName, defNamespace, hostname, configId, payloadChecksums, currentGeneration, -23, trace));
+ assertValidationFail(createReq(defName, defNamespace, "", configId, payloadChecksums, currentGeneration, timeout, trace));
}
private void assertValidationFail(JRTClientConfigRequest req) {
@@ -248,12 +258,12 @@ public class JRTConfigRequestV3Test {
}
private JRTClientConfigRequest createReq(String defName, String defNamespace,
- String hostname, String configId, String configMd5,
+ String hostname, String configId, PayloadChecksums payloadChecksums,
long currentGeneration, long timeout, Trace trace) {
return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace),
DefContent.fromList(List.of(configDefinition)),
hostname,
- configMd5,
+ payloadChecksums,
currentGeneration,
timeout,
trace,
@@ -276,13 +286,13 @@ public class JRTConfigRequestV3Test {
private JRTClientConfigRequest createReq() {
trace = Trace.createNew(3, new ManualClock());
trace.trace(1, "hei");
- return createReq(defName, defNamespace, hostname, configId, configMd5, currentGeneration, timeout, trace);
+ return createReq(defName, defNamespace, hostname, configId, payloadChecksums, currentGeneration, timeout, trace);
}
private JRTClientConfigRequest createReq(Payload payload) {
trace = Trace.createNew(3, new ManualClock());
trace.trace(1, "hei");
- return createReq(defName, defNamespace, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace);
+ return createReq(defName, defNamespace, hostname, configId, PayloadChecksums.fromPayload(payload), currentGeneration, timeout, trace);
}
private void request_is_parsed_base() {
@@ -294,7 +304,8 @@ public class JRTConfigRequestV3Test {
assertThat(serverReq.getDefContent().asStringArray(), is(configDefinition));
assertFalse(serverReq.noCache());
assertTrue(serverReq.getRequestTrace().toString().contains("hi"));
- assertThat(serverReq.getRequestConfigMd5(), is(configMd5));
+ assertThat(serverReq.getRequestConfigChecksums().getForType(MD5), is(payloadChecksums.getForType(MD5)));
+ assertThat(serverReq.getRequestConfigChecksums().getForType(XXHASH64), is(payloadChecksums.getForType(XXHASH64)));
assertThat(serverReq.getRequestGeneration(), is(currentGeneration));
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
index 820f5c15318..bad03862133 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
@@ -4,8 +4,8 @@ package com.yahoo.vespa.config.server.rpc;
import com.yahoo.cloud.config.SentinelConfig;
import com.yahoo.collections.Pair;
import com.yahoo.component.Version;
-import com.yahoo.config.ConfigInstance;
import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.Request;
import com.yahoo.net.HostName;
import com.yahoo.vespa.config.ConfigPayload;
@@ -13,18 +13,19 @@ import com.yahoo.vespa.config.ErrorCode;
import com.yahoo.vespa.config.UnknownConfigIdException;
import com.yahoo.vespa.config.protocol.ConfigResponse;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
-import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
+import com.yahoo.vespa.config.protocol.Payload;
import com.yahoo.vespa.config.protocol.Trace;
import com.yahoo.vespa.config.protocol.VespaVersion;
import com.yahoo.vespa.config.server.GetConfigContext;
import com.yahoo.vespa.config.server.UnknownConfigDefinitionException;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.config.util.ConfigUtils;
import java.util.Optional;
import java.util.logging.Level;
import java.util.logging.Logger;
+import static com.yahoo.vespa.config.protocol.SlimeConfigResponse.fromConfigPayload;
+
/**
* @author hmusum
*/
@@ -126,7 +127,7 @@ class GetConfigProcessor implements Runnable {
// config == null is not an error, but indicates that the config will be returned later.
if ((config != null) && (!config.hasEqualConfig(request) || config.hasNewerGeneration(request) || forceResponse)) {
// debugLog(trace, "config response before encoding:" + config.toString());
- request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.applyOnRestart(), config.getConfigMd5());
+ request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.applyOnRestart(), config.getPayloadChecksums());
if (logDebug(trace)) {
debugLog(trace, "return response: " + request.getShortDescription());
}
@@ -166,9 +167,11 @@ class GetConfigProcessor implements Runnable {
private void returnEmpty(JRTServerConfigRequest request) {
log.log(Level.FINE, () -> "Returning empty sentinel config for request from " + request.getClientHostName());
var emptyPayload = ConfigPayload.fromInstance(new SentinelConfig.Builder().build());
- String configMd5 = ConfigUtils.getMd5(emptyPayload);
- ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5);
- request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getConfigMd5());
+ ConfigResponse config = fromConfigPayload(emptyPayload,
+ 0,
+ false,
+ PayloadChecksums.fromPayload(Payload.from(emptyPayload)));
+ request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getPayloadChecksums());
respond(request);
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
index f309b30cf8d..3698a50217a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
@@ -1,6 +1,7 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.rpc;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.text.AbstractUtf8Array;
import com.yahoo.text.Utf8Array;
import com.yahoo.vespa.config.LZ4PayloadCompressor;
@@ -23,10 +24,10 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory {
public ConfigResponse createResponse(AbstractUtf8Array rawPayload,
long generation,
boolean applyOnRestart) {
- String configMd5 = ConfigUtils.getMd5(rawPayload);
CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength());
Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.wrap()));
- return new SlimeConfigResponse(compressed, generation, applyOnRestart, configMd5, info);
+ PayloadChecksums payloadChecksums = PayloadChecksums.from(ConfigUtils.getMd5(rawPayload), ConfigUtils.getXxhash64(rawPayload));
+ return new SlimeConfigResponse(compressed, generation, applyOnRestart, payloadChecksums, info);
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
index 889548196aa..97db683e348 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
@@ -1,6 +1,7 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.rpc;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.text.AbstractUtf8Array;
import com.yahoo.vespa.config.protocol.CompressionInfo;
import com.yahoo.vespa.config.protocol.CompressionType;
@@ -16,12 +17,11 @@ import com.yahoo.vespa.config.util.ConfigUtils;
public class UncompressedConfigResponseFactory implements ConfigResponseFactory {
@Override
- public ConfigResponse createResponse(AbstractUtf8Array rawPayload,
- long generation,
- boolean applyOnRestart) {
+ public ConfigResponse createResponse(AbstractUtf8Array rawPayload, long generation, boolean applyOnRestart) {
String configMd5 = ConfigUtils.getMd5(rawPayload);
+ String xxHash64 = ConfigUtils.getXxhash64(rawPayload);
CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength());
- return new SlimeConfigResponse(rawPayload, generation, applyOnRestart, configMd5, info);
+ return new SlimeConfigResponse(rawPayload, generation, applyOnRestart, PayloadChecksums.from(configMd5, xxHash64), info);
}
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
index eb1e541a540..5016107c411 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server;
import com.yahoo.cloud.config.LbServicesConfig;
@@ -13,11 +13,11 @@ import com.yahoo.config.provision.ApplicationName;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.Request;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.protocol.CompressionType;
import com.yahoo.vespa.config.protocol.DefContent;
-import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
import com.yahoo.vespa.config.protocol.Trace;
import com.yahoo.vespa.config.server.model.SuperModelConfigProvider;
@@ -36,6 +36,7 @@ import java.util.Map;
import java.util.Optional;
import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER;
+import static com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3.createWithParams;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -73,11 +74,12 @@ public class SuperModelControllerTest {
@Test(expected = UnknownConfigDefinitionException.class)
public void test_unknown_config_definition() {
- String md5 = "asdfasf";
- Request request = JRTClientConfigRequestV3.createWithParams(new ConfigKey<>("foo", "id", "bar", null), DefContent.fromList(Collections.emptyList()),
- "fromHost", md5, 1, 1, Trace.createDummy(), CompressionType.UNCOMPRESSED,
- Optional.empty())
- .getRequest();
+ PayloadChecksums payloadChecksums = PayloadChecksums.empty();
+ Request request = createWithParams(new ConfigKey<>("foo", "id", "bar", null),
+ DefContent.fromList(Collections.emptyList()), "fromHost",
+ payloadChecksums, 1, 1, Trace.createDummy(),
+ CompressionType.UNCOMPRESSED, Optional.empty())
+ .getRequest();
JRTServerConfigRequestV3 v3Request = JRTServerConfigRequestV3.createFromRequest(request);
handler.resolveConfig(v3Request);
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
index 44491667760..a7fc69d56df 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
@@ -11,6 +11,7 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ApplicationName;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.Request;
import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.ConfigDefinitionKey;
@@ -148,10 +149,13 @@ public class ApplicationTest {
}
private static GetConfigRequest createRequest(String name, String namespace, String[] schema) {
- Request request = JRTClientConfigRequestV3.
- createWithParams(new ConfigKey<>(name, "admin/model", namespace, null), DefContent.fromArray(schema),
- "fromHost", "", 0, 100, Trace.createDummy(), CompressionType.UNCOMPRESSED,
- Optional.empty()).getRequest();
+ Request request =
+ JRTClientConfigRequestV3.createWithParams(new ConfigKey<>(name, "admin/model", namespace, null),
+ DefContent.fromArray(schema), "fromHost",
+ PayloadChecksums.empty(), 0, 100,
+ Trace.createDummy(), CompressionType.UNCOMPRESSED,
+ Optional.empty())
+ .getRequest();
return JRTServerConfigRequestV3.createFromRequest(request);
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
index 738e8c9827d..21e1c1f3448 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
@@ -2,12 +2,11 @@
package com.yahoo.vespa.config.server.rpc;
import com.yahoo.config.provision.ApplicationId;
-
+import com.yahoo.vespa.config.PayloadChecksums;
import com.yahoo.jrt.Request;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.protocol.CompressionType;
import com.yahoo.vespa.config.protocol.DefContent;
-import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
import com.yahoo.vespa.config.protocol.Trace;
@@ -22,6 +21,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import static com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3.createWithParams;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
@@ -41,7 +41,7 @@ public class DelayedConfigResponseTest {
MockRpcServer rpc = new MockRpcServer(13337, temporaryFolder.newFolder());
DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
assertThat(responses.size(), is(0));
- JRTServerConfigRequest req = createRequest("foo", "myid", "mymd5", 3, 1000000, "bar");
+ JRTServerConfigRequest req = createRequest("foo", "myid", 3, 1000000, "bar");
req.setDelayedResponse(true);
GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId());
responses.delayResponse(req, context);
@@ -49,7 +49,7 @@ public class DelayedConfigResponseTest {
req.setDelayedResponse(false);
responses.delayResponse(req, context);
- responses.delayResponse(createRequest("foolio", "myid", "mymd5", 3, 100000, "bar"), context);
+ responses.delayResponse(createRequest("foolio", "myid", 3, 100000, "bar"), context);
assertThat(responses.size(), is(2));
assertTrue(req.isDelayedResponse());
List<DelayedConfigResponses.DelayedConfigResponse> it = responses.allDelayedResponses();
@@ -61,7 +61,7 @@ public class DelayedConfigResponseTest {
GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId());
MockRpcServer rpc = new MockRpcServer(13337, temporaryFolder.newFolder());
DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
- responses.delayResponse(createRequest("foolio", "myid", "mymd5", 3, 100000, "bar"), context);
+ responses.delayResponse(createRequest("foolio", "myid", 3, 100000, "bar"), context);
assertThat(responses.size(), is(1));
responses.allDelayedResponses().get(0).cancelAndRemove();
assertThat(responses.size(), is(0));
@@ -73,17 +73,18 @@ public class DelayedConfigResponseTest {
DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
assertThat(responses.size(), is(0));
assertThat(responses.toString(), is("DelayedConfigResponses. Average Size=0"));
- JRTServerConfigRequest req = createRequest("foo", "myid", "mymd5", 3, 100, "bar");
+ JRTServerConfigRequest req = createRequest("foo", "myid", 3, 100, "bar");
responses.delayResponse(req, GetConfigContext.testContext(ApplicationId.defaultId()));
rpc.waitUntilSet(Duration.ofSeconds(5));
assertThat(rpc.latestRequest, is(req));
}
- private JRTServerConfigRequest createRequest(String configName, String configId, String md5, long generation, long timeout, String namespace) {
- Request request = JRTClientConfigRequestV3.
- createWithParams(new ConfigKey<>(configName, configId, namespace, null), DefContent.fromList(Collections.emptyList()),
- "fromHost", md5, generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED,
- Optional.empty()).getRequest();
+ private JRTServerConfigRequest createRequest(String configName, String configId, long generation, long timeout, String namespace) {
+ Request request = createWithParams(new ConfigKey<>(configName, configId, namespace, null),
+ DefContent.fromList(Collections.emptyList()), "fromHost",
+ PayloadChecksums.empty(), generation, timeout, Trace.createDummy(),
+ CompressionType.UNCOMPRESSED, Optional.empty())
+ .getRequest();
return JRTServerConfigRequestV3.createFromRequest(request);
}