Use both xxhash64 and md5 for config payload in requests and responses

Chacking of equal config with hasEqualConfig() still uses only md5

6 files changed, 48 insertions, 37 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
index 820f5c15318..bad03862133 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
@@ -4,8 +4,8 @@ package com.yahoo.vespa.config.server.rpc;
 import com.yahoo.cloud.config.SentinelConfig;
 import com.yahoo.collections.Pair;
 import com.yahoo.component.Version;
-import com.yahoo.config.ConfigInstance;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.jrt.Request;
 import com.yahoo.net.HostName;
 import com.yahoo.vespa.config.ConfigPayload;
@@ -13,18 +13,19 @@ import com.yahoo.vespa.config.ErrorCode;
 import com.yahoo.vespa.config.UnknownConfigIdException;
 import com.yahoo.vespa.config.protocol.ConfigResponse;
 import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
-import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
+import com.yahoo.vespa.config.protocol.Payload;
 import com.yahoo.vespa.config.protocol.Trace;
 import com.yahoo.vespa.config.protocol.VespaVersion;
 import com.yahoo.vespa.config.server.GetConfigContext;
 import com.yahoo.vespa.config.server.UnknownConfigDefinitionException;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.config.util.ConfigUtils;
 
 import java.util.Optional;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import static com.yahoo.vespa.config.protocol.SlimeConfigResponse.fromConfigPayload;
+
 /**
 * @author hmusum
 */
@@ -126,7 +127,7 @@ class GetConfigProcessor implements Runnable {
         // config == null is not an error, but indicates that the config will be returned later.
         if ((config != null) && (!config.hasEqualConfig(request) || config.hasNewerGeneration(request) || forceResponse)) {
             // debugLog(trace, "config response before encoding:" + config.toString());
-            request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.applyOnRestart(), config.getConfigMd5());
+            request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.applyOnRestart(), config.getPayloadChecksums());
             if (logDebug(trace)) {
                 debugLog(trace, "return response: " + request.getShortDescription());
             }
@@ -166,9 +167,11 @@ class GetConfigProcessor implements Runnable {
     private void returnEmpty(JRTServerConfigRequest request) {
         log.log(Level.FINE, () -> "Returning empty sentinel config for request from " + request.getClientHostName());
         var emptyPayload = ConfigPayload.fromInstance(new SentinelConfig.Builder().build());
-        String configMd5 = ConfigUtils.getMd5(emptyPayload);
-        ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5);
-        request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getConfigMd5());
+        ConfigResponse config = fromConfigPayload(emptyPayload,
+                                                  0,
+                                                  false,
+                                                  PayloadChecksums.fromPayload(Payload.from(emptyPayload)));
+        request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getPayloadChecksums());
         respond(request);
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
index f309b30cf8d..3698a50217a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
@@ -1,6 +1,7 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.text.AbstractUtf8Array;
 import com.yahoo.text.Utf8Array;
 import com.yahoo.vespa.config.LZ4PayloadCompressor;
@@ -23,10 +24,10 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory {
     public ConfigResponse createResponse(AbstractUtf8Array rawPayload,
                                          long generation,
                                          boolean applyOnRestart) {
-        String configMd5 = ConfigUtils.getMd5(rawPayload);
         CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength());
         Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.wrap()));
-        return new SlimeConfigResponse(compressed, generation, applyOnRestart, configMd5, info);
+        PayloadChecksums payloadChecksums = PayloadChecksums.from(ConfigUtils.getMd5(rawPayload), ConfigUtils.getXxhash64(rawPayload));
+        return new SlimeConfigResponse(compressed, generation, applyOnRestart, payloadChecksums, info);
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
index 889548196aa..97db683e348 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
@@ -1,6 +1,7 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.text.AbstractUtf8Array;
 import com.yahoo.vespa.config.protocol.CompressionInfo;
 import com.yahoo.vespa.config.protocol.CompressionType;
@@ -16,12 +17,11 @@ import com.yahoo.vespa.config.util.ConfigUtils;
 public class UncompressedConfigResponseFactory implements ConfigResponseFactory {
 
     @Override
-    public ConfigResponse createResponse(AbstractUtf8Array rawPayload,
-                                         long generation,
-                                         boolean applyOnRestart) {
+    public ConfigResponse createResponse(AbstractUtf8Array rawPayload, long generation, boolean applyOnRestart) {
         String configMd5 = ConfigUtils.getMd5(rawPayload);
+        String xxHash64 = ConfigUtils.getXxhash64(rawPayload);
         CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength());
-        return new SlimeConfigResponse(rawPayload, generation, applyOnRestart, configMd5, info);
+        return new SlimeConfigResponse(rawPayload, generation, applyOnRestart, PayloadChecksums.from(configMd5, xxHash64), info);
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
index eb1e541a540..5016107c411 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server;
 
 import com.yahoo.cloud.config.LbServicesConfig;
@@ -13,11 +13,11 @@ import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.jrt.Request;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.protocol.CompressionType;
 import com.yahoo.vespa.config.protocol.DefContent;
-import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3;
 import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
 import com.yahoo.vespa.config.protocol.Trace;
 import com.yahoo.vespa.config.server.model.SuperModelConfigProvider;
@@ -36,6 +36,7 @@ import java.util.Map;
 import java.util.Optional;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER;
+import static com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3.createWithParams;
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -73,11 +74,12 @@ public class SuperModelControllerTest {
 
     @Test(expected = UnknownConfigDefinitionException.class)
     public void test_unknown_config_definition() {
-        String md5 = "asdfasf";
-        Request request = JRTClientConfigRequestV3.createWithParams(new ConfigKey<>("foo", "id", "bar", null), DefContent.fromList(Collections.emptyList()),
-                                                                    "fromHost", md5, 1, 1, Trace.createDummy(), CompressionType.UNCOMPRESSED,
-                                                                    Optional.empty())
-                                                  .getRequest();
+        PayloadChecksums payloadChecksums = PayloadChecksums.empty();
+        Request request = createWithParams(new ConfigKey<>("foo", "id", "bar", null),
+                                           DefContent.fromList(Collections.emptyList()), "fromHost",
+                                           payloadChecksums, 1, 1, Trace.createDummy(),
+                                           CompressionType.UNCOMPRESSED, Optional.empty())
+                .getRequest();
         JRTServerConfigRequestV3 v3Request = JRTServerConfigRequestV3.createFromRequest(request);
         handler.resolveConfig(v3Request);
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
index 44491667760..a7fc69d56df 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
@@ -11,6 +11,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.jrt.Request;
 import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.ConfigDefinitionKey;
@@ -148,10 +149,13 @@ public class ApplicationTest {
     }
 
     private static GetConfigRequest createRequest(String name, String namespace, String[] schema) {
-        Request request = JRTClientConfigRequestV3.
-                createWithParams(new ConfigKey<>(name, "admin/model", namespace, null), DefContent.fromArray(schema),
-                                 "fromHost", "", 0, 100, Trace.createDummy(), CompressionType.UNCOMPRESSED,
-                                 Optional.empty()).getRequest();
+        Request request =
+                JRTClientConfigRequestV3.createWithParams(new ConfigKey<>(name, "admin/model", namespace, null),
+                                                          DefContent.fromArray(schema), "fromHost",
+                                                          PayloadChecksums.empty(), 0, 100,
+                                                          Trace.createDummy(), CompressionType.UNCOMPRESSED,
+                                                          Optional.empty())
+                                        .getRequest();
         return JRTServerConfigRequestV3.createFromRequest(request);
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
index 738e8c9827d..21e1c1f3448 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
@@ -2,12 +2,11 @@
 package com.yahoo.vespa.config.server.rpc;
 
 import com.yahoo.config.provision.ApplicationId;
-
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.jrt.Request;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.protocol.CompressionType;
 import com.yahoo.vespa.config.protocol.DefContent;
-import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3;
 import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
 import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
 import com.yahoo.vespa.config.protocol.Trace;
@@ -22,6 +21,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 
+import static com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3.createWithParams;
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
@@ -41,7 +41,7 @@ public class DelayedConfigResponseTest {
         MockRpcServer rpc = new MockRpcServer(13337, temporaryFolder.newFolder());
         DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
         assertThat(responses.size(), is(0));
-        JRTServerConfigRequest req = createRequest("foo", "myid", "mymd5", 3, 1000000, "bar");
+        JRTServerConfigRequest req = createRequest("foo", "myid", 3, 1000000, "bar");
         req.setDelayedResponse(true);
         GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId());
         responses.delayResponse(req, context);
@@ -49,7 +49,7 @@ public class DelayedConfigResponseTest {
 
         req.setDelayedResponse(false);
         responses.delayResponse(req, context);
-        responses.delayResponse(createRequest("foolio", "myid", "mymd5", 3, 100000, "bar"), context);
+        responses.delayResponse(createRequest("foolio", "myid", 3, 100000, "bar"), context);
         assertThat(responses.size(), is(2));
         assertTrue(req.isDelayedResponse());
         List<DelayedConfigResponses.DelayedConfigResponse> it = responses.allDelayedResponses();
@@ -61,7 +61,7 @@ public class DelayedConfigResponseTest {
         GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId());
         MockRpcServer rpc = new MockRpcServer(13337, temporaryFolder.newFolder());
         DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
-        responses.delayResponse(createRequest("foolio", "myid", "mymd5", 3, 100000, "bar"), context);
+        responses.delayResponse(createRequest("foolio", "myid", 3, 100000, "bar"), context);
         assertThat(responses.size(), is(1));
         responses.allDelayedResponses().get(0).cancelAndRemove();
         assertThat(responses.size(), is(0));
@@ -73,17 +73,18 @@ public class DelayedConfigResponseTest {
         DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
         assertThat(responses.size(), is(0));
         assertThat(responses.toString(), is("DelayedConfigResponses. Average Size=0"));
-        JRTServerConfigRequest req = createRequest("foo", "myid", "mymd5", 3, 100, "bar");
+        JRTServerConfigRequest req = createRequest("foo", "myid", 3, 100, "bar");
         responses.delayResponse(req, GetConfigContext.testContext(ApplicationId.defaultId()));
         rpc.waitUntilSet(Duration.ofSeconds(5));
         assertThat(rpc.latestRequest, is(req));
     }
 
-    private JRTServerConfigRequest createRequest(String configName, String configId, String md5, long generation, long timeout, String namespace) {
-        Request request = JRTClientConfigRequestV3.
-                createWithParams(new ConfigKey<>(configName, configId, namespace, null), DefContent.fromList(Collections.emptyList()),
-                                 "fromHost", md5, generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED,
-                                 Optional.empty()).getRequest();
+    private JRTServerConfigRequest createRequest(String configName, String configId, long generation, long timeout, String namespace) {
+        Request request = createWithParams(new ConfigKey<>(configName, configId, namespace, null),
+                                           DefContent.fromList(Collections.emptyList()), "fromHost",
+                                           PayloadChecksums.empty(), generation, timeout, Trace.createDummy(),
+                                           CompressionType.UNCOMPRESSED, Optional.empty())
+                .getRequest();
         return JRTServerConfigRequestV3.createFromRequest(request);
     }