Add checksum types in response based on types in request

Return response with same type as in request, except when both are
empty, then return both types.

Note: No clients have started sending requests with only xxhash64 checksum yet.

12 files changed, 158 insertions, 28 deletions

diff --git a/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java
index 4e90ce532e4..35b503416ce 100644
--- a/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java
+++ b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java
@@ -44,4 +44,11 @@ public interface GetConfigRequest {
      */
     String getRequestDefMd5();
 
+    /**
+     * Returns the payload checksums from the config request.
+     *
+     * @return the payload checksums from request.
+     */
+    PayloadChecksums configPayloadChecksums();
+
 }
diff --git a/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java
index bb3ac3f76f1..a16e28ac6b6 100644
--- a/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java
+++ b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java
@@ -1,10 +1,17 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config;
 
+import com.yahoo.text.AbstractUtf8Array;
+import com.yahoo.vespa.config.protocol.Payload;
+import com.yahoo.vespa.config.util.ConfigUtils;
+
 import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
+
 /**
  * Checksums of config definition payload or config payload,
  * md5 and xxhash64 are the supported types at the moment.
@@ -27,6 +34,24 @@ public class PayloadChecksum {
         return new PayloadChecksum("", type);
     }
 
+    public static PayloadChecksum fromPayload(Payload payload, Type type) {
+        switch (type) {
+            case MD5: return fromMd5Data(payload.getData());
+            case XXHASH64: return fromXxhash64Data(payload.getData());
+            default: throw new IllegalArgumentException("Unknown type " + type);
+        }
+    }
+
+    private static PayloadChecksum fromMd5Data(AbstractUtf8Array data) {
+        return new PayloadChecksum(ConfigUtils.getMd5(data), MD5);
+    }
+
+    private static PayloadChecksum fromXxhash64Data(AbstractUtf8Array data) {
+        return new PayloadChecksum(ConfigUtils.getXxhash64(data), XXHASH64);
+    }
+
+    public boolean isEmpty() { return checksum.isEmpty(); }
+
     public String asString() { return checksum; }
 
     public Type type() { return type; }
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java
index 41106e138b7..938da855014 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java
@@ -85,4 +85,12 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque
      */
     Payload payloadFromResponse(ConfigResponse response);
 
+
+    /**
+     * Returns the payload checksums from the config request.
+     *
+     * @return the payload checksumss from request.
+     */
+    PayloadChecksums configPayloadChecksums();
+
 }
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
index fbb52e81679..13d0ca1119a 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
@@ -273,7 +273,9 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest {
     }
 
     @Override
-    public Optional<VespaVersion> getVespaVersion() {
-        return requestData.getVespaVersion();
-    }
+    public Optional<VespaVersion> getVespaVersion() { return requestData.getVespaVersion(); }
+
+    @Override
+    public PayloadChecksums configPayloadChecksums() { return requestData.getRequestConfigChecksums(); }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java
index 00d010e75c8..7f39d678fdf 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java
@@ -44,7 +44,10 @@ public class SuperModelController {
     public ConfigResponse resolveConfig(GetConfigRequest request) {
         ConfigKey<?> configKey = request.getConfigKey();
         validateConfigDefinition(request.getConfigKey(), request.getDefContent());
-        return responseFactory.createResponse(model.getConfig(configKey).toUtf8Array(true), generation, false);
+        return responseFactory.createResponse(model.getConfig(configKey).toUtf8Array(true),
+                                              generation,
+                                              false,
+                                              request.configPayloadChecksums());
     }
 
     private void validateConfigDefinition(ConfigKey<?> configKey, DefContent defContent) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java
index df1427bdf6d..0b409d38196 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java
@@ -130,12 +130,11 @@ public class Application implements ModelResult {
             metricUpdater.incrementFailedRequests();
             throw new UnknownConfigDefinitionException("Unable to find config definition for '" + configKey.getNamespace() + "." + configKey.getName());
         }
-        log.log(Level.FINE, () -> TenantRepository.logPre(getId()) + ("Resolving " + configKey + " with config definition " + def));
+        log.log(Level.FINE, () -> TenantRepository.logPre(getId()) + "Resolving " + configKey + " with config definition " + def);
 
         var payload = createPayload(configKey, def);
-        var response = responseFactory.createResponse(payload.getFirst(), applicationGeneration, payload.getSecond());
 
-        return response;
+        return responseFactory.createResponse(payload.getFirst(), applicationGeneration, payload.getSecond(), req.configPayloadChecksums());
     }
 
     private Pair<AbstractUtf8Array, Boolean> createPayload(ConfigKey<?> configKey, ConfigDefinition def) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java
index c01008fafa0..8abb701606c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java
@@ -11,6 +11,7 @@ import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.jdisc.application.BindingMatch;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.GetConfigRequest;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.vespa.config.protocol.DefContent;
 import com.yahoo.vespa.config.protocol.VespaVersion;
 import com.yahoo.vespa.config.server.RequestHandler;
@@ -195,4 +196,7 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest {
     @Override
     public String getRequestDefMd5() { return ConfigUtils.getDefMd5(getDefContent().asList()); }
 
+    @Override
+    public PayloadChecksums configPayloadChecksums() { return PayloadChecksums.empty(); }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java
index 7afeebdd3cf..e52b2c4af01 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java
@@ -1,10 +1,16 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.text.AbstractUtf8Array;
 import com.yahoo.vespa.config.ConfigPayload;
+import com.yahoo.vespa.config.PayloadChecksum;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.vespa.config.protocol.ConfigResponse;
+import com.yahoo.vespa.config.util.ConfigUtils;
+
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
 
 /**
  * Represents a component that creates config responses from a payload. Different implementations
@@ -28,12 +34,37 @@ public interface ConfigResponseFactory {
     /**
      * Creates a {@link ConfigResponse} for a given payload and generation.
      *
-     * @param rawPayload     the {@link ConfigPayload} to put in the response
-     * @param generation     the payload generation
-     * @param applyOnRestart true if this config change should only be applied on restart,
-     *                       false if it should be applied immediately
+     * @param rawPayload                the {@link ConfigPayload} to put in the response
+     * @param generation                the payload generation
+     * @param applyOnRestart            true if this config change should only be applied on restart,
+     *                                  false if it should be applied immediately
+     * @param requestsPayloadChecksums  payload checksums from requests
      * @return a {@link ConfigResponse} that can be sent to the client
      */
-    ConfigResponse createResponse(AbstractUtf8Array rawPayload, long generation, boolean applyOnRestart);
+    ConfigResponse createResponse(AbstractUtf8Array rawPayload,
+                                  long generation,
+                                  boolean applyOnRestart,
+                                  PayloadChecksums requestsPayloadChecksums);
+
+    /** Generates payload checksums based on what type of checksums exist in request */
+    default PayloadChecksums generatePayloadChecksums(AbstractUtf8Array rawPayload, PayloadChecksums requestsPayloadChecksums) {
+        PayloadChecksum requestChecksumMd5 = requestsPayloadChecksums.getForType(MD5);
+        PayloadChecksum requestChecksumXxhash64 = requestsPayloadChecksums.getForType(XXHASH64);
+
+        PayloadChecksum md5 = PayloadChecksum.empty(MD5);
+        PayloadChecksum xxhash64 = PayloadChecksum.empty(XXHASH64);
+        // Response contains same checksum type as in request, except when both are empty,
+        // then use both checksum types in response
+        if (requestChecksumMd5.isEmpty() && requestChecksumXxhash64.isEmpty()
+                || ( ! requestChecksumMd5.isEmpty() && ! requestChecksumXxhash64.isEmpty())) {
+            md5 = new PayloadChecksum(ConfigUtils.getMd5(rawPayload), MD5);
+            xxhash64 = new PayloadChecksum(ConfigUtils.getXxhash64(rawPayload), XXHASH64);
+        } else if ( ! requestChecksumMd5.isEmpty())
+            md5 = new PayloadChecksum(ConfigUtils.getMd5(rawPayload), MD5);
+        else
+            xxhash64 = new PayloadChecksum(ConfigUtils.getXxhash64(rawPayload), XXHASH64);
+
+        return PayloadChecksums.from(md5, xxhash64);
+    }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
index 3698a50217a..6a1ecfac7bb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
@@ -1,15 +1,14 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
-import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.text.AbstractUtf8Array;
 import com.yahoo.text.Utf8Array;
 import com.yahoo.vespa.config.LZ4PayloadCompressor;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.vespa.config.protocol.CompressionInfo;
 import com.yahoo.vespa.config.protocol.CompressionType;
 import com.yahoo.vespa.config.protocol.ConfigResponse;
 import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
-import com.yahoo.vespa.config.util.ConfigUtils;
 
 /**
  * Compressor that compresses config payloads to lz4.
@@ -23,10 +22,11 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory {
     @Override
     public ConfigResponse createResponse(AbstractUtf8Array rawPayload,
                                          long generation,
-                                         boolean applyOnRestart) {
+                                         boolean applyOnRestart,
+                                         PayloadChecksums requestsPayloadChecksums) {
         CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength());
         Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.wrap()));
-        PayloadChecksums payloadChecksums = PayloadChecksums.from(ConfigUtils.getMd5(rawPayload), ConfigUtils.getXxhash64(rawPayload));
+        PayloadChecksums payloadChecksums = generatePayloadChecksums(rawPayload, requestsPayloadChecksums);
         return new SlimeConfigResponse(compressed, generation, applyOnRestart, payloadChecksums, info);
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
index 97db683e348..ce973e538b7 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
@@ -1,13 +1,12 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
-import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.text.AbstractUtf8Array;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.vespa.config.protocol.CompressionInfo;
 import com.yahoo.vespa.config.protocol.CompressionType;
 import com.yahoo.vespa.config.protocol.ConfigResponse;
 import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
-import com.yahoo.vespa.config.util.ConfigUtils;
 
 /**
  * Simply returns an uncompressed payload.
@@ -17,11 +16,13 @@ import com.yahoo.vespa.config.util.ConfigUtils;
 public class UncompressedConfigResponseFactory implements ConfigResponseFactory {
 
     @Override
-    public ConfigResponse createResponse(AbstractUtf8Array rawPayload, long generation, boolean applyOnRestart) {
-        String configMd5 = ConfigUtils.getMd5(rawPayload);
-        String xxHash64 = ConfigUtils.getXxhash64(rawPayload);
+    public ConfigResponse createResponse(AbstractUtf8Array rawPayload,
+                                         long generation,
+                                         boolean applyOnRestart,
+                                         PayloadChecksums requestsPayloadChecksums) {
         CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength());
-        return new SlimeConfigResponse(rawPayload, generation, applyOnRestart, PayloadChecksums.from(configMd5, xxHash64), info);
+        PayloadChecksums payloadChecksums = generatePayloadChecksums(rawPayload, requestsPayloadChecksums);
+        return new SlimeConfigResponse(rawPayload, generation, applyOnRestart, payloadChecksums, info);
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index 41af0296534..28d50a5396e 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -26,6 +26,7 @@ import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.ConfigPayload;
 import com.yahoo.vespa.config.GetConfigRequest;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.vespa.config.protocol.ConfigResponse;
 import com.yahoo.vespa.config.protocol.DefContent;
 import com.yahoo.vespa.config.protocol.VespaVersion;
@@ -810,6 +811,9 @@ public class ApplicationRepositoryTest {
             @Override
             public String getRequestDefMd5() { return ""; }
 
+            @Override
+            public PayloadChecksums configPayloadChecksums() { return PayloadChecksums.empty(); }
+
         }, Optional.empty());
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java
index 747a0ad3241..b164c3e5cd5 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java
@@ -1,11 +1,16 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
 import com.yahoo.vespa.config.ConfigPayload;
+import com.yahoo.vespa.config.PayloadChecksum;
+import com.yahoo.vespa.config.PayloadChecksums;
 import com.yahoo.vespa.config.protocol.CompressionType;
 import com.yahoo.vespa.config.protocol.ConfigResponse;
+import com.yahoo.vespa.config.protocol.Payload;
 import org.junit.Test;
 
+import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5;
+import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -13,22 +18,63 @@ import static org.junit.Assert.assertEquals;
  */
 public class ConfigResponseFactoryTest {
 
+    private static final ConfigPayload payload = ConfigPayload.fromString("{ \"field1\": 11, \"field2\": 11 }");
+
+    private static final PayloadChecksums payloadChecksums = PayloadChecksums.fromPayload(Payload.from(payload));
+    private static final PayloadChecksums payloadChecksumsEmpty = PayloadChecksums.empty();
+    private static final PayloadChecksums payloadChecksumsOnlyMd5 =
+            PayloadChecksums.from(PayloadChecksum.fromPayload(Payload.from(payload), MD5));
+    private static final PayloadChecksums payloadChecksumsOnlyXxhash64 =
+            PayloadChecksums.from(PayloadChecksum.fromPayload(Payload.from(payload), XXHASH64));
+
     @Test
     public void testUncompressedFactory() {
         UncompressedConfigResponseFactory responseFactory = new UncompressedConfigResponseFactory();
-        ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty().toUtf8Array(true), 3, false);
+        ConfigResponse response = responseFactory.createResponse(payload.toUtf8Array(true), 3, false, payloadChecksums);
         assertEquals(CompressionType.UNCOMPRESSED, response.getCompressionInfo().getCompressionType());
         assertEquals(3L,response.getGeneration());
-        assertEquals(2, response.getPayload().getByteLength());
+        assertEquals(25, response.getPayload().getByteLength());
+        assertEquals(payloadChecksums, response.getPayloadChecksums());
     }
 
     @Test
     public void testLZ4CompressedFactory() {
+        // Both checksums in request
+        {
+            ConfigResponse response = createResponse(payloadChecksums);
+            assertEquals(payloadChecksums, response.getPayloadChecksums());
+        }
+
+        // No checksums in request (empty checksums), both checksums should be in response
+        {
+            ConfigResponse response = createResponse(payloadChecksumsEmpty);
+            assertEquals(payloadChecksums.getForType(MD5), response.getPayloadChecksums().getForType(MD5));
+            assertEquals(payloadChecksums.getForType(XXHASH64), response.getPayloadChecksums().getForType(XXHASH64));
+        }
+
+        // Only md5 checksums in request
+        {
+            ConfigResponse response = createResponse(payloadChecksumsOnlyMd5);
+            assertEquals(payloadChecksumsOnlyMd5.getForType(MD5), response.getPayloadChecksums().getForType(MD5));
+            assertEquals(payloadChecksumsOnlyMd5.getForType(XXHASH64), response.getPayloadChecksums().getForType(XXHASH64));
+        }
+
+        // Only xxhash64 checksums in request
+        {
+            ConfigResponse response = createResponse(payloadChecksumsOnlyXxhash64);
+            assertEquals(payloadChecksumsOnlyXxhash64.getForType(MD5), response.getPayloadChecksums().getForType(MD5));
+            assertEquals(payloadChecksumsOnlyXxhash64.getForType(XXHASH64), response.getPayloadChecksums().getForType(XXHASH64));
+        }
+    }
+
+    private ConfigResponse createResponse(PayloadChecksums payloadChecksums) {
         LZ4ConfigResponseFactory responseFactory = new LZ4ConfigResponseFactory();
-        ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty().toUtf8Array(true), 3, false);
+        ConfigResponse response = responseFactory.createResponse(payload.toUtf8Array(true), 3, false, payloadChecksums);
         assertEquals(CompressionType.LZ4, response.getCompressionInfo().getCompressionType());
         assertEquals(3L, response.getGeneration());
-        assertEquals(3, response.getPayload().getByteLength());
+        assertEquals(23, response.getPayload().getByteLength());
+
+        return response;
     }
 
 }