diff options
12 files changed, 158 insertions, 28 deletions
diff --git a/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java index 4e90ce532e4..35b503416ce 100644 --- a/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java @@ -44,4 +44,11 @@ public interface GetConfigRequest { */ String getRequestDefMd5(); + /** + * Returns the payload checksums from the config request. + * + * @return the payload checksums from request. + */ + PayloadChecksums configPayloadChecksums(); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java index bb3ac3f76f1..a16e28ac6b6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java +++ b/config/src/main/java/com/yahoo/vespa/config/PayloadChecksum.java @@ -1,10 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; +import com.yahoo.text.AbstractUtf8Array; +import com.yahoo.vespa.config.protocol.Payload; +import com.yahoo.vespa.config.util.ConfigUtils; + import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; + /** * Checksums of config definition payload or config payload, * md5 and xxhash64 are the supported types at the moment. @@ -27,6 +34,24 @@ public class PayloadChecksum { return new PayloadChecksum("", type); } + public static PayloadChecksum fromPayload(Payload payload, Type type) { + switch (type) { + case MD5: return fromMd5Data(payload.getData()); + case XXHASH64: return fromXxhash64Data(payload.getData()); + default: throw new IllegalArgumentException("Unknown type " + type); + } + } + + private static PayloadChecksum fromMd5Data(AbstractUtf8Array data) { + return new PayloadChecksum(ConfigUtils.getMd5(data), MD5); + } + + private static PayloadChecksum fromXxhash64Data(AbstractUtf8Array data) { + return new PayloadChecksum(ConfigUtils.getXxhash64(data), XXHASH64); + } + + public boolean isEmpty() { return checksum.isEmpty(); } + public String asString() { return checksum; } public Type type() { return type; } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java index 41106e138b7..938da855014 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java @@ -85,4 +85,12 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque */ Payload payloadFromResponse(ConfigResponse response); + + /** + * Returns the payload checksums from the config request. + * + * @return the payload checksumss from request. + */ + PayloadChecksums configPayloadChecksums(); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index fbb52e81679..13d0ca1119a 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -273,7 +273,9 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { } @Override - public Optional<VespaVersion> getVespaVersion() { - return requestData.getVespaVersion(); - } + public Optional<VespaVersion> getVespaVersion() { return requestData.getVespaVersion(); } + + @Override + public PayloadChecksums configPayloadChecksums() { return requestData.getRequestConfigChecksums(); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index 00d010e75c8..7f39d678fdf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -44,7 +44,10 @@ public class SuperModelController { public ConfigResponse resolveConfig(GetConfigRequest request) { ConfigKey<?> configKey = request.getConfigKey(); validateConfigDefinition(request.getConfigKey(), request.getDefContent()); - return responseFactory.createResponse(model.getConfig(configKey).toUtf8Array(true), generation, false); + return responseFactory.createResponse(model.getConfig(configKey).toUtf8Array(true), + generation, + false, + request.configPayloadChecksums()); } private void validateConfigDefinition(ConfigKey<?> configKey, DefContent defContent) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index df1427bdf6d..0b409d38196 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -130,12 +130,11 @@ public class Application implements ModelResult { metricUpdater.incrementFailedRequests(); throw new UnknownConfigDefinitionException("Unable to find config definition for '" + configKey.getNamespace() + "." + configKey.getName()); } - log.log(Level.FINE, () -> TenantRepository.logPre(getId()) + ("Resolving " + configKey + " with config definition " + def)); + log.log(Level.FINE, () -> TenantRepository.logPre(getId()) + "Resolving " + configKey + " with config definition " + def); var payload = createPayload(configKey, def); - var response = responseFactory.createResponse(payload.getFirst(), applicationGeneration, payload.getSecond()); - return response; + return responseFactory.createResponse(payload.getFirst(), applicationGeneration, payload.getSecond(), req.configPayloadChecksums()); } private Pair<AbstractUtf8Array, Boolean> createPayload(ConfigKey<?> configKey, ConfigDefinition def) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java index c01008fafa0..8abb701606c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java @@ -11,6 +11,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.RequestHandler; @@ -195,4 +196,7 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { @Override public String getRequestDefMd5() { return ConfigUtils.getDefMd5(getDefContent().asList()); } + @Override + public PayloadChecksums configPayloadChecksums() { return PayloadChecksums.empty(); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java index 7afeebdd3cf..e52b2c4af01 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java @@ -1,10 +1,16 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.text.AbstractUtf8Array; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.PayloadChecksum; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.util.ConfigUtils; + +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; /** * Represents a component that creates config responses from a payload. Different implementations @@ -28,12 +34,37 @@ public interface ConfigResponseFactory { /** * Creates a {@link ConfigResponse} for a given payload and generation. * - * @param rawPayload the {@link ConfigPayload} to put in the response - * @param generation the payload generation - * @param applyOnRestart true if this config change should only be applied on restart, - * false if it should be applied immediately + * @param rawPayload the {@link ConfigPayload} to put in the response + * @param generation the payload generation + * @param applyOnRestart true if this config change should only be applied on restart, + * false if it should be applied immediately + * @param requestsPayloadChecksums payload checksums from requests * @return a {@link ConfigResponse} that can be sent to the client */ - ConfigResponse createResponse(AbstractUtf8Array rawPayload, long generation, boolean applyOnRestart); + ConfigResponse createResponse(AbstractUtf8Array rawPayload, + long generation, + boolean applyOnRestart, + PayloadChecksums requestsPayloadChecksums); + + /** Generates payload checksums based on what type of checksums exist in request */ + default PayloadChecksums generatePayloadChecksums(AbstractUtf8Array rawPayload, PayloadChecksums requestsPayloadChecksums) { + PayloadChecksum requestChecksumMd5 = requestsPayloadChecksums.getForType(MD5); + PayloadChecksum requestChecksumXxhash64 = requestsPayloadChecksums.getForType(XXHASH64); + + PayloadChecksum md5 = PayloadChecksum.empty(MD5); + PayloadChecksum xxhash64 = PayloadChecksum.empty(XXHASH64); + // Response contains same checksum type as in request, except when both are empty, + // then use both checksum types in response + if (requestChecksumMd5.isEmpty() && requestChecksumXxhash64.isEmpty() + || ( ! requestChecksumMd5.isEmpty() && ! requestChecksumXxhash64.isEmpty())) { + md5 = new PayloadChecksum(ConfigUtils.getMd5(rawPayload), MD5); + xxhash64 = new PayloadChecksum(ConfigUtils.getXxhash64(rawPayload), XXHASH64); + } else if ( ! requestChecksumMd5.isEmpty()) + md5 = new PayloadChecksum(ConfigUtils.getMd5(rawPayload), MD5); + else + xxhash64 = new PayloadChecksum(ConfigUtils.getXxhash64(rawPayload), XXHASH64); + + return PayloadChecksums.from(md5, xxhash64); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java index 3698a50217a..6a1ecfac7bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java @@ -1,15 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; -import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.text.AbstractUtf8Array; import com.yahoo.text.Utf8Array; import com.yahoo.vespa.config.LZ4PayloadCompressor; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.CompressionInfo; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; -import com.yahoo.vespa.config.util.ConfigUtils; /** * Compressor that compresses config payloads to lz4. @@ -23,10 +22,11 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory { @Override public ConfigResponse createResponse(AbstractUtf8Array rawPayload, long generation, - boolean applyOnRestart) { + boolean applyOnRestart, + PayloadChecksums requestsPayloadChecksums) { CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength()); Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.wrap())); - PayloadChecksums payloadChecksums = PayloadChecksums.from(ConfigUtils.getMd5(rawPayload), ConfigUtils.getXxhash64(rawPayload)); + PayloadChecksums payloadChecksums = generatePayloadChecksums(rawPayload, requestsPayloadChecksums); return new SlimeConfigResponse(compressed, generation, applyOnRestart, payloadChecksums, info); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java index 97db683e348..ce973e538b7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; -import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.text.AbstractUtf8Array; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.CompressionInfo; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; -import com.yahoo.vespa.config.util.ConfigUtils; /** * Simply returns an uncompressed payload. @@ -17,11 +16,13 @@ import com.yahoo.vespa.config.util.ConfigUtils; public class UncompressedConfigResponseFactory implements ConfigResponseFactory { @Override - public ConfigResponse createResponse(AbstractUtf8Array rawPayload, long generation, boolean applyOnRestart) { - String configMd5 = ConfigUtils.getMd5(rawPayload); - String xxHash64 = ConfigUtils.getXxhash64(rawPayload); + public ConfigResponse createResponse(AbstractUtf8Array rawPayload, + long generation, + boolean applyOnRestart, + PayloadChecksums requestsPayloadChecksums) { CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength()); - return new SlimeConfigResponse(rawPayload, generation, applyOnRestart, PayloadChecksums.from(configMd5, xxHash64), info); + PayloadChecksums payloadChecksums = generatePayloadChecksums(rawPayload, requestsPayloadChecksums); + return new SlimeConfigResponse(rawPayload, generation, applyOnRestart, payloadChecksums, info); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 41af0296534..28d50a5396e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -26,6 +26,7 @@ import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.VespaVersion; @@ -810,6 +811,9 @@ public class ApplicationRepositoryTest { @Override public String getRequestDefMd5() { return ""; } + @Override + public PayloadChecksums configPayloadChecksums() { return PayloadChecksums.empty(); } + }, Optional.empty()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java index 747a0ad3241..b164c3e5cd5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java @@ -1,11 +1,16 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.PayloadChecksum; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.Payload; import org.junit.Test; +import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; +import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; import static org.junit.Assert.assertEquals; /** @@ -13,22 +18,63 @@ import static org.junit.Assert.assertEquals; */ public class ConfigResponseFactoryTest { + private static final ConfigPayload payload = ConfigPayload.fromString("{ \"field1\": 11, \"field2\": 11 }"); + + private static final PayloadChecksums payloadChecksums = PayloadChecksums.fromPayload(Payload.from(payload)); + private static final PayloadChecksums payloadChecksumsEmpty = PayloadChecksums.empty(); + private static final PayloadChecksums payloadChecksumsOnlyMd5 = + PayloadChecksums.from(PayloadChecksum.fromPayload(Payload.from(payload), MD5)); + private static final PayloadChecksums payloadChecksumsOnlyXxhash64 = + PayloadChecksums.from(PayloadChecksum.fromPayload(Payload.from(payload), XXHASH64)); + @Test public void testUncompressedFactory() { UncompressedConfigResponseFactory responseFactory = new UncompressedConfigResponseFactory(); - ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty().toUtf8Array(true), 3, false); + ConfigResponse response = responseFactory.createResponse(payload.toUtf8Array(true), 3, false, payloadChecksums); assertEquals(CompressionType.UNCOMPRESSED, response.getCompressionInfo().getCompressionType()); assertEquals(3L,response.getGeneration()); - assertEquals(2, response.getPayload().getByteLength()); + assertEquals(25, response.getPayload().getByteLength()); + assertEquals(payloadChecksums, response.getPayloadChecksums()); } @Test public void testLZ4CompressedFactory() { + // Both checksums in request + { + ConfigResponse response = createResponse(payloadChecksums); + assertEquals(payloadChecksums, response.getPayloadChecksums()); + } + + // No checksums in request (empty checksums), both checksums should be in response + { + ConfigResponse response = createResponse(payloadChecksumsEmpty); + assertEquals(payloadChecksums.getForType(MD5), response.getPayloadChecksums().getForType(MD5)); + assertEquals(payloadChecksums.getForType(XXHASH64), response.getPayloadChecksums().getForType(XXHASH64)); + } + + // Only md5 checksums in request + { + ConfigResponse response = createResponse(payloadChecksumsOnlyMd5); + assertEquals(payloadChecksumsOnlyMd5.getForType(MD5), response.getPayloadChecksums().getForType(MD5)); + assertEquals(payloadChecksumsOnlyMd5.getForType(XXHASH64), response.getPayloadChecksums().getForType(XXHASH64)); + } + + // Only xxhash64 checksums in request + { + ConfigResponse response = createResponse(payloadChecksumsOnlyXxhash64); + assertEquals(payloadChecksumsOnlyXxhash64.getForType(MD5), response.getPayloadChecksums().getForType(MD5)); + assertEquals(payloadChecksumsOnlyXxhash64.getForType(XXHASH64), response.getPayloadChecksums().getForType(XXHASH64)); + } + } + + private ConfigResponse createResponse(PayloadChecksums payloadChecksums) { LZ4ConfigResponseFactory responseFactory = new LZ4ConfigResponseFactory(); - ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty().toUtf8Array(true), 3, false); + ConfigResponse response = responseFactory.createResponse(payload.toUtf8Array(true), 3, false, payloadChecksums); assertEquals(CompressionType.LZ4, response.getCompressionInfo().getCompressionType()); assertEquals(3L, response.getGeneration()); - assertEquals(3, response.getPayload().getByteLength()); + assertEquals(23, response.getPayload().getByteLength()); + + return response; } } |