diff options
author | Håkon Hallingstad <hakon@yahooinc.com> | 2023-07-28 14:33:32 +0200 |
---|---|---|
committer | Håkon Hallingstad <hakon@yahooinc.com> | 2023-07-28 14:33:32 +0200 |
commit | 2ee24c4628c75abb8f8495eac978b3eb75c66162 (patch) | |
tree | f7d2d7a182ecee2f2387f97a930d666f81e78bcb /flags | |
parent | e9613c844167b4e8a55096043f42953182cd3482 (diff) |
Add cloud flag dimension
Diffstat (limited to 'flags')
9 files changed, 400 insertions, 70 deletions
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index c1877373ce2..8dff69b5a60 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -3,10 +3,13 @@ package com.yahoo.vespa.flags; import com.yahoo.vespa.flags.json.DimensionHelper; +import java.util.Collection; import java.util.EnumMap; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -20,20 +23,25 @@ public class FetchVector { * Note: If this enum is changed, you must also change {@link DimensionHelper}. */ public enum Dimension { - /** Value from TenantName::value, e.g. vespa-team */ - TENANT_ID, - /** Value from ApplicationId::serializedForm of the form tenant:applicationName:instance. */ APPLICATION_ID, - /** Node type from com.yahoo.config.provision.NodeType::name, e.g. tenant, host, confighost, controller, etc. */ - NODE_TYPE, + /** + * Cloud from com.yahoo.config.provision.CloudName::value, e.g. yahoo, aws, gcp. + * + * <p><em>Eager resolution</em>: This dimension is resolved before putting the flag data to the config server + * or controller, unless controller and the flag has declared this dimension. + */ + CLOUD, + + /** Cluster ID from com.yahoo.config.provision.ClusterSpec.Id::value, e.g. cluster-controllers, logserver. */ + CLUSTER_ID, /** Cluster type from com.yahoo.config.provision.ClusterSpec.Type::name, e.g. content, container, admin */ CLUSTER_TYPE, - /** Cluster ID from com.yahoo.config.provision.ClusterSpec.Id::value, e.g. cluster-controllers, logserver. */ - CLUSTER_ID, + /** Email address of user - provided by auth0 in console. */ + CONSOLE_USER_EMAIL, /** * Fully qualified hostname. @@ -44,6 +52,18 @@ public class FetchVector { */ HOSTNAME, + /** Node type from com.yahoo.config.provision.NodeType::name, e.g. tenant, host, confighost, controller, etc. */ + NODE_TYPE, + + /** + * Hosted Vespa system from com.yahoo.config.provision.SystemName::value, e.g. main, cd, public, publiccd. + * <em>Eager resolution</em>, see {@link #CLOUD}. + */ + SYSTEM, + + /** Value from TenantName::value, e.g. vespa-team */ + TENANT_ID, + /** * Vespa version from Version::toFullString of the form Major.Minor.Micro. * @@ -53,14 +73,9 @@ public class FetchVector { */ VESPA_VERSION, - /** Email address of user - provided by auth0 in console. */ - CONSOLE_USER_EMAIL, - /** - * Zone from ZoneId::value of the form environment.region. - * - * <p>NOTE: There is seldom any need to set ZONE_ID, as all flags are set per zone anyways. The controller - * could PERHAPS use this where it handles multiple zones. + * Virtual zone ID from com.yahoo.config.provision.zone.ZoneId::value of the form environment.region, + * see com.yahoo.config.provision.zone.ZoneApi::getVirtualId. <em>Eager resolution</em>, see {@link #CLOUD}. */ ZONE_ID } @@ -83,15 +98,13 @@ public class FetchVector { return Optional.ofNullable(map.get(dimension)); } - public Map<Dimension, String> toMap() { - return map; - } + public Map<Dimension, String> toMap() { return map; } public boolean isEmpty() { return map.isEmpty(); } - public boolean hasDimension(FetchVector.Dimension dimension) { - return map.containsKey(dimension); - } + public boolean hasDimension(FetchVector.Dimension dimension) { return map.containsKey(dimension);} + + public Set<Dimension> dimensions() { return map.keySet(); } /** * Returns a new FetchVector, identical to {@code this} except for its value in {@code dimension}. @@ -107,13 +120,28 @@ public class FetchVector { return makeFetchVector(vector -> vector.putAll(override.map)); } - private FetchVector makeFetchVector(Consumer<EnumMap<Dimension, String>> mapModifier) { - EnumMap<Dimension, String> mergedMap = new EnumMap<>(Dimension.class); + private FetchVector makeFetchVector(Consumer<Map<Dimension, String>> mapModifier) { + Map<Dimension, String> mergedMap = new EnumMap<>(Dimension.class); mergedMap.putAll(map); mapModifier.accept(mergedMap); return new FetchVector(mergedMap); } + public FetchVector without(Dimension dimension) { + return makeFetchVector(merged -> merged.remove(dimension)); + } + + public FetchVector without(Collection<Dimension> dimensions) { + return makeFetchVector(merged -> merged.keySet().removeAll(dimensions)); + } + + @Override + public String toString() { + return "FetchVector{" + + "map=" + map + + '}'; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a3402a55bea..8ee6584f93a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -526,6 +526,15 @@ public class Flags { * For instance, if APPLICATION is one of the dimensions here, you should make sure * APPLICATION is set to the ApplicationId in the fetch vector when fetching the RawFlag * from the FlagSource. + * SYSTEM, CLOUD, and ZONE_ID are special: These dimensions are resolved just before the + * flag data is published to a zone. This means there is never any need to set these + * dimensions when resolving a flag, and setting these dimensions just before resolving + * the flag will have no effect. + * There is one exception. If any of these dimensions are declared when defining a flag, + * then those dimensions are NOT resolved when published to the controllers. This allows + * the controller to resolve the flag to different values based on which cloud or zone + * it is operating on. Flags should NOT declare these dimensions unless they intend to + * use them in the controller in this way. * @param <T> The boxed type of the flag value, e.g. Boolean for flags guarding features. * @param <U> The type of the unbound flag, e.g. UnboundBooleanFlag. * @return An unbound flag with {@link FetchVector.Dimension#HOSTNAME HOSTNAME} and diff --git a/flags/src/main/java/com/yahoo/vespa/flags/JsonNodeRawFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/JsonNodeRawFlag.java index 753f19a44f6..27852790186 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/JsonNodeRawFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/JsonNodeRawFlag.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Collection; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import static com.yahoo.yolean.Exceptions.uncheck; @@ -60,6 +61,26 @@ public class JsonNodeRawFlag implements RawFlag { return jsonNode.toString(); } + @Override + public String toString() { + return "JsonNodeRawFlag{" + + "jsonNode=" + jsonNode + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JsonNodeRawFlag that = (JsonNodeRawFlag) o; + return jsonNode.equals(that.jsonNode); + } + + @Override + public int hashCode() { + return Objects.hash(jsonNode); + } + /** Initialize object mapper lazily */ private static ObjectMapper objectMapper() { // ObjectMapper is a heavy-weight object so we construct it only when we need it diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java index ad1242aa7e9..9516c12693d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java @@ -15,15 +15,17 @@ public class DimensionHelper { private static final Map<FetchVector.Dimension, String> serializedDimensions = new HashMap<>(); static { - serializedDimensions.put(FetchVector.Dimension.ZONE_ID, "zone"); - serializedDimensions.put(FetchVector.Dimension.HOSTNAME, "hostname"); serializedDimensions.put(FetchVector.Dimension.APPLICATION_ID, "application"); - serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, "node-type"); + serializedDimensions.put(FetchVector.Dimension.CLOUD, "cloud"); serializedDimensions.put(FetchVector.Dimension.CLUSTER_ID, "cluster-id"); serializedDimensions.put(FetchVector.Dimension.CLUSTER_TYPE, "cluster-type"); - serializedDimensions.put(FetchVector.Dimension.VESPA_VERSION, "vespa-version"); serializedDimensions.put(FetchVector.Dimension.CONSOLE_USER_EMAIL, "console-user-email"); + serializedDimensions.put(FetchVector.Dimension.HOSTNAME, "hostname"); + serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, "node-type"); + serializedDimensions.put(FetchVector.Dimension.SYSTEM, "system"); serializedDimensions.put(FetchVector.Dimension.TENANT_ID, "tenant"); + serializedDimensions.put(FetchVector.Dimension.VESPA_VERSION, "vespa-version"); + serializedDimensions.put(FetchVector.Dimension.ZONE_ID, "zone"); if (serializedDimensions.size() != FetchVector.Dimension.values().length) { throw new IllegalStateException(FetchVectorHelper.class.getName() + " is not in sync with " + diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java index 19837e7dbe1..acda3b9db42 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java @@ -13,9 +13,10 @@ import com.yahoo.vespa.flags.json.wire.WireRule; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -53,6 +54,27 @@ public class FlagData { public boolean isEmpty() { return rules.isEmpty() && defaultFetchVector.isEmpty(); } + public FlagData partialResolve(FetchVector fetchVector) { + // Note: As a result of partialResolve, there could be e.g. two identical rules, and the latter will always be ignored by resolve(). + // Consider deduping. Deduping is actually not specific to partialResolve and could be done e.g. at construction time. + + List<Rule> newRules = new ArrayList<>(); + for (var rule : rules) { + Optional<Rule> partialRule = rule.partialResolve(fetchVector); + if (partialRule.isPresent()) { + newRules.add(partialRule.get()); + if (partialRule.get().conditions().isEmpty()) { + // Any following rule will always be ignored during resolution. + break; + } + } + } + + FetchVector newDefaultFetchVector = defaultFetchVector.without(fetchVector.dimensions()); + + return new FlagData(id, newDefaultFetchVector, newRules); + } + public Optional<RawFlag> resolve(FetchVector fetchVector) { return rules.stream() .filter(rule -> rule.match(defaultFetchVector.with(fetchVector))) @@ -91,6 +113,36 @@ public class FlagData { return wireFlagData; } + /** E.g. verify all RawFlag can be deserialized. */ + public void validate(Deserializer<?> deserializer) { + rules.stream() + .flatMap(rule -> rule.getValueToApply().map(Stream::of).orElse(null)) + .forEach(deserializer::deserialize); + + } + + @Override + public String toString() { + return "FlagData{" + + "id=" + id + + ", rules=" + rules + + ", defaultFetchVector=" + defaultFetchVector + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FlagData flagData = (FlagData) o; + return id.equals(flagData.id) && rules.equals(flagData.rules) && defaultFetchVector.equals(flagData.defaultFetchVector); + } + + @Override + public int hashCode() { + return Objects.hash(id, rules, defaultFetchVector); + } + public static FlagData deserializeUtf8Json(byte[] bytes) { return fromWire(WireFlagData.deserialize(bytes)); } @@ -138,13 +190,5 @@ public class FlagData { if (wireRules == null) return List.of(); return wireRules.stream().map(Rule::fromWire).toList(); } - - /** E.g. verify all RawFlag can be deserialized. */ - public void validate(Deserializer<?> deserializer) { - rules.stream() - .flatMap(rule -> rule.getValueToApply().map(Stream::of).orElse(null)) - .forEach(deserializer::deserialize); - - } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java index c4b2d9be117..483f6750a73 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.json.wire.WireCondition; import java.util.List; +import java.util.Objects; /** * @author hakonhall @@ -55,4 +56,27 @@ public abstract class ListCondition implements Condition { condition.values = values.isEmpty() ? null : values; return condition; } + + @Override + public String toString() { + return "ListCondition{" + + "type=" + type + + ", dimension=" + dimension + + ", values=" + values + + ", isWhitelist=" + isWhitelist + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ListCondition that = (ListCondition) o; + return isWhitelist == that.isWhitelist && type == that.type && dimension == that.dimension && values.equals(that.values); + } + + @Override + public int hashCode() { + return Objects.hash(type, dimension, values, isWhitelist); + } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java index 0efeb831f2c..749f6830870 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.json.wire.WireCondition; +import java.util.Objects; import java.util.function.Predicate; /** @@ -75,4 +76,26 @@ public class RelationalCondition implements Condition { condition.predicate = relationalPredicate.toWire(); return condition; } + + @Override + public String toString() { + return "RelationalCondition{" + + "relationalPredicate=" + relationalPredicate + + ", predicate=" + predicate + + ", dimension=" + dimension + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RelationalCondition that = (RelationalCondition) o; + return relationalPredicate.equals(that.relationalPredicate) && predicate.equals(that.predicate) && dimension == that.dimension; + } + + @Override + public int hashCode() { + return Objects.hash(relationalPredicate, predicate, dimension); + } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java b/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java index bddaf8c9e0e..127c2b4f9da 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java @@ -6,10 +6,11 @@ import com.yahoo.vespa.flags.JsonNodeRawFlag; import com.yahoo.vespa.flags.RawFlag; import com.yahoo.vespa.flags.json.wire.WireRule; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; /** * @author hakonhall @@ -45,6 +46,25 @@ public class Rule { .allMatch(condition -> !fetchVector.hasDimension(condition.dimension()) || condition.test(fetchVector)); } + /** + * Returns a copy of this rule without those conditions that can be resolved by the fetch vector. Returns empty + * if any of those conditions are false. + */ + public Optional<Rule> partialResolve(FetchVector fetchVector) { + List<Condition> newConditions = new ArrayList<>(); + for (var condition : andConditions) { + if (fetchVector.hasDimension(condition.dimension())) { + if (!condition.test(fetchVector)) { + return Optional.empty(); + } + } else { + newConditions.add(condition); + } + } + + return Optional.of(new Rule(valueToApply, newConditions)); + } + public Optional<RawFlag> getValueToApply() { return valueToApply; } @@ -68,4 +88,25 @@ public class Rule { Optional<RawFlag> value = wireRule.value == null ? Optional.empty() : Optional.of(JsonNodeRawFlag.fromJsonNode(wireRule.value)); return new Rule(value, conditions); } + + @Override + public String toString() { + return "Rule{" + + "andConditions=" + andConditions + + ", valueToApply=" + valueToApply + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Rule rule = (Rule) o; + return andConditions.equals(rule.andConditions) && valueToApply.equals(rule.valueToApply); + } + + @Override + public int hashCode() { + return Objects.hash(andConditions, valueToApply); + } } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java index c89b5883fd1..c7da1abe7e2 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java @@ -15,44 +15,45 @@ import static org.junit.jupiter.api.Assertions.assertTrue; * @author hakonhall */ public class FlagDataTest { - private final String json = "{\n" + - " \"id\": \"id1\",\n" + - " \"rules\": [\n" + - " {\n" + - " \"conditions\": [\n" + - " {\n" + - " \"type\": \"whitelist\",\n" + - " \"dimension\": \"hostname\",\n" + - " \"values\": [ \"host1\", \"host2\" ]\n" + - " },\n" + - " {\n" + - " \"type\": \"blacklist\",\n" + - " \"dimension\": \"application\",\n" + - " \"values\": [ \"app1\", \"app2\" ]\n" + - " }\n" + - " ],\n" + - " \"value\": true\n" + - " },\n" + - " {\n" + - " \"conditions\": [\n" + - " {\n" + - " \"type\": \"whitelist\",\n" + - " \"dimension\": \"zone\",\n" + - " \"values\": [ \"zone1\", \"zone2\" ]\n" + - " }\n" + - " ],\n" + - " \"value\": false\n" + - " }\n" + - " ],\n" + - " \"attributes\": {\n" + - " \"zone\": \"zone1\"\n" + - " }\n" + - "}"; + private final String json = """ + { + "id": "id1", + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "hostname", + "values": [ "host1", "host2" ] + }, + { + "type": "blacklist", + "dimension": "application", + "values": [ "app1", "app2" ] + } + ], + "value": true + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "values": [ "zone1", "zone2" ] + } + ], + "value": false + } + ], + "attributes": { + "zone": "zone1" + } + }"""; private final FetchVector vector = new FetchVector(); @Test - void test() { + void testResolve() { // Second rule matches with the default zone matching verify(Optional.of("false"), vector); @@ -74,6 +75,143 @@ public class FlagDataTest { verify(Optional.empty(), vector.with(FetchVector.Dimension.ZONE_ID, "unknown zone")); } + @Test + void testPartialResolve() { + FlagData data = FlagData.deserialize(json); + assertEquals(data.partialResolve(vector), data); + assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.APPLICATION_ID, "app1")), + FlagData.deserialize(""" + { + "id": "id1", + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "values": [ "zone1", "zone2" ] + } + ], + "value": false + } + ], + "attributes": { + "zone": "zone1" + } + }""")); + + assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.APPLICATION_ID, "app1")), + FlagData.deserialize(""" + { + "id": "id1", + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "values": [ "zone1", "zone2" ] + } + ], + "value": false + } + ], + "attributes": { + "zone": "zone1" + } + }""")); + + assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.APPLICATION_ID, "app3")), + FlagData.deserialize(""" + { + "id": "id1", + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "hostname", + "values": [ "host1", "host2" ] + } + ], + "value": true + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "values": [ "zone1", "zone2" ] + } + ], + "value": false + } + ], + "attributes": { + "zone": "zone1" + } + }""")); + + assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.APPLICATION_ID, "app3") + .with(FetchVector.Dimension.HOSTNAME, "host1")), + FlagData.deserialize(""" + { + "id": "id1", + "rules": [ + { + "value": true + } + ], + "attributes": { + "zone": "zone1" + } + }""")); + + assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.APPLICATION_ID, "app3") + .with(FetchVector.Dimension.HOSTNAME, "host3")), + FlagData.deserialize(""" + { + "id": "id1", + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "values": [ "zone1", "zone2" ] + } + ], + "value": false + } + ], + "attributes": { + "zone": "zone1" + } + }""")); + + assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.APPLICATION_ID, "app3") + .with(FetchVector.Dimension.HOSTNAME, "host3") + .with(FetchVector.Dimension.ZONE_ID, "zone2")), + FlagData.deserialize(""" + { + "id": "id1", + "rules": [ + { + "value": false + } + ] + }""")); + + FlagData fullyResolved = data.partialResolve(vector.with(FetchVector.Dimension.APPLICATION_ID, "app3") + .with(FetchVector.Dimension.HOSTNAME, "host3") + .with(FetchVector.Dimension.ZONE_ID, "zone3")); + assertEquals(fullyResolved, FlagData.deserialize(""" + { + "id": "id1" + }""")); + assertTrue(fullyResolved.isEmpty()); + } + private void verify(Optional<String> expectedValue, FetchVector vector) { FlagData data = FlagData.deserialize(json); assertEquals("id1", data.id().toString()); |