From af82f15b8ec3a7c19d1b9ba48b53edf9feb6de48 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sun, 30 Dec 2018 20:10:16 +0100 Subject: Configserver flags REST API Adds a new ZooKeeper backed flag source. It is defined in a new module configserver-flags to allow as many as possible config server modules to depend on it by minimizing dependencies. The content of the ZK backed flag source can be viewed and modified through REST API on the config server/controller. The data stored per flag looks like { "rules": [ { "conditions": [ { "type": "whitelist", "dimension": "hostname", "values": ["host1"] } ], "value": true } ] } typical for enabling a feature flag on host1. 2 types of conditions are so far supported: whitelist and blacklist. All the conditions must match in order for the value to apply. If the value is null (or absent), the default value will be used. At the time the flag's value is retrieved, it is resolved against the conditions with the current zone, hostname, and/or application. The same data structure is used for FileFlagSource for files in /etc/vespa/flags with the ".2" extension. The FlagSource component injected in the config server is changed to: 1. Return the flag value if specified in /etc/vespa/flags, or otherwise 2. return flag value from ZooKeeper (same as REST API) The current flags (module) is also changed: - All flags must be defined in com.yahoo.vespa.flags.Flags. This allows the ZK backed flag source additional sanity checking when modifying flags. - If it makes sense to have different flag value depending on e.g. the application, then at some point before the value is retrieved, one has to bind the flag to that application (using with() to set up the fetch vector). Future changes would be to 0. make a merged FlagSource in host admin, 1. add support for viewing and modifying feature flags in dashboard, 2. in hv tool. --- CMakeLists.txt | 1 + configserver-flags/CMakeLists.txt | 2 + configserver-flags/OWNERS | 1 + configserver-flags/README.md | 3 + configserver-flags/pom.xml | 100 +++++++++++++ .../configserver/flags/ConfigServerFlagSource.java | 18 +++ .../yahoo/vespa/configserver/flags/FlagsDb.java | 25 ++++ .../vespa/configserver/flags/db/FlagsDbImpl.java | 56 +++++++ .../configserver/flags/db/ZooKeeperFlagSource.java | 25 ++++ .../vespa/configserver/flags/package-info.java | 7 + .../configserver/flags/db/FlagsDbImplTest.java | 56 +++++++ configserver/CMakeLists.txt | 1 + configserver/pom.xml | 6 + .../vespa/config/server/ConfigServerBootstrap.java | 8 +- .../config/server/application/Application.java | 18 ++- .../server/http/flags/FlagDataListResponse.java | 52 +++++++ .../config/server/http/flags/FlagDataResponse.java | 31 ++++ .../config/server/http/flags/FlagsHandler.java | 94 ++++++++++++ .../vespa/config/server/http/flags/OKResponse.java | 19 +++ .../vespa/config/server/http/flags/V1Response.java | 29 ++++ .../main/resources/configserver-app/services.xml | 8 +- .../config/server/http/flags/FlagsHandlerTest.java | 161 +++++++++++++++++++++ flags/pom.xml | 11 +- .../java/com/yahoo/vespa/flags/Deserializer.java | 10 ++ .../java/com/yahoo/vespa/flags/FeatureFlag.java | 62 -------- .../java/com/yahoo/vespa/flags/FetchVector.java | 82 +++++++++++ .../java/com/yahoo/vespa/flags/FileFlagSource.java | 43 +++++- .../src/main/java/com/yahoo/vespa/flags/Flag.java | 39 ++++- .../java/com/yahoo/vespa/flags/FlagDefinition.java | 41 ++++++ .../main/java/com/yahoo/vespa/flags/FlagId.java | 7 +- .../java/com/yahoo/vespa/flags/FlagSerializer.java | 8 + .../java/com/yahoo/vespa/flags/FlagSource.java | 3 +- .../src/main/java/com/yahoo/vespa/flags/Flags.java | 130 +++++++++++++++++ .../main/java/com/yahoo/vespa/flags/IntFlag.java | 49 ------- .../java/com/yahoo/vespa/flags/JacksonFlag.java | 58 -------- .../com/yahoo/vespa/flags/JacksonSerializer.java | 23 +++ .../com/yahoo/vespa/flags/JsonNodeRawFlag.java | 48 ++++++ .../main/java/com/yahoo/vespa/flags/LongFlag.java | 49 ------- .../com/yahoo/vespa/flags/OrderedFlagSource.java | 33 +++++ .../main/java/com/yahoo/vespa/flags/RawFlag.java | 14 ++ .../java/com/yahoo/vespa/flags/Serializer.java | 10 ++ .../yahoo/vespa/flags/SimpleFlagSerializer.java | 39 +++++ .../java/com/yahoo/vespa/flags/StringFlag.java | 49 ------- .../java/com/yahoo/vespa/flags/UnboundFlag.java | 38 +++++ .../java/com/yahoo/vespa/flags/json/Condition.java | 65 +++++++++ .../java/com/yahoo/vespa/flags/json/FlagData.java | 104 +++++++++++++ .../main/java/com/yahoo/vespa/flags/json/Rule.java | 58 ++++++++ .../com/yahoo/vespa/flags/json/package-info.java | 5 + .../vespa/flags/json/wire/DimensionHelper.java | 48 ++++++ .../vespa/flags/json/wire/FetchVectorHelper.java | 27 ++++ .../yahoo/vespa/flags/json/wire/WireCondition.java | 19 +++ .../yahoo/vespa/flags/json/wire/WireFlagData.java | 55 +++++++ .../com/yahoo/vespa/flags/json/wire/WireRule.java | 19 +++ .../com/yahoo/vespa/flags/FileFlagSourceTest.java | 47 ++++-- .../test/java/com/yahoo/vespa/flags/FlagsTest.java | 131 +++++++++++++++++ .../com/yahoo/vespa/flags/JacksonFlagTest.java | 66 --------- .../yahoo/vespa/flags/OrderedFlagSourceTest.java | 50 +++++++ .../com/yahoo/vespa/flags/json/ConditionTest.java | 38 +++++ .../com/yahoo/vespa/flags/json/FlagDataTest.java | 82 +++++++++++ .../yahoo/vespa/flags/json/SerializationTest.java | 130 +++++++++++++++++ parent/pom.xml | 28 ++-- pom.xml | 1 + .../vespa/service/duper/DuperModelManager.java | 10 +- .../vespa/service/health/HealthMonitorManager.java | 15 +- .../service/health/HealthMonitorManagerTest.java | 5 +- 65 files changed, 2165 insertions(+), 405 deletions(-) create mode 100644 configserver-flags/CMakeLists.txt create mode 100644 configserver-flags/OWNERS create mode 100644 configserver-flags/README.md create mode 100644 configserver-flags/pom.xml create mode 100644 configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java create mode 100644 configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/FlagsDb.java create mode 100644 configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java create mode 100644 configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java create mode 100644 configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/package-info.java create mode 100644 configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataListResponse.java create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataResponse.java create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagsHandler.java create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/OKResponse.java create mode 100644 configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/V1Response.java create mode 100644 configserver/src/test/java/com/yahoo/vespa/config/server/http/flags/FlagsHandlerTest.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/Deserializer.java delete mode 100644 flags/src/main/java/com/yahoo/vespa/flags/FeatureFlag.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/FlagSerializer.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/Flags.java delete mode 100644 flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java delete mode 100644 flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/JacksonSerializer.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/JsonNodeRawFlag.java delete mode 100644 flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/OrderedFlagSource.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/RawFlag.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/Serializer.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/SimpleFlagSerializer.java delete mode 100644 flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/UnboundFlag.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/package-info.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/wire/DimensionHelper.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/wire/FetchVectorHelper.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireCondition.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagData.java create mode 100644 flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireRule.java create mode 100644 flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java delete mode 100644 flags/src/test/java/com/yahoo/vespa/flags/JacksonFlagTest.java create mode 100644 flags/src/test/java/com/yahoo/vespa/flags/OrderedFlagSourceTest.java create mode 100644 flags/src/test/java/com/yahoo/vespa/flags/json/ConditionTest.java create mode 100644 flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java create mode 100644 flags/src/test/java/com/yahoo/vespa/flags/json/SerializationTest.java diff --git a/CMakeLists.txt b/CMakeLists.txt index 7676bc6e886..476915d44a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ add_subdirectory(config-model-fat) add_subdirectory(configd) add_subdirectory(configdefinitions) add_subdirectory(configserver) +add_subdirectory(configserver-flags) add_subdirectory(configutil) add_subdirectory(container-accesslogging) add_subdirectory(container-core) diff --git a/configserver-flags/CMakeLists.txt b/configserver-flags/CMakeLists.txt new file mode 100644 index 00000000000..75deaf42d9b --- /dev/null +++ b/configserver-flags/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(configserver-flags) diff --git a/configserver-flags/OWNERS b/configserver-flags/OWNERS new file mode 100644 index 00000000000..e131dacde49 --- /dev/null +++ b/configserver-flags/OWNERS @@ -0,0 +1 @@ +hakonhall diff --git a/configserver-flags/README.md b/configserver-flags/README.md new file mode 100644 index 00000000000..e23452a1f0c --- /dev/null +++ b/configserver-flags/README.md @@ -0,0 +1,3 @@ + +# Config Server Flags +Manages flags backed by the Config Server's ZooKeeper. diff --git a/configserver-flags/pom.xml b/configserver-flags/pom.xml new file mode 100644 index 00000000000..c6fdbe89b95 --- /dev/null +++ b/configserver-flags/pom.xml @@ -0,0 +1,100 @@ + + + + 4.0.0 + + com.yahoo.vespa + parent + 6-SNAPSHOT + ../parent/pom.xml + + configserver-flags + 6-SNAPSHOT + container-plugin + ${project.artifactId} + Config Server Flags. + + + + + com.yahoo.vespa + zkfacade + ${project.version} + provided + + + com.yahoo.vespa + flags + ${project.version} + provided + + + com.yahoo.vespa + annotations + ${project.version} + provided + + + com.yahoo.vespa + vespajlib + ${project.version} + provided + + + com.yahoo.vespa + yolean + ${project.version} + provided + + + com.google.inject + guice + provided + no_aop + + + com.fasterxml.jackson.core + jackson-databind + provided + + + + + com.yahoo.vespa + testutil + ${project.version} + test + + + junit + junit + test + + + org.apache.curator + curator-test + test + + + org.mockito + mockito-core + test + + + + + + com.yahoo.vespa + bundle-plugin + true + + + org.apache.maven.plugins + maven-compiler-plugin + + + + diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java new file mode 100644 index 00000000000..9768c42b477 --- /dev/null +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java @@ -0,0 +1,18 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.configserver.flags; + +import com.google.inject.Inject; +import com.yahoo.vespa.configserver.flags.db.FlagsDbImpl; +import com.yahoo.vespa.configserver.flags.db.ZooKeeperFlagSource; +import com.yahoo.vespa.flags.FileFlagSource; +import com.yahoo.vespa.flags.OrderedFlagSource; + +/** + * @author hakonhall + */ +public class ConfigServerFlagSource extends OrderedFlagSource { + @Inject + public ConfigServerFlagSource(FlagsDbImpl flagsDb) { + super(new FileFlagSource(), new ZooKeeperFlagSource(flagsDb)); + } +} diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/FlagsDb.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/FlagsDb.java new file mode 100644 index 00000000000..2c29ae0b818 --- /dev/null +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/FlagsDb.java @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.configserver.flags; + +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.util.Map; +import java.util.Optional; + +/** + * @author hakonhall + */ +public interface FlagsDb { + /** Get the String value of the flag. */ + Optional getValue(FlagId flagId); + + /** Set the String value of the flag. */ + void setValue(FlagId flagId, FlagData data); + + /** Remove the flag value if it exists. */ + void removeValue(FlagId flagId); + + /** Get all flags that have been set. */ + Map getAllFlags(); +} diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java new file mode 100644 index 00000000000..7b0a2f632cc --- /dev/null +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java @@ -0,0 +1,56 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.configserver.flags.db; + +import com.google.inject.Inject; +import com.yahoo.path.Path; +import com.yahoo.vespa.configserver.flags.FlagsDb; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * @author hakonhall + */ +public class FlagsDbImpl implements FlagsDb { + private static final Path ROOT_PATH = Path.fromString("/flags/v1"); + + private final Curator curator; + + @Inject + public FlagsDbImpl(Curator curator) { + this.curator = curator; + } + + @Override + public Optional getValue(FlagId flagId) { + return curator.getData(getZkPathFor(flagId)).map(FlagData::deserializeUtf8Json); + } + + @Override + public void setValue(FlagId flagId, FlagData data) { + curator.set(getZkPathFor(flagId), data.serializeToUtf8Json()); + } + + @Override + public Map getAllFlags() { + Map flags = new HashMap<>(); + for (String flagId : curator.getChildren(ROOT_PATH)) { + FlagId id = new FlagId(flagId); + getValue(id).ifPresent(data -> flags.put(id, data)); + } + return flags; + } + + @Override + public void removeValue(FlagId flagId) { + curator.delete(getZkPathFor(flagId)); + } + + private static Path getZkPathFor(FlagId flagId) { + return ROOT_PATH.append(flagId.toString()); + } +} diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java new file mode 100644 index 00000000000..bd99ac6eca9 --- /dev/null +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.configserver.flags.db; + +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.RawFlag; + +import java.util.Optional; + +/** + * @author hakonhall + */ +public class ZooKeeperFlagSource implements FlagSource { + private final FlagsDbImpl flagsDb; + + public ZooKeeperFlagSource(FlagsDbImpl flagsDb) { + this.flagsDb = flagsDb; + } + + @Override + public Optional fetch(FlagId id, FetchVector vector) { + return flagsDb.getValue(id).flatMap(data -> data.resolve(vector)); + } +} diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/package-info.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/package-info.java new file mode 100644 index 00000000000..97e66d95715 --- /dev/null +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.configserver.flags; + +import com.yahoo.osgi.annotation.ExportPackage; + +/** The node repository controls and allocates the nodes available in a hosted Vespa zone */ diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java new file mode 100644 index 00000000000..1fe61130348 --- /dev/null +++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java @@ -0,0 +1,56 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.configserver.flags.db; + +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.JsonNodeRawFlag; +import com.yahoo.vespa.flags.json.Condition; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.Rule; +import org.junit.Test; + +import java.util.Map; +import java.util.Optional; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class FlagsDbImplTest { + @Test + public void test() { + MockCurator curator = new MockCurator(); + FlagsDbImpl db = new FlagsDbImpl(curator); + + Condition condition1 = new Condition(Condition.Type.WHITELIST, FetchVector.Dimension.HOSTNAME, "host1"); + Rule rule1 = new Rule(Optional.of(JsonNodeRawFlag.fromJson("13")), condition1); + FlagData data = new FlagData(new FetchVector().with(FetchVector.Dimension.ZONE_ID, "zone-a"), rule1); + FlagId flagId = new FlagId("id"); + db.setValue(flagId, data); + + assertTrue(db.getValue(flagId).isPresent()); + Optional dataCopy = db.getValue(flagId); + assertTrue(dataCopy.isPresent()); + + assertEquals("{\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\"," + + "\"values\":[\"host1\"]}],\"value\":13}],\"attributes\":{\"zone\":\"zone-a\"}}", + dataCopy.get().serializeToJson()); + + FlagId flagId2 = new FlagId("id2"); + db.setValue(flagId2, data); + Map flags = db.getAllFlags(); + assertThat(flags.size(), equalTo(2)); + assertThat(flags.get(flagId), notNullValue()); + assertThat(flags.get(flagId2), notNullValue()); + + db.removeValue(flagId2); + assertFalse(db.getValue(flagId2).isPresent()); + } +} \ No newline at end of file diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index 5d3f88ac777..fe905ced973 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -14,6 +14,7 @@ install(CODE "execute_process(COMMAND mkdir -p \$ENV{DESTDIR}/\${CMAKE_INSTALL_P install(CODE "execute_process(COMMAND mkdir -p \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/config-models)") install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/config-model-fat.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/config-model-fat.jar)") install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/configserver-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/configserver.jar)") +install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/configserver-flags-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/configserver-flags.jar)") install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/flags-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/flags.jar)") install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/orchestrator-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/orchestrator.jar)") install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/service-monitor-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/service-monitor.jar)") diff --git a/configserver/pom.xml b/configserver/pom.xml index f932a963d14..210e584fd85 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -88,6 +88,12 @@ ${project.version} provided + + com.yahoo.vespa + configserver-flags + ${project.version} + provided + com.yahoo.vespa config-application-package diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 0b1684b6735..f240129eda1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -12,8 +12,8 @@ import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; -import com.yahoo.vespa.flags.FeatureFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.time.Duration; import java.time.Instant; @@ -30,7 +30,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; -import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApplicationsFails.*; +import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApplicationsFails.CONTINUE; +import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApplicationsFails.EXIT_JVM; /** * Main component that bootstraps and starts config server threads. @@ -45,7 +46,6 @@ import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApp public class ConfigServerBootstrap extends AbstractComponent implements Runnable { private static final Logger log = Logger.getLogger(ConfigServerBootstrap.class.getName()); - private static final String bootstrapFeatureFlag = "config-server-bootstrap-in-separate-thread"; // INITIALIZE_ONLY is for testing only enum Mode {BOOTSTRAP_IN_CONSTRUCTOR, BOOTSTRAP_IN_SEPARATE_THREAD, INITIALIZE_ONLY} @@ -69,7 +69,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, FlagSource flagSource) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, - new FeatureFlag(bootstrapFeatureFlag, true, flagSource).value() + Flags.CONFIG_SERVER_BOOTSTRAP_IN_SEPARATE_THREAD.bindTo(flagSource).value() ? Mode.BOOTSTRAP_IN_SEPARATE_THREAD : Mode.BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index 032bb88eaaa..2d5d5296762 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.component.Version; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.component.Version; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -15,17 +15,19 @@ import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.DefContent; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.ServerCache; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; import com.yahoo.vespa.config.server.UnknownConfigDefinitionException; import com.yahoo.vespa.config.server.modelfactory.ModelResult; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; +import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; +import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.util.ConfigUtils; -import com.yahoo.vespa.flags.FeatureFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FileFlagSource; +import com.yahoo.vespa.flags.Flag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.util.Objects; import java.util.Set; @@ -47,7 +49,7 @@ public class Application implements ModelResult { private final ServerCache cache; private final MetricUpdater metricUpdater; private final ApplicationId app; - private final FeatureFlag useConfigServerCache; + private final Flag useConfigServerCache; public Application(Model model, ServerCache cache, long appGeneration, boolean internalRedeploy, Version vespaVersion, MetricUpdater metricUpdater, ApplicationId app) { @@ -64,7 +66,9 @@ public class Application implements ModelResult { this.vespaVersion = vespaVersion; this.metricUpdater = metricUpdater; this.app = app; - this.useConfigServerCache = new FeatureFlag("use-config-server-cache", true, flagSource); + this.useConfigServerCache = Flags.USE_CONFIG_SERVER_CACHE + .with(FetchVector.Dimension.APPLICATION_ID, app.serializedForm()) + .bindTo(flagSource); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataListResponse.java new file mode 100644 index 00000000000..4d9ba96b791 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataListResponse.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.flags; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.io.OutputStream; +import java.util.Map; +import java.util.TreeMap; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author hakonhall + */ +public class FlagDataListResponse extends HttpResponse { + private static ObjectMapper mapper = new ObjectMapper(); + + private final String flagsV1Uri; + private final Map flags; + private final boolean showDataInsteadOfUrl; + + public FlagDataListResponse(String flagsV1Uri, Map flags, boolean showDataInsteadOfUrl) { + super(Response.Status.OK); + this.flagsV1Uri = flagsV1Uri; + this.flags = flags; + this.showDataInsteadOfUrl = showDataInsteadOfUrl; + } + + @Override + public void render(OutputStream outputStream) { + ObjectNode rootNode = mapper.createObjectNode(); + // Order flags by ID + new TreeMap<>(flags).forEach((flagId, flagData) -> { + if (showDataInsteadOfUrl) { + rootNode.set(flagId.toString(), flagData.toJsonNode()); + } else { + rootNode.putObject(flagId.toString()).put("url", flagsV1Uri + "/data/" + flagId.toString()); + } + }); + uncheck(() -> mapper.writeValue(outputStream, rootNode)); + } + + @Override + public String getContentType() { + return "application/json"; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataResponse.java new file mode 100644 index 00000000000..054b218ff2d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataResponse.java @@ -0,0 +1,31 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.flags; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.flags.json.FlagData; + +import java.io.OutputStream; + +/** + * @author hakonhall + */ +public class FlagDataResponse extends HttpResponse { + private final FlagData data; + + FlagDataResponse(FlagData data) { + super(Response.Status.OK); + this.data = data; + } + + @Override + public void render(OutputStream outputStream) { + data.serializeToOutputStream(outputStream); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagsHandler.java new file mode 100644 index 00000000000..3b4194fc474 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagsHandler.java @@ -0,0 +1,94 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.flags; + +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.Path; +import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.configserver.flags.FlagsDb; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.json.FlagData; + +import java.net.URI; +import java.util.Objects; + +/** + * Handles /flags/v1 requests + * + * @author hakonhall + */ +public class FlagsHandler extends HttpHandler { + private final FlagsDb flagsDb; + + @Inject + public FlagsHandler(LoggingRequestHandler.Context context, FlagsDb flagsDb) { + super(context); + this.flagsDb = flagsDb; + } + + @Override + protected HttpResponse handleGET(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/flags/v1")) return new V1Response(flagsV1Uri(request)); + if (path.matches("/flags/v1/data")) return getFlagDataList(request); + if (path.matches("/flags/v1/data/{flagId}")) return getFlagData(findFlagId(request, path)); + throw new NotFoundException("Nothing at path '" + path + "'"); + } + + @Override + protected HttpResponse handlePUT(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/flags/v1/data/{flagId}")) return putFlagData(request, findFlagId(request, path)); + throw new NotFoundException("Nothing at path '" + path + "'"); + } + + @Override + protected HttpResponse handleDELETE(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/flags/v1/data/{flagId}")) return deleteFlagData(findFlagId(request, path)); + throw new NotFoundException("Nothing at path '" + path + "'"); + } + + private String flagsV1Uri(HttpRequest request) { + URI uri = request.getUri(); + return uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + "/flags/v1"; + } + + private HttpResponse getFlagDataList(HttpRequest request) { + return new FlagDataListResponse(flagsV1Uri(request), flagsDb.getAllFlags(), + Objects.equals(request.getProperty("recursive"), "true")); + } + + private HttpResponse getFlagData(FlagId flagId) { + FlagData data = flagsDb.getValue(flagId).orElseThrow(() -> new NotFoundException("Flag " + flagId + " not set")); + return new FlagDataResponse(data); + } + + private HttpResponse putFlagData(HttpRequest request, FlagId flagId) { + flagsDb.setValue(flagId, FlagData.deserialize(request.getData())); + + // The set & get is not atomic, but no harm is done in showing an outdated flag value in the response. + return getFlagData(flagId); + } + + private HttpResponse deleteFlagData(FlagId flagId) { + flagsDb.removeValue(flagId); + return new OKResponse(); + } + + private FlagId findFlagId(HttpRequest request, Path path) { + FlagId flagId = new FlagId(path.get("flagId")); + + if (!Objects.equals(request.getProperty("force"), "true")) { + if (Flags.getAllFlags().stream().noneMatch(definition -> flagId.equals(definition.getUnboundFlag().id()))) { + throw new NotFoundException("There is no flag '" + flagId + "' (use ?force=true to override)"); + } + } + + return flagId; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/OKResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/OKResponse.java new file mode 100644 index 00000000000..19363fbdadc --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/OKResponse.java @@ -0,0 +1,19 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.flags; + +import com.yahoo.container.jdisc.EmptyResponse; +import com.yahoo.jdisc.Response; + +/** + * @author hakonhall + */ +public class OKResponse extends EmptyResponse { + public OKResponse() { + super(Response.Status.OK); + } + + @Override + public String getContentType() { + return "application/json"; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/V1Response.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/V1Response.java new file mode 100644 index 00000000000..26eb96f1ef5 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/V1Response.java @@ -0,0 +1,29 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.flags; + +import com.yahoo.jdisc.Response; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.http.StaticResponse; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author hakonhall + */ +public class V1Response extends StaticResponse { + public V1Response(String flagsV1Uri) { + super(Response.Status.OK, HttpConfigResponse.JSON_CONTENT_TYPE, generateBody(flagsV1Uri)); + } + + private static String generateBody(String flagsV1Uri) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor data = root.setObject("data"); + data.setString("url", flagsV1Uri + "/data"); + return Utf8.toString(uncheck(() -> SlimeUtils.toJsonBytes(slime))); + } +} diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 5ba05a629c6..3e0572a8c99 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -60,7 +60,7 @@ - + @@ -103,6 +103,12 @@ http://*/status https://*/status + + http://*/flags/v1 + https://*/flags/v1 + http://*/flags/v1/* + https://*/flags/v1/* + http://*/application/v2/tenant/ https://*/application/v2/tenant/ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/flags/FlagsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/flags/FlagsHandlerTest.java new file mode 100644 index 00000000000..d88bd6ab4a5 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/flags/FlagsHandlerTest.java @@ -0,0 +1,161 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.flags; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.configserver.flags.db.FlagsDbImpl; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.UnboundFlag; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; + +/** + * @author hakonhall + */ +public class FlagsHandlerTest { + private static final UnboundFlag FLAG1 = + Flags.defineBoolean("id1", false, "desc1", "mod1"); + private static final UnboundFlag FLAG2 = + Flags.defineBoolean("id2", true, "desc2", "mod2", + FetchVector.Dimension.HOSTNAME, FetchVector.Dimension.APPLICATION_ID); + + private static final String FLAGS_V1_URL = "https://foo.com:4443/flags/v1"; + + private final FlagsDbImpl flagsDb = new FlagsDbImpl(new MockCurator()); + private final FlagsHandler handler = new FlagsHandler(FlagsHandler.testOnlyContext(), flagsDb); + + @Test + public void testV1() { + verifySuccessfulRequest(Method.GET, "", "", + "{\"data\":{\"url\":\"https://foo.com:4443/flags/v1/data\"}}"); + verifySuccessfulRequest(Method.GET, "/", "", + "{\"data\":{\"url\":\"https://foo.com:4443/flags/v1/data\"}}"); + } + + @Test + public void testData() { + // PUT flag with ID id1 + verifySuccessfulRequest(Method.PUT, "/data/" + FLAG1.id(), + "{\n" + + " \"rules\": [\n" + + " {\n" + + " \"value\": true\n" + + " }\n" + + " ]\n" + + "}", + "{\"rules\":[{\"value\":true}]}"); + + // GET on ID id1 should return the same as the put (this will also issue a payload for the get, + // which we assume will be ignored). + verifySuccessfulRequest(Method.GET, "/data/" + FLAG1.id(), + "", "{\"rules\":[{\"value\":true}]}"); + + // List all flags should list only id1 + verifySuccessfulRequest(Method.GET, "/data", + "", "{\"id1\":{\"url\":\"https://foo.com:4443/flags/v1/data/id1\"}}"); + + // Should be identical to above: suffix / on path should be ignored + verifySuccessfulRequest(Method.GET, "/data/", + "", "{\"id1\":{\"url\":\"https://foo.com:4443/flags/v1/data/id1\"}}"); + + // PUT id2 + verifySuccessfulRequest(Method.PUT, "/data/" + FLAG2.id(), + "{\n" + + " \"rules\": [\n" + + " {\n" + + " \"conditions\": [\n" + + " {\n" + + " \"type\": \"whitelist\",\n" + + " \"dimension\": \"hostname\",\n" + + " \"values\": [ \"host1\", \"host2\" ]\n" + + " },\n" + + " {\n" + + " \"type\": \"blacklist\",\n" + + " \"dimension\": \"application\",\n" + + " \"values\": [ \"app1\", \"app2\" ]\n" + + " }\n" + + " ],\n" + + " \"value\": true\n" + + " }\n" + + " ],\n" + + " \"attributes\": {\n" + + " \"zone\": \"zone1\"\n" + + " }\n" + + "}\n", + "{\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"app2\",\"app1\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}"); + + // The list of flag data should return id1 and id2 + verifySuccessfulRequest(Method.GET, "/data", + "", + "{\"id1\":{\"url\":\"https://foo.com:4443/flags/v1/data/id1\"},\"id2\":{\"url\":\"https://foo.com:4443/flags/v1/data/id2\"}}"); + + // Putting (overriding) id1 should work silently + verifySuccessfulRequest(Method.PUT, "/data/" + FLAG1.id(), + "{\n" + + " \"rules\": [\n" + + " {\n" + + " \"value\": false\n" + + " }\n" + + " ]\n" + + "}\n", + "{\"rules\":[{\"value\":false}]}"); + + // Get all recursivelly displays all flag data + verifySuccessfulRequest(Method.GET, "/data?recursive=true", "", + "{\"id1\":{\"rules\":[{\"value\":false}]},\"id2\":{\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"app2\",\"app1\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}}"); + + // Deleting both flags + verifySuccessfulRequest(Method.DELETE, "/data/" + FLAG1.id(), "", ""); + verifySuccessfulRequest(Method.DELETE, "/data/" + FLAG2.id(), "", ""); + + // And the list of data flags should now be empty + verifySuccessfulRequest(Method.GET, "/data", "", "{}"); + } + + @Test + public void testForcing() { + FlagId undefinedFlagId = new FlagId("undef"); + HttpResponse response = handle(Method.PUT, "/data/" + undefinedFlagId, ""); + + assertEquals(404, response.getStatus()); + assertEquals("application/json", response.getContentType()); + + } + + private void verifySuccessfulRequest(Method method, String pathSuffix, String requestBody, String expectedResponseBody) { + HttpResponse response = handle(method, pathSuffix, requestBody); + + assertEquals(200, response.getStatus()); + assertEquals("application/json", response.getContentType()); + String actualResponse = uncheck(() -> SessionHandlerTest.getRenderedString(response)); + + assertThat(actualResponse, is(expectedResponseBody)); + } + + private HttpResponse handle(Method method, String pathSuffix, String requestBody) { + String uri = FLAGS_V1_URL + pathSuffix; + HttpRequest request = HttpRequest.createTestRequest(uri, method, makeInputStream(requestBody)); + return handler.handle(request); + } + + private String makeUrl(String component) { + return FLAGS_V1_URL + "/" + component; + } + + private InputStream makeInputStream(String content) { + return new ByteArrayInputStream(Utf8.toBytes(content)); + } +} \ No newline at end of file diff --git a/flags/pom.xml b/flags/pom.xml index 5a535ad4de8..ade598556de 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -13,9 +13,10 @@ flags - 6-SNAPSHOT container-plugin + 6-SNAPSHOT ${project.artifactId} + Feature flags. @@ -55,6 +56,14 @@ bundle-plugin true + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Deserializer.java b/flags/src/main/java/com/yahoo/vespa/flags/Deserializer.java new file mode 100644 index 00000000000..7ececa5489a --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/Deserializer.java @@ -0,0 +1,10 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +/** + * @author hakonhall + */ +@FunctionalInterface +public interface Deserializer { + T deserialize(RawFlag rawFlag); +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FeatureFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/FeatureFlag.java deleted file mode 100644 index 7cc5529646e..00000000000 --- a/flags/src/main/java/com/yahoo/vespa/flags/FeatureFlag.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags; - -import java.util.function.Function; - -/** - * A FeatureFlag is a boolean flag. - * - * @author hakonhall - */ -public class FeatureFlag implements Flag { - private final boolean defaultValue; - private final FlagId id; - private final FlagSource source; - - public static Function createUnbound(String flagId, boolean defaultValue) { - return createUnbound(new FlagId(flagId), defaultValue); - } - - public static Function createUnbound(FlagId id, boolean defaultValue) { - return source -> new FeatureFlag(id, defaultValue, source); - } - - public FeatureFlag(String flagId, boolean defaultValue, FlagSource source) { - this(new FlagId(flagId), defaultValue, source); - } - - public FeatureFlag(FlagId id, boolean defaultValue, FlagSource source) { - this.id = id; - this.defaultValue = defaultValue; - this.source = source; - } - - @Override - public FlagId id() { - return id; - } - - public boolean value() { - return source.getString(id).map(FeatureFlag::booleanFromString).orElse(defaultValue); - } - - private static boolean booleanFromString(String string) { - String canonicalString = string.trim().toLowerCase(); - if (canonicalString.equals("true")) { - return true; - } else if (canonicalString.equals("false")) { - return false; - } - - throw new IllegalArgumentException("Unable to convert to true or false: '" + string + "'"); - } - - @Override - public String toString() { - return "IntFlag{" + - "id=" + id + - ", defaultValue=" + defaultValue + - ", source=" + source + - '}'; - } -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java new file mode 100644 index 00000000000..581ec599aab --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -0,0 +1,82 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import javax.annotation.concurrent.Immutable; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; + +/** + * Denotes which RawFlag should be retrieved from {@link FlagSource} for a given {@link FlagId}, + * as the raw flag may depend on the hostname, application, etc. + * + * @author hakonhall + */ +@Immutable +public +class FetchVector { + public enum Dimension { + /** Value from ZoneId::value */ + ZONE_ID, + /** Value from ApplicationId::serializedForm */ + APPLICATION_ID, + /** Fully qualified hostname */ + HOSTNAME + } + + private final Map map; + + public FetchVector() { + this.map = Collections.emptyMap(); + } + + public static FetchVector fromMap(Map map) { + return new FetchVector(new HashMap<>(map)); + } + + private FetchVector(Map map) { + this.map = Collections.unmodifiableMap(map); + } + + public Optional getValue(Dimension dimension) { + return Optional.ofNullable(map.get(dimension)); + } + + public Map toMap() { + return map; + } + + /** Returns a new FetchVector, identical to {@code this} except for its value in {@code dimension}. */ + public FetchVector with(Dimension dimension, String value) { + return makeFetchVector(merged -> merged.put(dimension, value)); + } + + /** Returns a new FetchVector, identical to {@code this} except for its values in the override's dimensions. */ + public FetchVector with(FetchVector override) { + return makeFetchVector(vector -> vector.putAll(override.map)); + } + + private FetchVector makeFetchVector(Consumer> mapModifier) { + EnumMap mergedMap = new EnumMap<>(Dimension.class); + mergedMap.putAll(map); + mapModifier.accept(mergedMap); + return new FetchVector(mergedMap); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FetchVector that = (FetchVector) o; + return Objects.equals(map, that.map); + } + + @Override + public int hashCode() { + return Objects.hash(map); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java index 1a856fee60a..3403a15d7ff 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java @@ -1,7 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.Rule; import java.io.IOException; import java.io.UncheckedIOException; @@ -11,14 +14,15 @@ import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.util.Collections; import java.util.Optional; /** - * A {@link FlagSource} backed by local files. - * * @author hakonhall */ public class FileFlagSource implements FlagSource { + private static final ObjectMapper mapper = new ObjectMapper(); + static final String FLAGS_DIRECTORY = "/etc/vespa/flags"; private final Path flagsDirectory; @@ -37,13 +41,36 @@ public class FileFlagSource implements FlagSource { } @Override - public Optional getString(FlagId id) { - return getBytes(id).map(bytes -> new String(bytes, StandardCharsets.UTF_8)); + public Optional fetch(FlagId flagId, FetchVector vector) { + return getResolver(flagId).resolve(vector); + } + + private FlagData getResolver(FlagId flagId) { + Optional v2String = getString(flagId, ".2"); + if (v2String.isPresent()) { + return FlagData.deserialize(v2String.get()); + } + + Optional v1String = getString(flagId, ""); + if (v1String.isPresent()) { + // version 1: File contains value as a JSON + // version 2: File contains FileResolver as a JSON (which may contain many values, one for each rule) + // version 1 files should probably be discontinued + Rule rule = new Rule(Optional.of(JsonNodeRawFlag.fromJson(v1String.get())), Collections.emptyList()); + return new FlagData(new FetchVector(), Collections.singletonList(rule)); + } + + // Will eventually resolve to empty RawFlag + return new FlagData(); + } + + private Optional getString(FlagId id, String suffix) { + return getBytes(id, suffix).map(bytes -> new String(bytes, StandardCharsets.UTF_8)); } - public Optional getBytes(FlagId id) { + private Optional getBytes(FlagId id, String suffix) { try { - return Optional.of(Files.readAllBytes(getPath(id))); + return Optional.of(Files.readAllBytes(getPath(id, suffix))); } catch (NoSuchFileException e) { return Optional.empty(); } catch (IOException e) { @@ -51,8 +78,8 @@ public class FileFlagSource implements FlagSource { } } - private Path getPath(FlagId id) { - return flagsDirectory.resolve(id.toString()); + private Path getPath(FlagId id, String suffix) { + return flagsDirectory.resolve(id.toString() + suffix); } @Override diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flag.java b/flags/src/main/java/com/yahoo/vespa/flags/Flag.java index 831e0d0dab9..18c1db0d756 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flag.java @@ -1,9 +1,44 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; +import javax.annotation.concurrent.Immutable; + /** * @author hakonhall */ -public interface Flag { - FlagId id(); +@Immutable +public class Flag { + private final FlagId id; + private final T defaultValue; + private final FlagSource source; + private final Deserializer deserializer; + private final FetchVector fetchVector; + + public Flag(String flagId, T defaultValue, FlagSource source, Deserializer deserializer) { + this(new FlagId(flagId), defaultValue, source, deserializer); + } + + public Flag(FlagId id, T defaultValue, FlagSource source, Deserializer deserializer) { + this(id, defaultValue, deserializer, new FetchVector(), source); + } + + public Flag(FlagId id, T defaultValue, Deserializer deserializer, FetchVector fetchVector, FlagSource source) { + this.id = id; + this.defaultValue = defaultValue; + this.source = source; + this.deserializer = deserializer; + this.fetchVector = fetchVector; + } + + public FlagId id() { + return id; + } + + public Flag with(FetchVector.Dimension dimension, String value) { + return new Flag<>(id, defaultValue, deserializer, fetchVector.with(dimension, value), source); + } + + public T value() { + return source.fetch(id, fetchVector).map(deserializer::deserialize).orElse(defaultValue); + } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java new file mode 100644 index 00000000000..a3f490e2f96 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java @@ -0,0 +1,41 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import javax.annotation.concurrent.Immutable; +import java.util.Collections; +import java.util.List; + +/** + * @author hakonhall + */ +@Immutable +public class FlagDefinition { + private final UnboundFlag unboundFlag; + private final String description; + private final String modificationEffect; + private final List dimensions; + + public FlagDefinition(UnboundFlag unboundFlag, String description, String modificationEffect, + List dimensions) { + this.unboundFlag = unboundFlag; + this.description = description; + this.modificationEffect = modificationEffect; + this.dimensions = Collections.unmodifiableList(dimensions); + } + + public UnboundFlag getUnboundFlag() { + return unboundFlag; + } + + public List getDimensions() { + return dimensions; + } + + public String getDescription() { + return description; + } + + public String getModificationEffect() { + return modificationEffect; + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagId.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagId.java index f004df063ed..ae38fbe7dc1 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagId.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagId.java @@ -9,7 +9,7 @@ import java.util.regex.Pattern; * @author hakonhall */ @Immutable -public class FlagId { +public class FlagId implements Comparable { private static final Pattern ID_PATTERN = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9._-]*$"); private final String id; @@ -22,6 +22,11 @@ public class FlagId { this.id = id; } + @Override + public int compareTo(FlagId that) { + return this.id.compareTo(that.id); + } + @Override public String toString() { return id; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagSerializer.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagSerializer.java new file mode 100644 index 00000000000..697cbfcbb4a --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagSerializer.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +/** + * @author hakonhall + */ +public interface FlagSerializer extends Serializer, Deserializer { +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java index 509db40c4d4..da8e6b29cab 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java @@ -7,6 +7,5 @@ import java.util.Optional; * @author hakonhall */ public interface FlagSource { - /** The String value of a flag, or empty if not set by the source. */ - Optional getString(FlagId id); + Optional fetch(FlagId id, FetchVector vector); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java new file mode 100644 index 00000000000..b66ba9fc0c9 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -0,0 +1,130 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.LongNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.yahoo.vespa.defaults.Defaults; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Definitions of most/all flags. + * + *

The flags are centrally defined in this module to allow 1. all code to access flags that may be used in + * quite different modules and processes, and 2. in particular allow the config server to access all flags + * so operators have a nicer UI for setting, modifying, or removing flag values. + * + *

This class should have been an enum, but unfortunately enums cannot be generic, which will eventually be + * fixed with JEP 301: Enhanced Enums. + * + * @author hakonhall + */ +public class Flags { + public static final FlagSerializer BOOLEAN_SERIALIZER = new SimpleFlagSerializer<>(BooleanNode::valueOf, JsonNode::isBoolean, JsonNode::asBoolean); + public static final FlagSerializer STRING_SERIALIZER = new SimpleFlagSerializer<>(TextNode::new, JsonNode::isTextual, JsonNode::asText); + public static final FlagSerializer INT_SERIALIZER = new SimpleFlagSerializer<>(IntNode::new, JsonNode::isIntegralNumber, JsonNode::asInt); + public static final FlagSerializer LONG_SERIALIZER = new SimpleFlagSerializer<>(LongNode::new, JsonNode::isIntegralNumber, JsonNode::asLong); + + private static final ConcurrentHashMap> flags = new ConcurrentHashMap<>(); + + public static final UnboundFlag HEALTHMONITOR_MONITOR_INFRA = defineBoolean( + "healthmonitor-monitorinfra", true, + "Whether the health monitor in service monitor monitors the health of infrastructure applications.", + "Affects all applications activated after the value is changed.", + FetchVector.Dimension.HOSTNAME); + + public static final UnboundFlag DUPERMODEL_CONTAINS_INFRA = defineBoolean( + "dupermodel-contains-infra", true, + "Whether the DuperModel in config server/controller includes active infrastructure applications " + + "(except from controller/config apps).", + "Requires restart of config server/controller to take effect.", + FetchVector.Dimension.HOSTNAME); + + public static final UnboundFlag DUPERMODEL_USE_CONFIGSERVERCONFIG = defineBoolean( + "dupermodel-use-configserverconfig", true, + "For historical reasons, the ApplicationInfo in the DuperModel for controllers and config servers " + + "is based on the ConfigserverConfig (this flag is true). We want to transition to use the " + + "infrastructure application activated by the InfrastructureProvisioner once that supports health.", + "Requires restart of config server/controller to take effect.", + FetchVector.Dimension.HOSTNAME); + + public static final UnboundFlag USE_CONFIG_SERVER_CACHE = defineBoolean( + "use-config-server-cache", true, + "Whether config server will use cache to answer config requests.", + "Takes effect immediately when changed.", + FetchVector.Dimension.HOSTNAME, FetchVector.Dimension.APPLICATION_ID); + + public static final UnboundFlag CONFIG_SERVER_BOOTSTRAP_IN_SEPARATE_THREAD = defineBoolean( + "config-server-bootstrap-in-separate-thread", true, + "Whether to run config server/controller bootstrap in a separate thread.", + "Takes effect only at bootstrap of config server/controller", + FetchVector.Dimension.HOSTNAME); + + public static UnboundFlag defineBoolean(String flagId, boolean defaultValue, String description, + String modificationEffect, FetchVector.Dimension... dimensions) { + return define(flagId, defaultValue, BOOLEAN_SERIALIZER, description, modificationEffect, dimensions); + } + + public static UnboundFlag defineString(String flagId, String defaultValue, String description, + String modificationEffect, FetchVector.Dimension... dimensions) { + return define(flagId, defaultValue, STRING_SERIALIZER, description, modificationEffect, dimensions); + } + + public static UnboundFlag defineInt(String flagId, Integer defaultValue, String description, + String modificationEffect, FetchVector.Dimension... dimensions) { + return define(flagId, defaultValue, INT_SERIALIZER, description, modificationEffect, dimensions); + } + + public static UnboundFlag defineLong(String flagId, Long defaultValue, String description, + String modificationEffect, FetchVector.Dimension... dimensions) { + return define(flagId, defaultValue, LONG_SERIALIZER, description, modificationEffect, dimensions); + } + + public static UnboundFlag defineJackson(String flagId, Class jacksonClass, T defaultValue, String description, + String modificationEffect, FetchVector.Dimension... dimensions) { + return define(flagId, defaultValue, new JacksonSerializer<>(jacksonClass), description, modificationEffect, dimensions); + } + + /** + * Defines a Flag. + * + * @param flagId The globally unique FlagId. + * @param defaultValue The default value if none is present after resolution. + * @param deserializer Deserialize JSON to value type. + * @param description Description of how the flag is used. + * @param modificationEffect What is required for the flag to take effect? A restart of process? immediately? etc. + * @param dimensions What dimensions will be set in the {@link FetchVector} when fetching + * the flag value in + * {@link FlagSource#fetch(FlagId, FetchVector) FlagSource::fetch}. + * For instance, if APPLICATION is one of the dimensions here, you should make sure + * APPLICATION is set to the ApplicationId in the fetch vector when fetching the RawFlag + * from the FlagSource. + * @param The type of the flag value, typically Boolean for flags guarding features. + * @return An unbound flag with {@link FetchVector.Dimension#HOSTNAME HOSTNAME} environment. The ZONE environment + * is typically implicit. + */ + private static UnboundFlag define(String flagId, T defaultValue, Deserializer deserializer, + String description, String modificationEffect, + FetchVector.Dimension... dimensions) { + UnboundFlag flag = new UnboundFlag<>(flagId, defaultValue, deserializer) + .with(FetchVector.Dimension.HOSTNAME, Defaults.getDefaults().vespaHostname()); + FlagDefinition definition = new FlagDefinition<>(flag, description, modificationEffect, Arrays.asList(dimensions)); + flags.put(flag.id(), definition); + return flag; + } + + public static List> getAllFlags() { + return new ArrayList<>(flags.values()); + } + + public static Optional> getFlag(FlagId flagId) { + return Optional.ofNullable(flags.get(flagId)); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java deleted file mode 100644 index f7c9645c5db..00000000000 --- a/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags; - -import java.util.function.Function; - -/** - * @author hakonhall - */ -public class IntFlag implements Flag { - private final FlagId id; - private final int defaultValue; - private final FlagSource source; - - public static Function createUnbound(String flagId, int defaultValue) { - return createUnbound(new FlagId(flagId), defaultValue); - } - - public static Function createUnbound(FlagId id, int defaultValue) { - return source -> new IntFlag(id, defaultValue, source); - } - - public IntFlag(String flagId, int defaultValue, FlagSource source) { - this(new FlagId(flagId), defaultValue, source); - } - - public IntFlag(FlagId id, int defaultValue, FlagSource source) { - this.id = id; - this.defaultValue = defaultValue; - this.source = source; - } - - @Override - public FlagId id() { - return id; - } - - public int value() { - return source.getString(id).map(String::trim).map(Integer::parseInt).orElse(defaultValue); - } - - @Override - public String toString() { - return "IntFlag{" + - "id=" + id + - ", defaultValue=" + defaultValue + - ", source=" + source + - '}'; - } -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java deleted file mode 100644 index 99add358e75..00000000000 --- a/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.function.Function; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author hakonhall - */ -public class JacksonFlag implements Flag { - private final static ObjectMapper mapper = new ObjectMapper(); - - private final FlagId id; - private final Class jacksonClass; - private final T defaultValue; - private final FlagSource source; - - public static Function> createUnbound(String flagId, Class jacksonClass, T defaultValue) { - return createUnbound(new FlagId(flagId), jacksonClass, defaultValue); - } - - public static Function> createUnbound(FlagId id, Class jacksonClass, T defaultValue) { - return source -> new JacksonFlag<>(id, jacksonClass, defaultValue, source); - } - - public JacksonFlag(String flagId, Class jacksonClass, T defaultValue, FlagSource source) { - this(new FlagId(flagId), jacksonClass, defaultValue, source); - } - - public JacksonFlag(FlagId id, Class jacksonClass, T defaultValue, FlagSource source) { - this.id = id; - this.jacksonClass = jacksonClass; - this.defaultValue = defaultValue; - this.source = source; - } - - @Override - public FlagId id() { - return id; - } - - public T value() { - return source.getString(id).map(string -> uncheck(() -> mapper.readValue(string, jacksonClass))).orElse(defaultValue); - } - - @Override - public String toString() { - return "JacksonFlag{" + - "id=" + id + - ", jacksonClass=" + jacksonClass + - ", defaultValue=" + defaultValue + - ", source=" + source + - '}'; - } -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/JacksonSerializer.java b/flags/src/main/java/com/yahoo/vespa/flags/JacksonSerializer.java new file mode 100644 index 00000000000..b6a4c9b87c6 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/JacksonSerializer.java @@ -0,0 +1,23 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +/** + * @author hakonhall + */ +public class JacksonSerializer implements FlagSerializer { + private final Class jacksonClass; + + public JacksonSerializer(Class jacksonClass) { + this.jacksonClass = jacksonClass; + } + + @Override + public T deserialize(RawFlag rawFlag) { + return JsonNodeRawFlag.fromJsonNode(rawFlag.asJsonNode()).toJacksonClass(jacksonClass); + } + + @Override + public RawFlag serialize(T value) { + return JsonNodeRawFlag.fromJacksonClass(value); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/JsonNodeRawFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/JsonNodeRawFlag.java new file mode 100644 index 00000000000..f41dd9f9e5c --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/JsonNodeRawFlag.java @@ -0,0 +1,48 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * {@link RawFlag} using Jackson's {@link JsonNode}. + * + * @author hakonhall + */ +public class JsonNodeRawFlag implements RawFlag { + private static final ObjectMapper mapper = new ObjectMapper(); + + private final JsonNode jsonNode; + + private JsonNodeRawFlag(JsonNode jsonNode) { + this.jsonNode = jsonNode; + } + + public static JsonNodeRawFlag fromJson(String json) { + return new JsonNodeRawFlag(uncheck(() -> mapper.readTree(json))); + } + + public static JsonNodeRawFlag fromJsonNode(JsonNode jsonNode) { + return new JsonNodeRawFlag(jsonNode); + } + + public static JsonNodeRawFlag fromJacksonClass(T value) { + return new JsonNodeRawFlag(uncheck(() -> mapper.valueToTree(value))); + } + + public T toJacksonClass(Class jacksonClass) { + return uncheck(() -> mapper.treeToValue(jsonNode, jacksonClass)); + } + + @Override + public JsonNode asJsonNode() { + return jsonNode; + } + + @Override + public String asJson() { + return jsonNode.toString(); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java deleted file mode 100644 index d60dc7b5adc..00000000000 --- a/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags; - -import java.util.function.Function; - -/** - * @author hakonhall - */ -public class LongFlag implements Flag { - private final FlagId id; - private final long defaultValue; - private final FlagSource source; - - public static Function createUnbound(String flagId, int defaultValue) { - return createUnbound(new FlagId(flagId), defaultValue); - } - - public static Function createUnbound(FlagId id, int defaultValue) { - return source -> new LongFlag(id, defaultValue, source); - } - - public LongFlag(String flagId, long defaultValue, FlagSource source) { - this(new FlagId(flagId), defaultValue, source); - } - - public LongFlag(FlagId id, long defaultValue, FlagSource source) { - this.id = id; - this.defaultValue = defaultValue; - this.source = source; - } - - @Override - public FlagId id() { - return id; - } - - public long value() { - return source.getString(id).map(String::trim).map(Long::parseLong).orElse(defaultValue); - } - - @Override - public String toString() { - return "LongFlag{" + - "id=" + id + - ", defaultValue=" + defaultValue + - ", source=" + source + - '}'; - } -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/OrderedFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/OrderedFlagSource.java new file mode 100644 index 00000000000..6ca74715999 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/OrderedFlagSource.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +/** + * A {@link FlagSource#fetch(FlagId, FetchVector) fetch} on this flag source will return the {@link RawFlag} + * from the first (highest priority) flag source that returns a raw flag for the fetch vector. + * + * @author hakonhall + */ +public class OrderedFlagSource implements FlagSource { + private final List sources; + + /** + * + * @param sources Flag sources in descending priority order. + */ + public OrderedFlagSource(FlagSource... sources) { + this.sources = Arrays.asList(sources); + } + + @Override + public Optional fetch(FlagId id, FetchVector vector) { + return sources.stream() + .map(source -> source.fetch(id, vector)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/RawFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/RawFlag.java new file mode 100644 index 00000000000..2308659470e --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/RawFlag.java @@ -0,0 +1,14 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * A {@link RawFlag} represents the typeless flag value, possibly partially deserialized. + * + * @author hakonhall + */ +public interface RawFlag { + JsonNode asJsonNode(); + String asJson(); +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Serializer.java b/flags/src/main/java/com/yahoo/vespa/flags/Serializer.java new file mode 100644 index 00000000000..3569b10c8f4 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/Serializer.java @@ -0,0 +1,10 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +/** + * @author hakonhall + */ +@FunctionalInterface +public interface Serializer { + RawFlag serialize(T value); +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/SimpleFlagSerializer.java b/flags/src/main/java/com/yahoo/vespa/flags/SimpleFlagSerializer.java new file mode 100644 index 00000000000..2340588f54a --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/SimpleFlagSerializer.java @@ -0,0 +1,39 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.util.function.Function; +import java.util.function.Predicate; + +/** + * @author hakonhall + */ +public class SimpleFlagSerializer implements FlagSerializer { + private final Function serializer; + private final Predicate isCorrectType; + private final Function deserializer; + + public SimpleFlagSerializer(Function serializer, + Predicate isCorrectType, + Function deserializer) { + this.serializer = serializer; + this.isCorrectType = isCorrectType; + this.deserializer = deserializer; + } + + @Override + public JsonNodeRawFlag serialize(T value) { + return JsonNodeRawFlag.fromJsonNode(serializer.apply(value)); + } + + @Override + public T deserialize(RawFlag rawFlag) { + JsonNode jsonNode = rawFlag.asJsonNode(); + if (!isCorrectType.test(jsonNode)) { + throw new IllegalArgumentException("Wrong type of JsonNode: " + jsonNode.getNodeType()); + } + + return deserializer.apply(jsonNode); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java deleted file mode 100644 index 8226e999238..00000000000 --- a/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags; - -import java.util.function.Function; - -/** - * @author hakonhall - */ -public class StringFlag implements Flag { - private final FlagId id; - private final String defaultValue; - private final FlagSource source; - - public static Function createUnbound(String flagId, String defaultValue) { - return createUnbound(new FlagId(flagId), defaultValue); - } - - public static Function createUnbound(FlagId id, String defaultValue) { - return source -> new StringFlag(id, defaultValue, source); - } - - public StringFlag(String flagId, String defaultValue, FlagSource source) { - this(new FlagId(flagId), defaultValue, source); - } - - public StringFlag(FlagId id, String defaultValue, FlagSource source) { - this.id = id; - this.defaultValue = defaultValue; - this.source = source; - } - - @Override - public FlagId id() { - return id; - } - - public String value() { - return source.getString(id).orElse(defaultValue); - } - - @Override - public String toString() { - return "StringFlag{" + - "id=" + id + - ", defaultValue='" + defaultValue + '\'' + - ", source=" + source + - '}'; - } -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlag.java new file mode 100644 index 00000000000..597e19080ab --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlag.java @@ -0,0 +1,38 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import javax.annotation.concurrent.Immutable; + +/** + * @author hakonhall + */ +@Immutable +public class UnboundFlag { + private final FlagId id; + private final T defaultValue; + private final Deserializer deserializer; + private final FetchVector fetchVector; + + public UnboundFlag(String flagId, T defaultValue, Deserializer deserializer) { + this(new FlagId(flagId), defaultValue, deserializer, new FetchVector()); + } + + public UnboundFlag(FlagId id, T defaultValue, Deserializer deserializer, FetchVector fetchVector) { + this.id = id; + this.defaultValue = defaultValue; + this.deserializer = deserializer; + this.fetchVector = fetchVector; + } + + public FlagId id() { + return id; + } + + public UnboundFlag with(FetchVector.Dimension dimension, String value) { + return new UnboundFlag<>(id, defaultValue, deserializer, fetchVector.with(dimension, value)); + } + + public Flag bindTo(FlagSource source) { + return new Flag<>(id, defaultValue, deserializer, fetchVector, source); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java new file mode 100644 index 00000000000..c07232bfe66 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java @@ -0,0 +1,65 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json; + +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.json.wire.DimensionHelper; +import com.yahoo.vespa.flags.json.wire.WireCondition; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; + +/** + * @author hakonhall + */ +public class Condition implements Predicate { + public enum Type { WHITELIST, BLACKLIST } + + private final Type type; + private final FetchVector.Dimension dimension; + private final Set values; + + public Condition(Type type, FetchVector.Dimension dimension, String... values) { + this(type, dimension, new HashSet<>(Arrays.asList(values))); + } + + public Condition(Type type, FetchVector.Dimension dimension, Set values) { + this.type = type; + this.dimension = dimension; + this.values = values; + } + + @Override + public boolean test(FetchVector vector) { + boolean isMember = vector.getValue(dimension).filter(values::contains).isPresent(); + + switch (type) { + case WHITELIST: return isMember; + case BLACKLIST: return !isMember; + default: throw new IllegalArgumentException("Unknown type " + type); + } + } + + public static Condition fromWire(WireCondition wireCondition) { + Objects.requireNonNull(wireCondition.type); + Type type = Type.valueOf(wireCondition.type.toUpperCase()); + + Objects.requireNonNull(wireCondition.dimension); + FetchVector.Dimension dimension = DimensionHelper.fromWire(wireCondition.dimension); + + Set values = wireCondition.values == null ? Collections.emptySet() : wireCondition.values; + + return new Condition(type, dimension, values); + } + + public WireCondition toWire() { + WireCondition wire = new WireCondition(); + wire.type = type.name().toLowerCase(); + wire.dimension = DimensionHelper.toWire(dimension); + wire.values = values.isEmpty() ? null : values; + return wire; + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java new file mode 100644 index 00000000000..9ee3b2dcd85 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java @@ -0,0 +1,104 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.RawFlag; +import com.yahoo.vespa.flags.json.wire.FetchVectorHelper; +import com.yahoo.vespa.flags.json.wire.WireFlagData; +import com.yahoo.vespa.flags.json.wire.WireRule; + +import javax.annotation.concurrent.Immutable; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * A data structure containing all data for a single flag, that can be serialized to/from JSON, + * and that can be used to implement {@link FlagSource}. + * + * @author hakonhall + */ +@Immutable +public class FlagData { + private final List rules; + private final FetchVector defaultFetchVector; + + public FlagData() { + this(new FetchVector(), Collections.emptyList()); + } + + public FlagData(FetchVector defaultFetchVector, Rule... rules) { + this(defaultFetchVector, Arrays.asList(rules)); + } + + public FlagData(FetchVector defaultFetchVector, List rules) { + this.rules = Collections.unmodifiableList(new ArrayList<>(rules)); + this.defaultFetchVector = defaultFetchVector; + } + + public Optional resolve(FetchVector fetchVector) { + return rules.stream() + .filter(rule -> rule.match(defaultFetchVector.with(fetchVector))) + .findFirst() + .flatMap(Rule::getValueToApply); + } + + public String serializeToJson() { + return toWire().serializeToJson(); + } + + public byte[] serializeToUtf8Json() { + return toWire().serializeToBytes(); + } + + public void serializeToOutputStream(OutputStream outputStream) { + toWire().serializeToOutputStream(outputStream); + } + + public JsonNode toJsonNode() { + return toWire().serializeToJsonNode(); + } + + private WireFlagData toWire() { + WireFlagData wireFlagData = new WireFlagData(); + + if (!rules.isEmpty()) { + wireFlagData.rules = rules.stream().map(Rule::toWire).collect(Collectors.toList()); + } + + wireFlagData.defaultFetchVector = FetchVectorHelper.toWire(defaultFetchVector); + + return wireFlagData; + } + + public static FlagData deserializeUtf8Json(byte[] bytes) { + return fromWire(WireFlagData.deserialize(bytes)); + } + + public static FlagData deserialize(InputStream inputStream) { + return fromWire(WireFlagData.deserialize(inputStream)); + } + + public static FlagData deserialize(String string) { + return fromWire(WireFlagData.deserialize(string)); + } + + private static FlagData fromWire(WireFlagData wireFlagData) { + return new FlagData( + FetchVectorHelper.fromWire(wireFlagData.defaultFetchVector), rulesFromWire(wireFlagData.rules) + ); + } + + private static List rulesFromWire(List wireRules) { + if (wireRules == null) return Collections.emptyList(); + return wireRules.stream().map(Rule::fromWire).collect(Collectors.toList()); + } +} + diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java b/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java new file mode 100644 index 00000000000..b7d60889419 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java @@ -0,0 +1,58 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json; + +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.JsonNodeRawFlag; +import com.yahoo.vespa.flags.RawFlag; +import com.yahoo.vespa.flags.json.wire.WireRule; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author hakonhall + */ +public class Rule { + private final List andConditions; + private final Optional valueToApply; + + public Rule(Optional valueToApply, Condition... andConditions) { + this(valueToApply, Arrays.asList(andConditions)); + } + + public Rule(Optional valueToApply, List andConditions) { + this.andConditions = andConditions; + this.valueToApply = valueToApply; + } + + public boolean match(FetchVector fetchVector) { + return andConditions.stream().allMatch(condition -> condition.test(fetchVector)); + } + + public Optional getValueToApply() { + return valueToApply; + } + + public WireRule toWire() { + WireRule wireRule = new WireRule(); + + if (!andConditions.isEmpty()) { + wireRule.andConditions = andConditions.stream().map(Condition::toWire).collect(Collectors.toList()); + } + + wireRule.value = valueToApply.map(RawFlag::asJsonNode).orElse(null); + + return wireRule; + } + + public static Rule fromWire(WireRule wireRule) { + List conditions = wireRule.andConditions == null ? + Collections.emptyList() : + wireRule.andConditions.stream().map(Condition::fromWire).collect(Collectors.toList()); + Optional value = wireRule.value == null ? Optional.empty() : Optional.of(JsonNodeRawFlag.fromJsonNode(wireRule.value)); + return new Rule(value, conditions); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/package-info.java b/flags/src/main/java/com/yahoo/vespa/flags/json/package-info.java new file mode 100644 index 00000000000..6fbe3d587c5 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.flags.json; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/DimensionHelper.java new file mode 100644 index 00000000000..e2cb6dd0932 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/DimensionHelper.java @@ -0,0 +1,48 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json.wire; + +import com.yahoo.vespa.flags.FetchVector; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author hakonhall + */ +public class DimensionHelper { + private static Map serializedDimensions = new HashMap<>(); + static { + serializedDimensions.put(FetchVector.Dimension.ZONE_ID, "zone"); + serializedDimensions.put(FetchVector.Dimension.HOSTNAME, "hostname"); + serializedDimensions.put(FetchVector.Dimension.APPLICATION_ID, "application"); + + if (serializedDimensions.size() != FetchVector.Dimension.values().length) { + throw new IllegalStateException(FetchVectorHelper.class.getName() + " is not in sync with " + + FetchVector.Dimension.class.getName()); + } + } + + private static Map deserializedDimensions = serializedDimensions. + entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + public static String toWire(FetchVector.Dimension dimension) { + String serializedDimension = serializedDimensions.get(dimension); + if (serializedDimension == null) { + throw new IllegalArgumentException("Unsupported dimension (please add it): '" + dimension + "'"); + } + + return serializedDimension; + } + + public static FetchVector.Dimension fromWire(String serializedDimension) { + FetchVector.Dimension dimension = deserializedDimensions.get(serializedDimension); + if (dimension == null) { + throw new IllegalArgumentException("Unknown serialized dimension: '" + serializedDimension + "'"); + } + + return dimension; + } + + private DimensionHelper() { } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/FetchVectorHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/FetchVectorHelper.java new file mode 100644 index 00000000000..834e4024a6a --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/FetchVectorHelper.java @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json.wire; + +import com.yahoo.vespa.flags.FetchVector; + +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author hakonhall + */ +public class FetchVectorHelper { + public static Map toWire(FetchVector vector) { + Map map = vector.toMap(); + if (map.isEmpty()) return null; + return map.entrySet().stream().collect(Collectors.toMap( + entry -> DimensionHelper.toWire(entry.getKey()), + Map.Entry::getValue)); + } + + public static FetchVector fromWire(Map wireMap) { + if (wireMap == null) return new FetchVector(); + return FetchVector.fromMap(wireMap.entrySet().stream().collect(Collectors.toMap( + entry -> DimensionHelper.fromWire(entry.getKey()), + Map.Entry::getValue))); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireCondition.java new file mode 100644 index 00000000000..31aa6200fdb --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireCondition.java @@ -0,0 +1,19 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json.wire; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Set; + +/** + * @author hakonhall + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireCondition { + @JsonProperty("type") public String type; + @JsonProperty("dimension") public String dimension; + @JsonProperty("values") public Set values; +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagData.java new file mode 100644 index 00000000000..b4a000d7b70 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagData.java @@ -0,0 +1,55 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json.wire; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import java.util.Map; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author hakonhall + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireFlagData { + @JsonProperty("rules") public List rules; + @JsonProperty("attributes") public Map defaultFetchVector; + + private static final ObjectMapper mapper = new ObjectMapper(); + + public byte[] serializeToBytes() { + return uncheck(() -> mapper.writeValueAsBytes(this)); + } + + public String serializeToJson() { + return uncheck(() -> mapper.writeValueAsString(this)); + } + + public JsonNode serializeToJsonNode() { + return uncheck(() -> mapper.valueToTree(this)); + } + + public void serializeToOutputStream(OutputStream outputStream) { + uncheck(() -> mapper.writeValue(outputStream, this)); + } + + public static WireFlagData deserialize(byte[] bytes) { + return uncheck(() -> mapper.readValue(bytes, WireFlagData.class)); + } + + public static WireFlagData deserialize(String string) { + return uncheck(() -> mapper.readValue(string, WireFlagData.class)); + } + + public static WireFlagData deserialize(InputStream inputStream) { + return uncheck(() -> mapper.readValue(inputStream, WireFlagData.class)); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireRule.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireRule.java new file mode 100644 index 00000000000..38619e87488 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireRule.java @@ -0,0 +1,19 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json.wire; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; + +import java.util.List; + +/** + * @author hakonhall + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireRule { + @JsonProperty("conditions") public List andConditions; + @JsonProperty("value") public JsonNode value; +} diff --git a/flags/src/test/java/com/yahoo/vespa/flags/FileFlagSourceTest.java b/flags/src/test/java/com/yahoo/vespa/flags/FileFlagSourceTest.java index 9e7508706ed..7d9d7868308 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/FileFlagSourceTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/FileFlagSourceTest.java @@ -5,9 +5,11 @@ import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; @@ -22,18 +24,18 @@ public class FileFlagSourceTest { @Test public void testFeatureLikeFlags() throws IOException { - FeatureFlag featureFlag = new FeatureFlag(id, false, source); - FeatureFlag byDefaultTrue = new FeatureFlag(id, true, source); + Flag featureFlag = new Flag<>(id, false, source, Flags.BOOLEAN_SERIALIZER); + Flag byDefaultTrue = new Flag<>(id, true, source, Flags.BOOLEAN_SERIALIZER); assertFalse(featureFlag.value()); assertTrue(byDefaultTrue.value()); - writeFlag(id.toString(), "True\n"); + writeFlag(id.toString(), "true\n"); assertTrue(featureFlag.value()); assertTrue(byDefaultTrue.value()); - writeFlag(id.toString(), "False\n"); + writeFlag(id.toString(), "false\n"); assertFalse(featureFlag.value()); assertFalse(byDefaultTrue.value()); @@ -41,35 +43,48 @@ public class FileFlagSourceTest { @Test public void testIntegerLikeFlags() throws IOException { - StringFlag stringFlag = new StringFlag(id, "default", source); - IntFlag intFlag = new IntFlag(id, -1, source); - LongFlag longFlag = new LongFlag(id, -2L, source); + Flag intFlag = new Flag<>(id, -1, source, Flags.INT_SERIALIZER); + Flag longFlag = new Flag<>(id, -2L, source, Flags.LONG_SERIALIZER); - assertFalse(source.getString(id).isPresent()); - assertEquals("default", stringFlag.value()); - assertEquals(-1, intFlag.value()); - assertEquals(-2L, longFlag.value()); + assertFalse(fetch().isPresent()); + assertFalse(fetch().isPresent()); + assertEquals(-1, (int) intFlag.value()); + assertEquals(-2L, (long) longFlag.value()); writeFlag(id.toString(), "1\n"); - assertTrue(source.getString(id).isPresent()); + assertTrue(fetch().isPresent()); + assertTrue(fetch().isPresent()); + assertEquals(1, (int) intFlag.value()); + assertEquals(1L, (long) longFlag.value()); + } + + @Test + public void testStringFlag() throws IOException { + Flag stringFlag = new Flag<>(id, "default", source, Flags.STRING_SERIALIZER); + assertFalse(fetch().isPresent()); + assertEquals("default", stringFlag.value()); + + writeFlag(id.toString(), "\"1\\n\"\n"); assertEquals("1\n", stringFlag.value()); - assertEquals(1, intFlag.value()); - assertEquals(1L, longFlag.value()); } @Test public void parseFailure() throws IOException { - FeatureFlag featureFlag = new FeatureFlag(id, false, source); + Flag featureFlag = new Flag<>(id, false, source, Flags.BOOLEAN_SERIALIZER); writeFlag(featureFlag.id().toString(), "garbage"); try { featureFlag.value(); - } catch (IllegalArgumentException e) { + } catch (UncheckedIOException e) { assertThat(e.getMessage(), containsString("garbage")); } } + private Optional fetch() { + return source.fetch(id, new FetchVector()); + } + private void writeFlag(String flagId, String value) throws IOException { Path featurePath = fileSystem.getPath(FileFlagSource.FLAGS_DIRECTORY).resolve(flagId); Files.createDirectories(featurePath.getParent()); diff --git a/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java b/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java new file mode 100644 index 00000000000..4f7d797e07d --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java @@ -0,0 +1,131 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.node.BooleanNode; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.util.Objects; +import java.util.Optional; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.emptyOrNullString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class FlagsTest { + @Test + public void testBoolean() { + final boolean defaultValue = false; + FlagSource source = mock(FlagSource.class); + Flag booleanFlag = Flags.defineBoolean("id", defaultValue, "description", + "modification effect", FetchVector.Dimension.ZONE_ID, FetchVector.Dimension.HOSTNAME) + .with(FetchVector.Dimension.ZONE_ID, "a-zone") + .bindTo(source); + assertThat(booleanFlag.id().toString(), equalTo("id")); + + when(source.fetch(eq(new FlagId("id")), any())).thenReturn(Optional.empty()); + // default value without raw flag + assertThat(booleanFlag.value(), equalTo(defaultValue)); + + ArgumentCaptor vector = ArgumentCaptor.forClass(FetchVector.class); + verify(source).fetch(any(), vector.capture()); + // hostname is set by default + assertThat(vector.getValue().getValue(FetchVector.Dimension.HOSTNAME).isPresent(), is(true)); + assertThat(vector.getValue().getValue(FetchVector.Dimension.HOSTNAME).get(), is(not(emptyOrNullString()))); + // zone is set because it was set on the unbound flag above + assertThat(vector.getValue().getValue(FetchVector.Dimension.ZONE_ID), is(Optional.of("a-zone"))); + // application is not set + assertThat(vector.getValue().getValue(FetchVector.Dimension.APPLICATION_ID), is(Optional.empty())); + + RawFlag rawFlag = mock(RawFlag.class); + when(source.fetch(eq(new FlagId("id")), any())).thenReturn(Optional.of(rawFlag)); + when(rawFlag.asJsonNode()).thenReturn(BooleanNode.getTrue()); + + // raw flag deserializes to true + assertThat(booleanFlag.with(FetchVector.Dimension.APPLICATION_ID, "an-app").value(), equalTo(true)); + + verify(source, times(2)).fetch(any(), vector.capture()); + // application was set on the (bound) flag. + assertThat(vector.getValue().getValue(FetchVector.Dimension.APPLICATION_ID), is(Optional.of("an-app"))); + } + + @Test + public void testString() { + testGeneric(Flags.defineString("string-id", "default value", "description", + "modification effect", FetchVector.Dimension.ZONE_ID, FetchVector.Dimension.HOSTNAME), + "default value", "other value"); + } + + @Test + public void testInt() { + testGeneric(Flags.defineInt("int-id", 2, "desc", "mod"), 2, 3); + } + + @Test + public void testLong() { + testGeneric(Flags.defineLong("long-id", 1L, "desc", "mod"), 1L, 2L); + } + + @Test + public void testJacksonClass() { + ExampleJacksonClass defaultInstance = new ExampleJacksonClass(); + ExampleJacksonClass instance = new ExampleJacksonClass(); + instance.integer = -2; + instance.string = "foo"; + + testGeneric(Flags.defineJackson("jackson-id", ExampleJacksonClass.class, defaultInstance, + "description", "modification effect", FetchVector.Dimension.HOSTNAME), + defaultInstance, instance); + } + + private void testGeneric(UnboundFlag unboundFlag, T defaultValue, T value) { + FlagSource source = mock(FlagSource.class); + Flag flag = unboundFlag.bindTo(source); + + when(source.fetch(any(), any())).thenReturn(Optional.empty()); + assertThat(flag.value(), equalTo(defaultValue)); + + when(source.fetch(any(), any())).thenReturn(Optional.of(JsonNodeRawFlag.fromJacksonClass(value))); + assertThat(flag.value(), equalTo(value)); + + assertTrue(Flags.getFlag(unboundFlag.id()).isPresent()); + } + + + @JsonIgnoreProperties(ignoreUnknown = true) + private static class ExampleJacksonClass { + @JsonProperty("integer") + public int integer = 1; + + @JsonProperty("string") + public String string = "2"; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExampleJacksonClass that = (ExampleJacksonClass) o; + return integer == that.integer && + Objects.equals(string, that.string); + } + + @Override + public int hashCode() { + return Objects.hash(integer, string); + } + } +} diff --git a/flags/src/test/java/com/yahoo/vespa/flags/JacksonFlagTest.java b/flags/src/test/java/com/yahoo/vespa/flags/JacksonFlagTest.java deleted file mode 100644 index 8bd486f5b47..00000000000 --- a/flags/src/test/java/com/yahoo/vespa/flags/JacksonFlagTest.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.junit.Test; - -import java.util.Objects; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class JacksonFlagTest { - private final FlagId id = new FlagId("id"); - private final ExampleJacksonClass defaultValue = new ExampleJacksonClass(); - private final FlagSource source = mock(FlagSource.class); - private final JacksonFlag jacksonFlag = new JacksonFlag<>(id.toString(), ExampleJacksonClass.class, defaultValue, source); - - @Test - public void unsetThenSet() { - when(source.getString(id)).thenReturn(Optional.empty()); - ExampleJacksonClass value = jacksonFlag.value(); - assertEquals(1, value.integer); - assertEquals("2", value.string); - assertEquals("3", value.dummy); - - when(source.getString(id)).thenReturn(Optional.of("{\"integer\": 4, \"string\": \"foo\", \"stray\": 6}")); - value = jacksonFlag.value(); - assertEquals(4, value.integer); - assertEquals("foo", value.string); - assertEquals("3", value.dummy); - - assertEquals(4, value.integer); - assertEquals("foo", value.string); - assertEquals("3", value.dummy); - } - - @JsonIgnoreProperties(ignoreUnknown = true) - private static class ExampleJacksonClass { - @JsonProperty("integer") - public int integer = 1; - - @JsonProperty("string") - public String string = "2"; - - @JsonProperty("dummy") - public String dummy = "3"; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ExampleJacksonClass that = (ExampleJacksonClass) o; - return integer == that.integer && - Objects.equals(string, that.string) && - Objects.equals(dummy, that.dummy); - } - - @Override - public int hashCode() { - return Objects.hash(integer, string, dummy); - } - } -} \ No newline at end of file diff --git a/flags/src/test/java/com/yahoo/vespa/flags/OrderedFlagSourceTest.java b/flags/src/test/java/com/yahoo/vespa/flags/OrderedFlagSourceTest.java new file mode 100644 index 00000000000..5465f89e2eb --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/OrderedFlagSourceTest.java @@ -0,0 +1,50 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class OrderedFlagSourceTest { + @Test + public void test() { + FlagSource source1 = mock(FlagSource.class); + FlagSource source2 = mock(FlagSource.class); + OrderedFlagSource orderedSource = new OrderedFlagSource(source1, source2); + + FlagId id = new FlagId("id"); + FetchVector vector = new FetchVector(); + + when(source1.fetch(any(), any())).thenReturn(Optional.empty()); + when(source2.fetch(any(), any())).thenReturn(Optional.empty()); + assertFalse(orderedSource.fetch(id, vector).isPresent()); + verify(source1, times(1)).fetch(any(), any()); + verify(source2, times(1)).fetch(any(), any()); + + RawFlag rawFlag = mock(RawFlag.class); + + when(source1.fetch(any(), any())).thenReturn(Optional.empty()); + when(source2.fetch(any(), any())).thenReturn(Optional.of(rawFlag)); + assertEquals(orderedSource.fetch(id, vector), Optional.of(rawFlag)); + verify(source1, times(2)).fetch(any(), any()); + verify(source2, times(2)).fetch(any(), any()); + + when(source1.fetch(any(), any())).thenReturn(Optional.of(rawFlag)); + when(source2.fetch(any(), any())).thenReturn(Optional.empty()); + assertEquals(orderedSource.fetch(id, vector), Optional.of(rawFlag)); + verify(source1, times(3)).fetch(any(), any()); + // Not invoked as source1 provided raw flag + verify(source2, times(2)).fetch(any(), any()); + } +} \ No newline at end of file diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/ConditionTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/ConditionTest.java new file mode 100644 index 00000000000..96cbce71fa8 --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/ConditionTest.java @@ -0,0 +1,38 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json; + +import com.yahoo.vespa.flags.FetchVector; +import org.junit.Test; + +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class ConditionTest { + @Test + public void testWhitelist() { + String hostname1 = "host1"; + Condition condition = new Condition(Condition.Type.WHITELIST, FetchVector.Dimension.HOSTNAME, + Stream.of(hostname1).collect(Collectors.toSet())); + assertFalse(condition.test(new FetchVector())); + assertFalse(condition.test(new FetchVector().with(FetchVector.Dimension.APPLICATION_ID, "foo"))); + assertFalse(condition.test(new FetchVector().with(FetchVector.Dimension.HOSTNAME, "bar"))); + assertTrue(condition.test(new FetchVector().with(FetchVector.Dimension.HOSTNAME, hostname1))); + } + + @Test + public void testBlacklist() { + String hostname1 = "host1"; + Condition condition = new Condition(Condition.Type.BLACKLIST, FetchVector.Dimension.HOSTNAME, + Stream.of(hostname1).collect(Collectors.toSet())); + assertTrue(condition.test(new FetchVector())); + assertTrue(condition.test(new FetchVector().with(FetchVector.Dimension.APPLICATION_ID, "foo"))); + assertTrue(condition.test(new FetchVector().with(FetchVector.Dimension.HOSTNAME, "bar"))); + assertFalse(condition.test(new FetchVector().with(FetchVector.Dimension.HOSTNAME, hostname1))); + } +} diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java new file mode 100644 index 00000000000..2eb12e53ddc --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java @@ -0,0 +1,82 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json; + +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.RawFlag; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class FlagDataTest { + private final String json = "{\n" + + " \"rules\": [\n" + + " {\n" + + " \"conditions\": [\n" + + " {\n" + + " \"type\": \"whitelist\",\n" + + " \"dimension\": \"hostname\",\n" + + " \"values\": [ \"host1\", \"host2\" ]\n" + + " },\n" + + " {\n" + + " \"type\": \"blacklist\",\n" + + " \"dimension\": \"application\",\n" + + " \"values\": [ \"app1\", \"app2\" ]\n" + + " }\n" + + " ],\n" + + " \"value\": true\n" + + " },\n" + + " {\n" + + " \"conditions\": [\n" + + " {\n" + + " \"type\": \"whitelist\",\n" + + " \"dimension\": \"zone\",\n" + + " \"values\": [ \"zone1\", \"zone2\" ]\n" + + " }\n" + + " ],\n" + + " \"value\": false\n" + + " }\n" + + " ],\n" + + " \"attributes\": {\n" + + " \"zone\": \"zone1\"\n" + + " }\n" + + "}"; + + private final FetchVector vector = new FetchVector(); + + @Test + public void test() { + // Second rule matches with the default zone matching + verify(Optional.of("false"), vector); + + // First rule matches only if both conditions match + verify(Optional.of("false"), vector + .with(FetchVector.Dimension.HOSTNAME, "host1") + .with(FetchVector.Dimension.APPLICATION_ID, "app2")); + verify(Optional.of("true"), vector + .with(FetchVector.Dimension.HOSTNAME, "host1") + .with(FetchVector.Dimension.APPLICATION_ID, "app3")); + + // No rules apply if zone is overridden to an unknown zone + verify(Optional.empty(), vector.with(FetchVector.Dimension.ZONE_ID, "unknown zone")); + } + + private void verify(Optional expectedValue, FetchVector vector) { + FlagData data = FlagData.deserialize(json); + Optional rawFlag = data.resolve(vector); + + if (expectedValue.isPresent()) { + assertTrue(rawFlag.isPresent()); + assertEquals(expectedValue.get(), rawFlag.get().asJson()); + } else { + assertFalse(rawFlag.isPresent()); + } + + } +} \ No newline at end of file diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/SerializationTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/SerializationTest.java new file mode 100644 index 00000000000..f3f8c147212 --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/SerializationTest.java @@ -0,0 +1,130 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeType; +import com.yahoo.vespa.flags.json.wire.WireCondition; +import com.yahoo.vespa.flags.json.wire.WireFlagData; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +/** + * @author hakonhall + */ +public class SerializationTest { + @Test + public void emptyJson() throws IOException { + String json = "{}"; + WireFlagData wireData = WireFlagData.deserialize(json); + assertThat(wireData.defaultFetchVector, nullValue()); + assertThat(wireData.rules, nullValue()); + assertThat(wireData.serializeToJson(), equalTo(json)); + + assertThat(FlagData.deserialize(json).serializeToJson(), equalTo("{}")); + } + + @Test + public void deserialization() throws IOException { + String json = "{\n" + + " \"rules\": [\n" + + " {\n" + + " \"conditions\": [\n" + + " {\n" + + " \"type\": \"whitelist\",\n" + + " \"dimension\": \"application\",\n" + + " \"values\": [ \"a1\", \"a2\" ]\n" + + " },\n" + + " {\n" + + " \"type\": \"blacklist\",\n" + + " \"dimension\": \"hostname\",\n" + + " \"values\": [ \"h1\" ]\n" + + " }\n" + + " ],\n" + + " \"value\": true\n" + + " }\n" + + " ],\n" + + " \"attributes\": {\n" + + " \"zone\": \"z1\",\n" + + " \"application\": \"a1\",\n" + + " \"hostname\": \"h1\"\n" + + " }\n" + + "}"; + + WireFlagData wireData = WireFlagData.deserialize(json); + + // rule + assertThat(wireData.rules.size(), equalTo(1)); + assertThat(wireData.rules.get(0).andConditions.size(), equalTo(2)); + assertThat(wireData.rules.get(0).value.getNodeType(), equalTo(JsonNodeType.BOOLEAN)); + assertThat(wireData.rules.get(0).value.asBoolean(), equalTo(true)); + // first condition + WireCondition whitelistCondition = wireData.rules.get(0).andConditions.get(0); + assertThat(whitelistCondition.type, equalTo("whitelist")); + assertThat(whitelistCondition.dimension, equalTo("application")); + assertThat(whitelistCondition.values, equalTo(new HashSet<>(Arrays.asList("a1", "a2")))); + // second condition + WireCondition blacklistCondition = wireData.rules.get(0).andConditions.get(1); + assertThat(blacklistCondition.type, equalTo("blacklist")); + assertThat(blacklistCondition.dimension, equalTo("hostname")); + assertThat(blacklistCondition.values, equalTo(new HashSet<>(Arrays.asList("h1")))); + + // attributes + assertThat(wireData.defaultFetchVector, notNullValue()); + assertThat(wireData.defaultFetchVector.get("zone"), equalTo("z1")); + assertThat(wireData.defaultFetchVector.get("application"), equalTo("a1")); + assertThat(wireData.defaultFetchVector.get("hostname"), equalTo("h1")); + + // Verify serialization of RawFlag == serialization by ObjectMapper + ObjectMapper mapper = new ObjectMapper(); + String serializedWithObjectMapper = mapper.writeValueAsString(mapper.readTree(json)); + assertThat(wireData.serializeToJson(), equalTo(serializedWithObjectMapper)); + + // Unfortunately the order of attributes members are different... + // assertThat(FlagData.deserialize(json).serializeToJson(), equalTo(serializedWithObjectMapper)); + } + + @Test + public void jsonWithStrayFields() { + String json = "{\n" + + " \"foo\": true,\n" + + " \"rules\": [\n" + + " {\n" + + " \"conditions\": [\n" + + " {\n" + + " \"type\": \"whitelist\",\n" + + " \"dimension\": \"zone\",\n" + + " \"bar\": \"zoo\"\n" + + " }\n" + + " ],\n" + + " \"other\": true\n" + + " }\n" + + " ],\n" + + " \"attributes\": {\n" + + " }\n" + + "}"; + + WireFlagData wireData = WireFlagData.deserialize(json); + + assertThat(wireData.rules.size(), equalTo(1)); + assertThat(wireData.rules.get(0).andConditions.size(), equalTo(1)); + WireCondition whitelistCondition = wireData.rules.get(0).andConditions.get(0); + assertThat(whitelistCondition.type, equalTo("whitelist")); + assertThat(whitelistCondition.dimension, equalTo("zone")); + assertThat(whitelistCondition.values, nullValue()); + assertThat(wireData.rules.get(0).value, nullValue()); + assertThat(wireData.defaultFetchVector, anEmptyMap()); + + assertThat(wireData.serializeToJson(), equalTo("{\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"zone\"}]}],\"attributes\":{}}")); + + assertThat(FlagData.deserialize(json).serializeToJson(), equalTo("{\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"zone\"}]}]}")); + } +} diff --git a/parent/pom.xml b/parent/pom.xml index bb74e6a5f24..168e5ec8d7e 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -104,15 +104,15 @@ true true false - - -Xlint:all - -Xlint:-serial - -Xlint:-try - -Xlint:-processing - -Xlint:-varargs + + -Xlint:all + -Xlint:-serial + -Xlint:-try + -Xlint:-processing + -Xlint:-varargs -Xlint:-options - -Werror - + -Werror + @@ -404,11 +404,11 @@ commons-exec 1.3 - - org.apache.velocity - velocity - 1.7 - + + org.apache.velocity + velocity + 1.7 + io.airlift airline @@ -723,7 +723,7 @@ UTF-8 UTF-8 true - all + all 2.21.0 diff --git a/pom.xml b/pom.xml index 68b0f1e2f9a..c3ad8fe9320 100644 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,7 @@ config-provisioning config-proxy configserver + configserver-flags config_test container container-core diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index e0e2b64bdae..57c206ee570 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -10,8 +10,8 @@ import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.flags.FeatureFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.InfraApplicationApi; @@ -59,12 +59,8 @@ public class DuperModelManager implements DuperModelInfraApi { @Inject public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) { this( - // Whether to include activate infrastructure applications (except from controller/config apps - see below). - new FeatureFlag("dupermodel-contains-infra", true, flagSource).value(), - // For historical reasons, the ApplicationInfo in the DuperModel for controllers and config servers - // is based on the ConfigserverConfig (this flag is true). We want to transition to use the - // infrastructure application activated by the InfrastructureProvisioner once that supports health. - new FeatureFlag("dupermodel-use-configserverconfig", true, flagSource).value(), + Flags.DUPERMODEL_CONTAINS_INFRA.bindTo(flagSource).value(), + Flags.DUPERMODEL_USE_CONFIGSERVERCONFIG.bindTo(flagSource).value(), configServerConfig.multitenant(), configServerApplication.makeApplicationInfoFromConfig(configServerConfig), superModelProvider, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index 2ad37faf593..340d30e9f41 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -8,8 +8,9 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.flags.FeatureFlag; -import com.yahoo.vespa.flags.FileFlagSource; +import com.yahoo.vespa.flags.Flag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.RunletExecutorImpl; @@ -49,23 +50,23 @@ public class HealthMonitorManager implements MonitorManager { private final ConcurrentHashMap healthMonitors = new ConcurrentHashMap<>(); private final DuperModelManager duperModel; private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory; - private final FeatureFlag monitorInfra; + private final Flag monitorInfra; @Inject - public HealthMonitorManager(DuperModelManager duperModel, FileFlagSource flagSource) { + public HealthMonitorManager(DuperModelManager duperModel, FlagSource flagSource) { this(duperModel, - new FeatureFlag("healthmonitor-monitorinfra", true, flagSource), + Flags.HEALTHMONITOR_MONITOR_INFRA.bindTo(flagSource), new StateV1HealthModel(TARGET_HEALTH_STALENESS, HEALTH_REQUEST_TIMEOUT, KEEP_ALIVE, new RunletExecutorImpl(THREAD_POOL_SIZE))); } private HealthMonitorManager(DuperModelManager duperModel, - FeatureFlag monitorInfra, + Flag monitorInfra, StateV1HealthModel healthModel) { this(duperModel, monitorInfra, id -> new ApplicationHealthMonitor(id, healthModel)); } HealthMonitorManager(DuperModelManager duperModel, - FeatureFlag monitorInfra, + Flag monitorInfra, ApplicationHealthMonitorFactory applicationHealthMonitorFactory) { this.duperModel = duperModel; this.monitorInfra = monitorInfra; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index 86b0ee4a8f3..39147955d4c 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.flags.FeatureFlag; +import com.yahoo.vespa.flags.Flag; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.duper.ControllerHostApplication; import com.yahoo.vespa.service.duper.DuperModelManager; @@ -33,7 +33,8 @@ import static org.mockito.Mockito.when; public class HealthMonitorManagerTest { private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); private final DuperModelManager duperModel = mock(DuperModelManager.class); - private final FeatureFlag monitorInfra = mock(FeatureFlag.class); + @SuppressWarnings("unchecked") + private final Flag monitorInfra = (Flag) mock(Flag.class); private final ApplicationHealthMonitor monitor = mock(ApplicationHealthMonitor.class); private final ApplicationHealthMonitorFactory monitorFactory = mock(ApplicationHealthMonitorFactory.class); private final HealthMonitorManager manager = new HealthMonitorManager(duperModel, monitorInfra, monitorFactory); -- cgit v1.2.3