Configserver flags REST API

Adds a new ZooKeeper backed flag source. It is defined in a new module
configserver-flags to allow as many as possible config server modules to depend
on it by minimizing dependencies.

The content of the ZK backed flag source can be viewed and modified through
REST API on the config server/controller. The data stored per flag looks like

  {
    "rules": [
      {
        "conditions": [
          { "type": "whitelist", "dimension": "hostname", "values": ["host1"] }
        ],
        "value": true
      }
    ]
  }

typical for enabling a feature flag on host1.

2 types of conditions are so far supported: whitelist and blacklist. All the
conditions must match in order for the value to apply. If the value is null (or
absent), the default value will be used. At the time the flag's value is
retrieved, it is resolved against the conditions with the current zone,
hostname, and/or application.

The same data structure is used for FileFlagSource for files in
/etc/vespa/flags with the ".2" extension.

The FlagSource component injected in the config server is changed to:
 1. Return the flag value if specified in /etc/vespa/flags, or otherwise
 2. return flag value from ZooKeeper (same as REST API)

The current flags (module) is also changed:
 - All flags must be defined in com.yahoo.vespa.flags.Flags. This allows the ZK
   backed flag source additional sanity checking when modifying flags.
 - If it makes sense to have different flag value depending on e.g. the
   application, then at some point before the value is retrieved, one has to
   bind the flag to that application (using with() to set up the fetch vector).

Future changes would be to 0. make a merged FlagSource in host admin, 1. add
support for viewing and modifying feature flags in dashboard, 2. in hv tool.

10 files changed, 293 insertions, 0 deletions

diff --git a/configserver-flags/CMakeLists.txt b/configserver-flags/CMakeLists.txt
new file mode 100644
index 00000000000..75deaf42d9b
--- /dev/null
+++ b/configserver-flags/CMakeLists.txt
@@ -0,0 +1,2 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+install_fat_java_artifact(configserver-flags)
diff --git a/configserver-flags/OWNERS b/configserver-flags/OWNERS
new file mode 100644
index 00000000000..e131dacde49
--- /dev/null
+++ b/configserver-flags/OWNERS
@@ -0,0 +1 @@
+hakonhall
diff --git a/configserver-flags/README.md b/configserver-flags/README.md
new file mode 100644
index 00000000000..e23452a1f0c
--- /dev/null
+++ b/configserver-flags/README.md
@@ -0,0 +1,3 @@
+<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+# Config Server Flags
+Manages flags backed by the Config Server's ZooKeeper.
diff --git a/configserver-flags/pom.xml b/configserver-flags/pom.xml
new file mode 100644
index 00000000000..c6fdbe89b95
--- /dev/null
+++ b/configserver-flags/pom.xml
@@ -0,0 +1,100 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                             http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>6-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>configserver-flags</artifactId>
+    <version>6-SNAPSHOT</version>
+    <packaging>container-plugin</packaging>
+    <name>${project.artifactId}</name>
+    <description>Config Server Flags.</description>
+
+    <dependencies>
+        <!-- provided -->
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>zkfacade</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>flags</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>annotations</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespajlib</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>yolean</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+            <scope>provided</scope>
+            <classifier>no_aop</classifier>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- test -->
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>testutil</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>com.yahoo.vespa</groupId>
+                <artifactId>bundle-plugin</artifactId>
+                <extensions>true</extensions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java
new file mode 100644
index 00000000000..9768c42b477
--- /dev/null
+++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java
@@ -0,0 +1,18 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configserver.flags;
+
+import com.google.inject.Inject;
+import com.yahoo.vespa.configserver.flags.db.FlagsDbImpl;
+import com.yahoo.vespa.configserver.flags.db.ZooKeeperFlagSource;
+import com.yahoo.vespa.flags.FileFlagSource;
+import com.yahoo.vespa.flags.OrderedFlagSource;
+
+/**
+ * @author hakonhall
+ */
+public class ConfigServerFlagSource extends OrderedFlagSource {
+    @Inject
+    public ConfigServerFlagSource(FlagsDbImpl flagsDb) {
+        super(new FileFlagSource(), new ZooKeeperFlagSource(flagsDb));
+    }
+}
diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/FlagsDb.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/FlagsDb.java
new file mode 100644
index 00000000000..2c29ae0b818
--- /dev/null
+++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/FlagsDb.java
@@ -0,0 +1,25 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configserver.flags;
+
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author hakonhall
+ */
+public interface FlagsDb {
+    /** Get the String value of the flag. */
+    Optional<FlagData> getValue(FlagId flagId);
+
+    /** Set the String value of the flag. */
+    void setValue(FlagId flagId, FlagData data);
+
+    /** Remove the flag value if it exists. */
+    void removeValue(FlagId flagId);
+
+    /** Get all flags that have been set. */
+    Map<FlagId, FlagData> getAllFlags();
+}
diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java
new file mode 100644
index 00000000000..7b0a2f632cc
--- /dev/null
+++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java
@@ -0,0 +1,56 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configserver.flags.db;
+
+import com.google.inject.Inject;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.configserver.flags.FlagsDb;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author hakonhall
+ */
+public class FlagsDbImpl implements FlagsDb {
+    private static final Path ROOT_PATH = Path.fromString("/flags/v1");
+
+    private final Curator curator;
+
+    @Inject
+    public FlagsDbImpl(Curator curator) {
+        this.curator = curator;
+    }
+
+    @Override
+    public Optional<FlagData> getValue(FlagId flagId) {
+        return curator.getData(getZkPathFor(flagId)).map(FlagData::deserializeUtf8Json);
+    }
+
+    @Override
+    public void setValue(FlagId flagId, FlagData data) {
+        curator.set(getZkPathFor(flagId), data.serializeToUtf8Json());
+    }
+
+    @Override
+    public Map<FlagId, FlagData> getAllFlags() {
+        Map<FlagId, FlagData> flags = new HashMap<>();
+        for (String flagId : curator.getChildren(ROOT_PATH)) {
+            FlagId id = new FlagId(flagId);
+            getValue(id).ifPresent(data -> flags.put(id, data));
+        }
+        return flags;
+    }
+
+    @Override
+    public void removeValue(FlagId flagId) {
+        curator.delete(getZkPathFor(flagId));
+    }
+
+    private static Path getZkPathFor(FlagId flagId) {
+        return ROOT_PATH.append(flagId.toString());
+    }
+}
diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java
new file mode 100644
index 00000000000..bd99ac6eca9
--- /dev/null
+++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java
@@ -0,0 +1,25 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configserver.flags.db;
+
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.RawFlag;
+
+import java.util.Optional;
+
+/**
+ * @author hakonhall
+ */
+public class ZooKeeperFlagSource implements FlagSource {
+    private final FlagsDbImpl flagsDb;
+
+    public ZooKeeperFlagSource(FlagsDbImpl flagsDb) {
+        this.flagsDb = flagsDb;
+    }
+
+    @Override
+    public Optional<RawFlag> fetch(FlagId id, FetchVector vector) {
+        return flagsDb.getValue(id).flatMap(data -> data.resolve(vector));
+    }
+}
diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/package-info.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/package-info.java
new file mode 100644
index 00000000000..97e66d95715
--- /dev/null
+++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.configserver.flags;
+
+import com.yahoo.osgi.annotation.ExportPackage;
+
+/** The node repository controls and allocates the nodes available in a hosted Vespa zone */
diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java
new file mode 100644
index 00000000000..1fe61130348
--- /dev/null
+++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java
@@ -0,0 +1,56 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configserver.flags.db;
+
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.JsonNodeRawFlag;
+import com.yahoo.vespa.flags.json.Condition;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.flags.json.Rule;
+import org.junit.Test;
+
+import java.util.Map;
+import java.util.Optional;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author hakonhall
+ */
+public class FlagsDbImplTest {
+    @Test
+    public void test() {
+        MockCurator curator = new MockCurator();
+        FlagsDbImpl db = new FlagsDbImpl(curator);
+
+        Condition condition1 = new Condition(Condition.Type.WHITELIST, FetchVector.Dimension.HOSTNAME, "host1");
+        Rule rule1 = new Rule(Optional.of(JsonNodeRawFlag.fromJson("13")), condition1);
+        FlagData data = new FlagData(new FetchVector().with(FetchVector.Dimension.ZONE_ID, "zone-a"), rule1);
+        FlagId flagId = new FlagId("id");
+        db.setValue(flagId, data);
+
+        assertTrue(db.getValue(flagId).isPresent());
+        Optional<FlagData> dataCopy = db.getValue(flagId);
+        assertTrue(dataCopy.isPresent());
+
+        assertEquals("{\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\"," +
+                "\"values\":[\"host1\"]}],\"value\":13}],\"attributes\":{\"zone\":\"zone-a\"}}",
+                dataCopy.get().serializeToJson());
+
+        FlagId flagId2 = new FlagId("id2");
+        db.setValue(flagId2, data);
+        Map<FlagId, FlagData> flags = db.getAllFlags();
+        assertThat(flags.size(), equalTo(2));
+        assertThat(flags.get(flagId), notNullValue());
+        assertThat(flags.get(flagId2), notNullValue());
+
+        db.removeValue(flagId2);
+        assertFalse(db.getValue(flagId2).isPresent());
+    }
+}
\ No newline at end of file