Read override file flags from file only once for config server

Up until now, every lookup of a flag in the ConfigServerFlagSource would
 1. try to read 2 flag files under /etc/vespa/flags, causing exceptions
    because they are typically not set, and
 2. then read flag from ZooKeeper through ZooKeeperFlagSource

Optimization was deliberately held off until later (now). This PR fixes (1).

Changes the ConfigServerFlagSource to:
 1'. Read VESPA_HOME/var/vespa/flag.db once during component graph
     construction.  As before, if a flag is defined on file, the flag is not
     looked up in ZK, which may be useful in emergencies.
 2.  As before.

Also, removes the last usages of FileFlagSource and its reading of flags in
/etc/vespa/flags.

4 files changed, 157 insertions, 2 deletions

diff --git a/configserver-flags/pom.xml b/configserver-flags/pom.xml
index a55a8db5f56..8c96512c4c0 100644
--- a/configserver-flags/pom.xml
+++ b/configserver-flags/pom.xml
@@ -63,6 +63,12 @@
         <!-- test -->
         <dependency>
             <groupId>com.yahoo.vespa</groupId>
+            <artifactId>defaults</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
             <artifactId>testutil</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java
index d452d8a4aad..b1ffc05e70c 100644
--- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java
+++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java
@@ -2,16 +2,23 @@
 package com.yahoo.vespa.configserver.flags;
 
 import com.google.inject.Inject;
+import com.yahoo.vespa.configserver.flags.db.BootstrapFlagSource;
 import com.yahoo.vespa.configserver.flags.db.ZooKeeperFlagSource;
-import com.yahoo.vespa.flags.FileFlagSource;
 import com.yahoo.vespa.flags.OrderedFlagSource;
 
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+
 /**
  * @author hakonhall
  */
 public class ConfigServerFlagSource extends OrderedFlagSource {
     @Inject
     public ConfigServerFlagSource(FlagsDb flagsDb) {
-        super(new FileFlagSource(), new ZooKeeperFlagSource(flagsDb));
+        this(FileSystems.getDefault(), flagsDb);
+    }
+
+    ConfigServerFlagSource(FileSystem fileSystem, FlagsDb flagsDb) {
+        super(new BootstrapFlagSource(fileSystem), new ZooKeeperFlagSource(flagsDb));
     }
 }
diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/BootstrapFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/BootstrapFlagSource.java
new file mode 100644
index 00000000000..0c467a0e7c8
--- /dev/null
+++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/BootstrapFlagSource.java
@@ -0,0 +1,36 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configserver.flags.db;
+
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.RawFlag;
+import com.yahoo.vespa.flags.file.FlagDbFile;
+import com.yahoo.vespa.flags.json.FlagData;
+
+import java.nio.file.FileSystem;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A flag source that can be used in emergencies, see test for path and format of file.
+ *
+ * <p>Use file if the ZooKeeper flag makes it difficult to start the config server, would make irreparable damage,
+ * or is impossible to change through ZooKeeper due to a bug. This flag source should take precedence over ZooKeeper
+ * whenever it has specified the value for a flag.
+ *
+ * @author hakonhall
+ */
+public class BootstrapFlagSource implements FlagSource {
+    private final Map<FlagId, FlagData> flagData;
+
+    public BootstrapFlagSource(FileSystem fileSystem) {
+        // The flags on disk is read once now and never again.
+        this.flagData = new FlagDbFile(fileSystem).read();
+    }
+
+    @Override
+    public Optional<RawFlag> fetch(FlagId id, FetchVector vector) {
+        return Optional.ofNullable(flagData.get(id)).flatMap(data -> data.resolve(vector));
+    }
+}
diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSourceTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSourceTest.java
new file mode 100644
index 00000000000..d0d1d61628c
--- /dev/null
+++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSourceTest.java
@@ -0,0 +1,106 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configserver.flags;
+
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.RawFlag;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakonhall
+ */
+public class ConfigServerFlagSourceTest {
+    private final String vespaHome = Defaults.getDefaults().vespaHome();
+    private final FileSystem fileSystem = TestFileSystem.create();
+    private final FlagsDb flagsDb = mock(FlagsDb.class);
+    private final FlagId flagId = new FlagId("fid");
+
+    private Flags.Replacer flagsReplacer;
+    private ConfigServerFlagSource flagSource;
+    private BooleanFlag flag;
+
+    @Before
+    public void setUp() {
+        flagsReplacer = Flags.clearFlagsForTesting();
+    }
+
+    @After
+    public void tearDown() {
+        flagsReplacer.close();
+    }
+
+    private void initialize() {
+        flagSource = new ConfigServerFlagSource(fileSystem, flagsDb);
+        flag = Flags.defineFeatureFlag(flagId.toString(), false, "", "").bindTo(flagSource);
+    }
+
+    @Test
+    public void testAbsentInFileSystemForwardsToFlagsDb() {
+        initialize();
+
+        when(flagsDb.getValue(flagId)).thenReturn(Optional.empty());
+        Optional<RawFlag> rawFlag = flagSource.fetch(flagId, new FetchVector());
+        assertFalse(rawFlag.isPresent());
+        verify(flagsDb, times(1)).getValue(flagId);
+    }
+
+    @Test
+    public void testAvoidingZooKeeperWhenOverridingInFile() throws IOException {
+        // Here is how to set the value of a flag, such that ZooKeeper will NOT be queried when getting that value:
+        //  - Make a flag.db file with the override
+        Path flagPath = fileSystem.getPath(vespaHome + "/var/vespa/flag.db");
+        String json = "{\n" +
+                "    \"flags\": [\n" +
+                "        {\n" +
+                "            \"id\" : \"fid\",\n" +
+                "            \"rules\" : [\n" +
+                "                {\n" +
+                "                    \"value\" : true\n" +
+                "                }\n" +
+                "            ]\n" +
+                "        }\n" +
+                "    ]\n" +
+                "}";
+        Files.createDirectories(flagPath.getParent());
+        Files.write(flagPath, json.getBytes(StandardCharsets.UTF_8));
+
+        // Alright, verify we have accomplished overriding that flag and avoid flagDb/ZooKeeper
+
+        initialize();
+
+        Optional<RawFlag> rawFlag = flagSource.fetch(flagId, new FetchVector());
+        assertTrue(rawFlag.isPresent());
+        assertTrue(flag.value());
+
+        verify(flagsDb, times(0)).getValue(any());
+
+        // Other flags DO hit flagDb/ZooKeeper
+
+        FlagId flagId2 = new FlagId("fooId");
+        when(flagsDb.getValue(flagId2)).thenReturn(Optional.empty());
+        Optional<RawFlag> rawFlag2 = flagSource.fetch(flagId2, new FetchVector());
+        assertFalse(rawFlag2.isPresent());
+        verify(flagsDb, times(1)).getValue(flagId2);
+    }
+}
\ No newline at end of file