Store flags locally in one file

6 files changed, 156 insertions, 185 deletions

diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
index 581ec599aab..7d84efa52b2 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
@@ -17,8 +17,7 @@ import java.util.function.Consumer;
  * @author hakonhall
  */
 @Immutable
-public
-class FetchVector {
+public class FetchVector {
     public enum Dimension {
         /** Value from ZoneId::value */
         ZONE_ID,
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java
new file mode 100644
index 00000000000..abe8d407ab0
--- /dev/null
+++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java
@@ -0,0 +1,111 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.flags.file;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Java API for a flag database stored in a single file
+ *
+ * @author hakonhall
+ */
+public class FlagDbFile {
+    private static final Logger logger = Logger.getLogger(FlagDbFile.class.getName());
+
+    private final Path path;
+
+    public FlagDbFile() {
+        this(FileSystems.getDefault());
+    }
+
+    public FlagDbFile(FileSystem fileSystem) {
+        this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flag.db"));
+    }
+
+    public FlagDbFile(Path path) {
+        this.path = path;
+    }
+
+    public Path getPath() {
+        return path;
+    }
+
+    public Map<FlagId, FlagData> read() {
+        Optional<byte[]> bytes = readFile();
+        if (!bytes.isPresent()) return Collections.emptyMap();
+        return FlagData.deserializeList(bytes.get()).stream().collect(Collectors.toMap(FlagData::id, Function.identity()));
+    }
+
+    public boolean sync(Map<FlagId, FlagData> flagData) {
+        boolean modified = false;
+        Map<FlagId, FlagData> currentFlagData = read();
+        Set<FlagId> flagIdsToBeRemoved = new HashSet<>(currentFlagData.keySet());
+        List<FlagData> flagDataList = new ArrayList<>(flagData.values());
+
+        for (FlagData data : flagDataList) {
+            flagIdsToBeRemoved.remove(data.id());
+
+            FlagData existingFlagData = currentFlagData.get(data.id());
+            if (existingFlagData == null) {
+                logger.log(LogLevel.INFO, "New flag " + data.id() + ": " + data.serializeToJson());
+                modified = true;
+
+                // Could also consider testing with FlagData::equals, but that would be too fragile?
+            } else if (!Objects.equals(data.serializeToJson(), existingFlagData.serializeToJson())){
+                logger.log(LogLevel.INFO, "Updating flag " + data.id() + " from " +
+                        existingFlagData.serializeToJson() + " to " + data.serializeToJson());
+                modified = true;
+            }
+        }
+
+        if (!flagIdsToBeRemoved.isEmpty()) {
+            String flagIdsString = flagIdsToBeRemoved.stream().map(FlagId::toString).collect(Collectors.joining(", "));
+            logger.log(LogLevel.INFO, "Removing flags " + flagIdsString);
+            modified = true;
+        }
+
+        if (!modified) return false;
+
+        writeFile(FlagData.serializeListToUtf8Json(flagDataList));
+
+        return modified;
+    }
+
+    private Optional<byte[]> readFile() {
+        try {
+            return Optional.of(Files.readAllBytes(path));
+        } catch (NoSuchFileException e) {
+            return Optional.empty();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private void writeFile(byte[] bytes) {
+        uncheck(() -> Files.createDirectories(path.getParent()));
+        uncheck(() -> Files.write(path, bytes));
+    }
+}
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java
deleted file mode 100644
index 7dd61a18ad1..00000000000
--- a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java
+++ /dev/null
@@ -1,156 +0,0 @@
-// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.flags.file;
-
-import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.defaults.Defaults;
-import com.yahoo.vespa.flags.FlagId;
-import com.yahoo.vespa.flags.json.FlagData;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystem;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.NotDirectoryException;
-import java.nio.file.Path;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static com.yahoo.yolean.Exceptions.uncheck;
-
-/**
- * Java API for a local file-based flag repository.
- *
- * @author hakonhall
- */
-public class FlagDirectory {
-    private static final Logger logger = Logger.getLogger(FlagDirectory.class.getName());
-
-    private final Path flagDirectory;
-
-    public FlagDirectory() {
-        this(FileSystems.getDefault());
-    }
-
-    FlagDirectory(FileSystem fileSystem) {
-        this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flags"));
-    }
-
-    public FlagDirectory(Path flagDirectory) {
-        this.flagDirectory = flagDirectory;
-    }
-
-    public Path getPath() {
-        return flagDirectory;
-    }
-
-    public Map<FlagId, FlagData> read() {
-        return getAllRegularFilesStream()
-                .map(path -> {
-                    FlagId flagId = new FlagId(getFilenameOf(path));
-                    Optional<FlagData> flagData = readFlagData(flagId);
-                    if (!flagData.isPresent()) return null;
-                    if (!Objects.equals(flagData.get().id(), flagId)) {
-                        logger.log(LogLevel.WARNING, "Flag file " + path + " contains conflicting id " +
-                                flagData.get().id() + ", ignoring flag");
-                        return null;
-                    }
-                    return flagData.get();
-                })
-                .filter(Objects::nonNull)
-                .collect(Collectors.toMap(FlagData::id, Function.identity()));
-    }
-
-    public Optional<FlagData> readFlagData(FlagId flagId) {
-        return readUtf8File(getPathFor(flagId)).map(FlagData::deserialize);
-    }
-
-    /**
-     * Modify the flag directory as necessary, such that a later {@link #read()} will return {@code flagData}.
-     *
-     * @return true if any modifications were done.
-     */
-    public boolean sync(Map<FlagId, FlagData> flagData) {
-        boolean modified = false;
-
-        Set<Path> pathsToDelete = getAllRegularFilesStream().collect(Collectors.toCollection(HashSet::new));
-
-        uncheck(() -> Files.createDirectories(flagDirectory));
-        for (Map.Entry<FlagId, FlagData> entry : flagData.entrySet()) {
-            FlagId flagId = entry.getKey();
-            FlagData data = entry.getValue();
-            Path path = getPathFor(flagId);
-
-            pathsToDelete.remove(path);
-
-            String serializedData = data.serializeToJson();
-            Optional<String> fileContent = readUtf8File(path);
-            if (fileContent.isPresent()) {
-                if (!Objects.equals(fileContent.get(), serializedData)) {
-                    logger.log(LogLevel.INFO, "Updating flag " + flagId + " from " + fileContent.get() +
-                            " to " + serializedData);
-                    writeUtf8File(path, serializedData);
-                    modified = true;
-                }
-            } else {
-                logger.log(LogLevel.INFO, "New flag " + flagId + ": " + serializedData);
-                writeUtf8File(path, serializedData);
-                modified = true;
-            }
-        }
-
-        for (Path path : pathsToDelete) {
-            logger.log(LogLevel.INFO, "Removing flag file " + path);
-            uncheck(() -> Files.deleteIfExists(path));
-            modified = true;
-        }
-
-        return modified;
-    }
-
-    private Stream<Path> getAllRegularFilesStream() {
-        try {
-            return Files.list(flagDirectory).filter(Files::isRegularFile);
-        } catch (NotDirectoryException | NoSuchFileException e) {
-            return Stream.empty();
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    private static String getFilenameOf(Path path) {
-        return path.getName(path.getNameCount() - 1).toString();
-    }
-
-    private Path getPathFor(FlagId flagId) {
-        return flagDirectory.resolve(flagId.toString());
-    }
-
-    private Optional<String> readUtf8File(Path path) {
-        try {
-            return Optional.of(new String(Files.readAllBytes(path), StandardCharsets.UTF_8));
-        } catch (NoSuchFileException e) {
-            return Optional.empty();
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    private void writeUtf8File(Path path, String content) {
-        uncheck(() -> Files.write(path, content.getBytes(StandardCharsets.UTF_8)));
-    }
-
-    @Override
-    public String toString() {
-        return "FlagDirectory{" + flagDirectory + '}';
-    }
-}
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java
index 572ec511607..64c4bbe7616 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.flags.FlagId;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.RawFlag;
 import com.yahoo.vespa.flags.json.wire.WireFlagData;
+import com.yahoo.vespa.flags.json.wire.WireFlagDataList;
 import com.yahoo.vespa.flags.json.wire.WireRule;
 
 import javax.annotation.concurrent.Immutable;
@@ -114,6 +115,24 @@ public class FlagData {
         );
     }
 
+    public static byte[] serializeListToUtf8Json(List<FlagData> list) {
+        return listToWire(list).serializeToBytes();
+    }
+
+    public static List<FlagData> deserializeList(byte[] bytes) {
+        return listFromWire(WireFlagDataList.deserializeFrom(bytes));
+    }
+
+    public static WireFlagDataList listToWire(List<FlagData> list) {
+        WireFlagDataList wireList = new WireFlagDataList();
+        wireList.flags = list.stream().map(FlagData::toWire).collect(Collectors.toList());
+        return wireList;
+    }
+
+    public static List<FlagData> listFromWire(WireFlagDataList wireList) {
+        return wireList.flags.stream().map(FlagData::fromWire).collect(Collectors.toList());
+    }
+
     private static List<Rule> rulesFromWire(List<WireRule> wireRules) {
         if (wireRules == null) return Collections.emptyList();
         return wireRules.stream().map(Rule::fromWire).collect(Collectors.toList());
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java
index 09a7f635dd4..60b35d9b69e 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java
@@ -26,4 +26,12 @@ public class WireFlagDataList {
     public void serializeToOutputStream(OutputStream outputStream) {
         uncheck(() -> mapper.writeValue(outputStream, this));
     }
+
+    public byte[] serializeToBytes() {
+        return uncheck(() -> mapper.writeValueAsBytes(this));
+    }
+
+    public static WireFlagDataList deserializeFrom(byte[] bytes) {
+        return uncheck(() -> mapper.readValue(bytes, WireFlagDataList.class));
+    }
 }
diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java
index fec98fab164..fd1a71e4b4a 100644
--- a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java
+++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java
@@ -12,6 +12,7 @@ import org.junit.Test;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -22,27 +23,12 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  * @author hakonhall
  */
-public class FlagDirectoryTest {
+public class FlagDbFileTest {
     private final FileSystem fileSystem = TestFileSystem.create();
-    private final FlagDirectory flagDirectory = new FlagDirectory(fileSystem);
+    private final FlagDbFile flagDb = new FlagDbFile(fileSystem);
 
     @Test
-    public void testReadingOnly() {
-        Map<FlagId, FlagData> data = flagDirectory.read();
-        assertThat(data, IsMapWithSize.anEmptyMap());
-
-        FlagId id1 = new FlagId("id1");
-        String json1 = "{\"id\":\"id1\"}";
-        writeUtf8FlagFile(id1.toString(), json1);
-        data = flagDirectory.read();
-        assertThat(data, IsMapWithSize.aMapWithSize(1));
-        assertThat(data, IsMapContaining.hasKey(id1));
-        assertThat(data.get(id1).id(), equalTo(id1));
-        assertThat(data.get(id1).serializeToJson(), equalTo(json1));
-    }
-
-    @Test
-    public void testSync() {
+    public void test() {
         Map<FlagId, FlagData> dataMap = new HashMap<>();
         FlagId id1 = new FlagId("id1");
         FlagData data1 = new FlagData(id1, new FetchVector());
@@ -52,17 +38,19 @@ public class FlagDirectoryTest {
         dataMap.put(id2, data2);
 
         // Non-existing directory => empty map
-        assertThat(flagDirectory.read(), IsMapWithSize.anEmptyMap());
+        assertThat(flagDb.read(), IsMapWithSize.anEmptyMap());
 
         // sync() will create directory with map content
-        assertThat(flagDirectory.sync(dataMap), equalTo(true));
-        Map<FlagId, FlagData> readDataMap = flagDirectory.read();
+        assertThat(flagDb.sync(dataMap), equalTo(true));
+        Map<FlagId, FlagData> readDataMap = flagDb.read();
         assertThat(readDataMap, IsMapWithSize.aMapWithSize(2));
         assertThat(readDataMap, IsMapContaining.hasKey(id1));
         assertThat(readDataMap, IsMapContaining.hasKey(id2));
 
+        assertThat(getDbContent(), equalTo("{\"flags\":[{\"id\":\"id1\"},{\"id\":\"id2\"}]}"));
+
         // another sync with the same data is a no-op
-        assertThat(flagDirectory.sync(dataMap), equalTo(false));
+        assertThat(flagDb.sync(dataMap), equalTo(false));
 
         // Changing value of id1, removing id2, adding id3
         dataMap.remove(id2);
@@ -71,16 +59,18 @@ public class FlagDirectoryTest {
         FlagId id3 = new FlagId("id3");
         FlagData data3 = new FlagData(id3, new FetchVector());
         dataMap.put(id3, data3);
-        assertThat(flagDirectory.sync(dataMap), equalTo(true));
-        Map<FlagId, FlagData> anotherReadDataMap = flagDirectory.read();
+        assertThat(flagDb.sync(dataMap), equalTo(true));
+        Map<FlagId, FlagData> anotherReadDataMap = flagDb.read();
         assertThat(anotherReadDataMap, IsMapWithSize.aMapWithSize(2));
         assertThat(anotherReadDataMap, IsMapContaining.hasKey(id1));
         assertThat(anotherReadDataMap, IsMapContaining.hasKey(id3));
         assertThat(anotherReadDataMap.get(id1).serializeToJson(), equalTo("{\"id\":\"id1\",\"attributes\":{\"hostname\":\"h1\"}}"));
+
+        assertThat(flagDb.sync(Collections.emptyMap()), equalTo(true));
+        assertThat(getDbContent(), equalTo("{\"flags\":[]}"));
     }
 
-    private void writeUtf8FlagFile(String flagIdAkaFilename, String content) {
-        uncheck(() -> Files.createDirectories(flagDirectory.getPath()));
-        uncheck(() -> Files.write(flagDirectory.getPath().resolve(flagIdAkaFilename), content.getBytes(StandardCharsets.UTF_8)));
+    public String getDbContent() {
+        return uncheck(() -> new String(Files.readAllBytes(flagDb.getPath()), StandardCharsets.UTF_8));
     }
 }
\ No newline at end of file