Merge pull request #8071 from vespa-engine/hakonhall/flag-repository-cfg-client-and-flag-directory

Flag repository cfg client and flag directory

9 files changed, 258 insertions, 5 deletions

diff --git a/flags/pom.xml b/flags/pom.xml
index 99fde3faebb..fc38676ff20 100644
--- a/flags/pom.xml
+++ b/flags/pom.xml
@@ -38,6 +38,12 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespalog</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
             <scope>provided</scope>
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
index 581ec599aab..7d84efa52b2 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
@@ -17,8 +17,7 @@ import java.util.function.Consumer;
  * @author hakonhall
  */
 @Immutable
-public
-class FetchVector {
+public class FetchVector {
     public enum Dimension {
         /** Value from ZoneId::value */
         ZONE_ID,
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java
index f4e23144449..1a31ecd713e 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java
@@ -1,7 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.flags;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.inject.Inject;
 import com.yahoo.vespa.flags.json.FlagData;
 import com.yahoo.vespa.flags.json.Rule;
@@ -21,8 +20,6 @@ import java.util.Optional;
  * @author hakonhall
  */
 public class FileFlagSource implements FlagSource {
-    private static final ObjectMapper mapper = new ObjectMapper();
-
     static final String FLAGS_DIRECTORY = "/etc/vespa/flags";
 
     private final Path flagsDirectory;
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java
index da8e6b29cab..182ab85858c 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java
@@ -4,8 +4,11 @@ package com.yahoo.vespa.flags;
 import java.util.Optional;
 
 /**
+ * A source of raw flag values that can be converted to typed flag values elsewhere.
+ *
  * @author hakonhall
  */
 public interface FlagSource {
+    /** Get raw flag for the given vector (specifying hostname, application id, etc). */
     Optional<RawFlag> fetch(FlagId id, FetchVector vector);
 }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java
new file mode 100644
index 00000000000..abe8d407ab0
--- /dev/null
+++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java
@@ -0,0 +1,111 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.flags.file;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Java API for a flag database stored in a single file
+ *
+ * @author hakonhall
+ */
+public class FlagDbFile {
+    private static final Logger logger = Logger.getLogger(FlagDbFile.class.getName());
+
+    private final Path path;
+
+    public FlagDbFile() {
+        this(FileSystems.getDefault());
+    }
+
+    public FlagDbFile(FileSystem fileSystem) {
+        this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flag.db"));
+    }
+
+    public FlagDbFile(Path path) {
+        this.path = path;
+    }
+
+    public Path getPath() {
+        return path;
+    }
+
+    public Map<FlagId, FlagData> read() {
+        Optional<byte[]> bytes = readFile();
+        if (!bytes.isPresent()) return Collections.emptyMap();
+        return FlagData.deserializeList(bytes.get()).stream().collect(Collectors.toMap(FlagData::id, Function.identity()));
+    }
+
+    public boolean sync(Map<FlagId, FlagData> flagData) {
+        boolean modified = false;
+        Map<FlagId, FlagData> currentFlagData = read();
+        Set<FlagId> flagIdsToBeRemoved = new HashSet<>(currentFlagData.keySet());
+        List<FlagData> flagDataList = new ArrayList<>(flagData.values());
+
+        for (FlagData data : flagDataList) {
+            flagIdsToBeRemoved.remove(data.id());
+
+            FlagData existingFlagData = currentFlagData.get(data.id());
+            if (existingFlagData == null) {
+                logger.log(LogLevel.INFO, "New flag " + data.id() + ": " + data.serializeToJson());
+                modified = true;
+
+                // Could also consider testing with FlagData::equals, but that would be too fragile?
+            } else if (!Objects.equals(data.serializeToJson(), existingFlagData.serializeToJson())){
+                logger.log(LogLevel.INFO, "Updating flag " + data.id() + " from " +
+                        existingFlagData.serializeToJson() + " to " + data.serializeToJson());
+                modified = true;
+            }
+        }
+
+        if (!flagIdsToBeRemoved.isEmpty()) {
+            String flagIdsString = flagIdsToBeRemoved.stream().map(FlagId::toString).collect(Collectors.joining(", "));
+            logger.log(LogLevel.INFO, "Removing flags " + flagIdsString);
+            modified = true;
+        }
+
+        if (!modified) return false;
+
+        writeFile(FlagData.serializeListToUtf8Json(flagDataList));
+
+        return modified;
+    }
+
+    private Optional<byte[]> readFile() {
+        try {
+            return Optional.of(Files.readAllBytes(path));
+        } catch (NoSuchFileException e) {
+            return Optional.empty();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private void writeFile(byte[] bytes) {
+        uncheck(() -> Files.createDirectories(path.getParent()));
+        uncheck(() -> Files.write(path, bytes));
+    }
+}
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java
new file mode 100644
index 00000000000..27ad44f938e
--- /dev/null
+++ b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.flags.file;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java
index 572ec511607..64c4bbe7616 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.flags.FlagId;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.RawFlag;
 import com.yahoo.vespa.flags.json.wire.WireFlagData;
+import com.yahoo.vespa.flags.json.wire.WireFlagDataList;
 import com.yahoo.vespa.flags.json.wire.WireRule;
 
 import javax.annotation.concurrent.Immutable;
@@ -114,6 +115,24 @@ public class FlagData {
         );
     }
 
+    public static byte[] serializeListToUtf8Json(List<FlagData> list) {
+        return listToWire(list).serializeToBytes();
+    }
+
+    public static List<FlagData> deserializeList(byte[] bytes) {
+        return listFromWire(WireFlagDataList.deserializeFrom(bytes));
+    }
+
+    public static WireFlagDataList listToWire(List<FlagData> list) {
+        WireFlagDataList wireList = new WireFlagDataList();
+        wireList.flags = list.stream().map(FlagData::toWire).collect(Collectors.toList());
+        return wireList;
+    }
+
+    public static List<FlagData> listFromWire(WireFlagDataList wireList) {
+        return wireList.flags.stream().map(FlagData::fromWire).collect(Collectors.toList());
+    }
+
     private static List<Rule> rulesFromWire(List<WireRule> wireRules) {
         if (wireRules == null) return Collections.emptyList();
         return wireRules.stream().map(Rule::fromWire).collect(Collectors.toList());
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java
new file mode 100644
index 00000000000..60b35d9b69e
--- /dev/null
+++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java
@@ -0,0 +1,37 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.flags.json.wire;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author hakonhall
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class WireFlagDataList {
+    @JsonProperty("flags")
+    public List<WireFlagData> flags = new ArrayList<>();
+
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    public void serializeToOutputStream(OutputStream outputStream) {
+        uncheck(() -> mapper.writeValue(outputStream, this));
+    }
+
+    public byte[] serializeToBytes() {
+        return uncheck(() -> mapper.writeValueAsBytes(this));
+    }
+
+    public static WireFlagDataList deserializeFrom(byte[] bytes) {
+        return uncheck(() -> mapper.readValue(bytes, WireFlagDataList.class));
+    }
+}
diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java
new file mode 100644
index 00000000000..fd1a71e4b4a
--- /dev/null
+++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java
@@ -0,0 +1,76 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.flags.file;
+
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.hamcrest.collection.IsMapContaining;
+import org.hamcrest.collection.IsMapWithSize;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * @author hakonhall
+ */
+public class FlagDbFileTest {
+    private final FileSystem fileSystem = TestFileSystem.create();
+    private final FlagDbFile flagDb = new FlagDbFile(fileSystem);
+
+    @Test
+    public void test() {
+        Map<FlagId, FlagData> dataMap = new HashMap<>();
+        FlagId id1 = new FlagId("id1");
+        FlagData data1 = new FlagData(id1, new FetchVector());
+        dataMap.put(id1, data1);
+        FlagId id2 = new FlagId("id2");
+        FlagData data2 = new FlagData(id2, new FetchVector());
+        dataMap.put(id2, data2);
+
+        // Non-existing directory => empty map
+        assertThat(flagDb.read(), IsMapWithSize.anEmptyMap());
+
+        // sync() will create directory with map content
+        assertThat(flagDb.sync(dataMap), equalTo(true));
+        Map<FlagId, FlagData> readDataMap = flagDb.read();
+        assertThat(readDataMap, IsMapWithSize.aMapWithSize(2));
+        assertThat(readDataMap, IsMapContaining.hasKey(id1));
+        assertThat(readDataMap, IsMapContaining.hasKey(id2));
+
+        assertThat(getDbContent(), equalTo("{\"flags\":[{\"id\":\"id1\"},{\"id\":\"id2\"}]}"));
+
+        // another sync with the same data is a no-op
+        assertThat(flagDb.sync(dataMap), equalTo(false));
+
+        // Changing value of id1, removing id2, adding id3
+        dataMap.remove(id2);
+        FlagData newData1 = new FlagData(id1, new FetchVector().with(FetchVector.Dimension.HOSTNAME, "h1"));
+        dataMap.put(id1, newData1);
+        FlagId id3 = new FlagId("id3");
+        FlagData data3 = new FlagData(id3, new FetchVector());
+        dataMap.put(id3, data3);
+        assertThat(flagDb.sync(dataMap), equalTo(true));
+        Map<FlagId, FlagData> anotherReadDataMap = flagDb.read();
+        assertThat(anotherReadDataMap, IsMapWithSize.aMapWithSize(2));
+        assertThat(anotherReadDataMap, IsMapContaining.hasKey(id1));
+        assertThat(anotherReadDataMap, IsMapContaining.hasKey(id3));
+        assertThat(anotherReadDataMap.get(id1).serializeToJson(), equalTo("{\"id\":\"id1\",\"attributes\":{\"hostname\":\"h1\"}}"));
+
+        assertThat(flagDb.sync(Collections.emptyMap()), equalTo(true));
+        assertThat(getDbContent(), equalTo("{\"flags\":[]}"));
+    }
+
+    public String getDbContent() {
+        return uncheck(() -> new String(Files.readAllBytes(flagDb.getPath()), StandardCharsets.UTF_8));
+    }
+}
\ No newline at end of file