Add type representing an archive of flag data files

8 files changed, 273 insertions, 0 deletions

diff --git a/controller-api/pom.xml b/controller-api/pom.xml
index 6b7c01a863c..680ad35cace 100644
--- a/controller-api/pom.xml
+++ b/controller-api/pom.xml
@@ -83,6 +83,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
new file mode 100644
index 00000000000..bababd378d5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
@@ -0,0 +1,138 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
+
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Represents a hierarchy of flag data files. See {@link FlagsTarget} for file naming convention.
+ *
+ * The flag files must reside in a 'flags/' root directory containing a directory for each flag name:
+ * {@code ./flags/<feature-flag-name>/*.json}
+ *
+ * @author bjorncs
+ */
+public class SystemFlagsDataArchive {
+
+    private final Map<FlagId, Map<String, FlagData>> files;
+
+    private SystemFlagsDataArchive(Map<FlagId, Map<String, FlagData>> files) {
+        this.files = files;
+    }
+
+    public static SystemFlagsDataArchive fromZip(InputStream rawIn) {
+        Builder builder = new Builder();
+        try (ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(rawIn))) {
+            ZipEntry entry;
+            while ((entry = zipIn.getNextEntry()) != null) {
+                String name = entry.getName();
+                if (!entry.isDirectory() && name.startsWith("flags/") && name.endsWith(".json")) {
+                    Path filePath = Paths.get(name);
+                    String filename = filePath.getFileName().toString();
+                    FlagData flagData = FlagData.deserializeUtf8Json(zipIn.readAllBytes());
+                    verifyFlagDataMatchesDirectoryName(filePath, flagData);
+                    builder.addFile(filename, flagData);
+                }
+            }
+            return builder.build();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static SystemFlagsDataArchive fromDirectory(Path directory) {
+        Path root = directory.toAbsolutePath();
+        try (Stream<Path> directoryStream = Files.walk(root)) {
+            Builder builder = new Builder();
+            directoryStream.forEach(absolutePath -> {
+                Path relativePath = root.relativize(absolutePath);
+                if (!Files.isDirectory(absolutePath) && relativePath.startsWith("flags")) {
+                    String filename = relativePath.getFileName().toString();
+                    if (filename.endsWith(".json")) {
+                        FlagData flagData = FlagData.deserializeUtf8Json(uncheck(() -> Files.readAllBytes(absolutePath)));
+                        verifyFlagDataMatchesDirectoryName(relativePath, flagData);
+                        builder.addFile(filename, flagData);
+                    }
+                }
+            });
+            return builder.build();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public void toZip(OutputStream out) {
+        ZipOutputStream zipOut = new ZipOutputStream(out);
+        files.forEach((flagId, fileMap) -> {
+            fileMap.forEach((filename, flagData) -> {
+                uncheck(() -> {
+                    zipOut.putNextEntry(new ZipEntry("flags/" + flagId.toString() + "/" + filename));
+                    zipOut.write(flagData.serializeToUtf8Json());
+                    zipOut.closeEntry();
+                });
+            });
+        });
+        uncheck(zipOut::flush);
+    }
+
+    public Set<FlagData> flagData(FlagsTarget target) {
+        List<String> filenames = target.flagDataFilesPrioritized();
+        Set<FlagData> targetData = new HashSet<>();
+        files.forEach((flagId, fileMap) -> {
+            for (String filename : filenames) {
+                FlagData data = fileMap.get(filename);
+                if (data != null) {
+                    targetData.add(data);
+                    break;
+                }
+            }
+        });
+        return targetData;
+    }
+
+    private static void verifyFlagDataMatchesDirectoryName(Path filePath, FlagData flagData) {
+        String flagDirectoryName = filePath.getName(1).toString();
+        if (!flagDirectoryName.equals(flagData.id().toString())) {
+            throw new IllegalArgumentException(
+                    String.format("Flag data file with flag id '%s' in directory for '%s'", flagData.id(), flagDirectoryName));
+        }
+    }
+
+    public static class Builder {
+        private final Map<FlagId, Map<String, FlagData>> files = new TreeMap<>();
+
+        public Builder() {}
+
+        public Builder addFile(String filename, FlagData data) {
+            files.computeIfAbsent(data.id(), k -> new TreeMap<>()).put(filename, data);
+            return this;
+        }
+
+        public SystemFlagsDataArchive build() {
+            Map<FlagId, Map<String, FlagData>> copy = new TreeMap<>();
+            files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map)));
+            return new SystemFlagsDataArchive(copy);
+        }
+
+    }
+}
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
new file mode 100644
index 00000000000..eb0e90f8ca1
--- /dev/null
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
@@ -0,0 +1,89 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.athenz.api.AthenzService;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.RawFlag;
+import com.yahoo.vespa.flags.json.FlagData;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author bjorncs
+ */
+public class SystemFlagsDataArchiveTest {
+
+    private static final SystemName SYSTEM = SystemName.main;
+
+    @Rule
+    public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private static FlagsTarget mainControllerTarget = FlagsTarget.forController(SYSTEM);
+    private static FlagsTarget prodUsWestCfgTarget = createConfigserverTarget(Environment.prod, "us-west-1");
+    private static FlagsTarget prodUsEast3CfgTarget = createConfigserverTarget(Environment.prod, "us-east-3");
+    private static FlagsTarget devUsEast1CfgTarget = createConfigserverTarget(Environment.dev, "us-east-1");
+
+    private static FlagsTarget createConfigserverTarget(Environment environment, String region) {
+        return new ConfigserverFlagsTarget(
+                SYSTEM,
+                ZoneId.from(environment, RegionName.from(region)),
+                URI.create("https://cfg-" + region),
+                new AthenzService("vespa.cfg-" + region));
+    }
+
+    @Test
+    public void can_serialize_and_deserialize_archive() throws IOException {
+        File tempFile = temporaryFolder.newFile("serialized-flags-archive");
+        try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) {
+            var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/"));
+            archive.toZip(out);
+        }
+        try (InputStream in = new BufferedInputStream(new FileInputStream(tempFile))) {
+            SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromZip(in);
+            assertArchiveReturnsCorrectDataForTarget(archive);
+        }
+    }
+
+    @Test
+    public void retrieves_correct_flag_data_for_target() {
+        var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/"));
+        assertArchiveReturnsCorrectDataForTarget(archive);
+    }
+
+    private static void assertArchiveReturnsCorrectDataForTarget(SystemFlagsDataArchive archive) {
+        assertFlagDataHasValue(archive, mainControllerTarget, "main.controller");
+        assertFlagDataHasValue(archive, prodUsWestCfgTarget, "main.prod.us-west-1.json");
+        assertFlagDataHasValue(archive, prodUsEast3CfgTarget, "main.prod");
+        assertFlagDataHasValue(archive, devUsEast1CfgTarget, "main");
+    }
+
+    private static void assertFlagDataHasValue(SystemFlagsDataArchive archive, FlagsTarget target, String value) {
+        Set<FlagData> data = archive.flagData(target);
+        assertThat(data).hasSize(1);
+        FlagData flagData = data.iterator().next();
+        RawFlag rawFlag = flagData.resolve(FetchVector.fromMap(Map.of())).get();
+        assertThat(rawFlag.asJson()).isEqualTo(String.format("\"%s\"", value));
+    }
+
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json
new file mode 100644
index 00000000000..5924eb860c0
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "default"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json
new file mode 100644
index 00000000000..2860c833533
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "main.controller"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json
new file mode 100644
index 00000000000..d94390cd2a4
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "main"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json
new file mode 100644
index 00000000000..28d2f068160
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "main.prod"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json
new file mode 100644
index 00000000000..87b435cdab1
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "main.prod.us-west-1.json"
+    }
+  ]
+}
\ No newline at end of file