From c3c189e1d9836cd985eaae0549390323900472cc Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 7 Nov 2019 11:20:43 +0100 Subject: Add type representing an archive of flag data files --- controller-api/pom.xml | 6 + .../api/systemflags/v1/SystemFlagsDataArchive.java | 138 +++++++++++++++++++++ .../systemflags/v1/SystemFlagsDataArchiveTest.java | 89 +++++++++++++ .../system-flags/flags/my-test-flag/default.json | 8 ++ .../flags/my-test-flag/main.controller.json | 8 ++ .../system-flags/flags/my-test-flag/main.json | 8 ++ .../system-flags/flags/my-test-flag/main.prod.json | 8 ++ .../flags/my-test-flag/main.prod.us-west-1.json | 8 ++ 8 files changed, 273 insertions(+) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java create mode 100644 controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java create mode 100644 controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json create mode 100644 controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json create mode 100644 controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json create mode 100644 controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json create mode 100644 controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json (limited to 'controller-api') diff --git a/controller-api/pom.xml b/controller-api/pom.xml index 6b7c01a863c..680ad35cace 100644 --- a/controller-api/pom.xml +++ b/controller-api/pom.xml @@ -83,6 +83,12 @@ test + + org.assertj + assertj-core + test + + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java new file mode 100644 index 00000000000..bababd378d5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -0,0 +1,138 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Represents a hierarchy of flag data files. See {@link FlagsTarget} for file naming convention. + * + * The flag files must reside in a 'flags/' root directory containing a directory for each flag name: + * {@code ./flags//*.json} + * + * @author bjorncs + */ +public class SystemFlagsDataArchive { + + private final Map> files; + + private SystemFlagsDataArchive(Map> files) { + this.files = files; + } + + public static SystemFlagsDataArchive fromZip(InputStream rawIn) { + Builder builder = new Builder(); + try (ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(rawIn))) { + ZipEntry entry; + while ((entry = zipIn.getNextEntry()) != null) { + String name = entry.getName(); + if (!entry.isDirectory() && name.startsWith("flags/") && name.endsWith(".json")) { + Path filePath = Paths.get(name); + String filename = filePath.getFileName().toString(); + FlagData flagData = FlagData.deserializeUtf8Json(zipIn.readAllBytes()); + verifyFlagDataMatchesDirectoryName(filePath, flagData); + builder.addFile(filename, flagData); + } + } + return builder.build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static SystemFlagsDataArchive fromDirectory(Path directory) { + Path root = directory.toAbsolutePath(); + try (Stream directoryStream = Files.walk(root)) { + Builder builder = new Builder(); + directoryStream.forEach(absolutePath -> { + Path relativePath = root.relativize(absolutePath); + if (!Files.isDirectory(absolutePath) && relativePath.startsWith("flags")) { + String filename = relativePath.getFileName().toString(); + if (filename.endsWith(".json")) { + FlagData flagData = FlagData.deserializeUtf8Json(uncheck(() -> Files.readAllBytes(absolutePath))); + verifyFlagDataMatchesDirectoryName(relativePath, flagData); + builder.addFile(filename, flagData); + } + } + }); + return builder.build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void toZip(OutputStream out) { + ZipOutputStream zipOut = new ZipOutputStream(out); + files.forEach((flagId, fileMap) -> { + fileMap.forEach((filename, flagData) -> { + uncheck(() -> { + zipOut.putNextEntry(new ZipEntry("flags/" + flagId.toString() + "/" + filename)); + zipOut.write(flagData.serializeToUtf8Json()); + zipOut.closeEntry(); + }); + }); + }); + uncheck(zipOut::flush); + } + + public Set flagData(FlagsTarget target) { + List filenames = target.flagDataFilesPrioritized(); + Set targetData = new HashSet<>(); + files.forEach((flagId, fileMap) -> { + for (String filename : filenames) { + FlagData data = fileMap.get(filename); + if (data != null) { + targetData.add(data); + break; + } + } + }); + return targetData; + } + + private static void verifyFlagDataMatchesDirectoryName(Path filePath, FlagData flagData) { + String flagDirectoryName = filePath.getName(1).toString(); + if (!flagDirectoryName.equals(flagData.id().toString())) { + throw new IllegalArgumentException( + String.format("Flag data file with flag id '%s' in directory for '%s'", flagData.id(), flagDirectoryName)); + } + } + + public static class Builder { + private final Map> files = new TreeMap<>(); + + public Builder() {} + + public Builder addFile(String filename, FlagData data) { + files.computeIfAbsent(data.id(), k -> new TreeMap<>()).put(filename, data); + return this; + } + + public SystemFlagsDataArchive build() { + Map> copy = new TreeMap<>(); + files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map))); + return new SystemFlagsDataArchive(copy); + } + + } +} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java new file mode 100644 index 00000000000..eb0e90f8ca1 --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -0,0 +1,89 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.RawFlag; +import com.yahoo.vespa.flags.json.FlagData; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author bjorncs + */ +public class SystemFlagsDataArchiveTest { + + private static final SystemName SYSTEM = SystemName.main; + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static FlagsTarget mainControllerTarget = FlagsTarget.forController(SYSTEM); + private static FlagsTarget prodUsWestCfgTarget = createConfigserverTarget(Environment.prod, "us-west-1"); + private static FlagsTarget prodUsEast3CfgTarget = createConfigserverTarget(Environment.prod, "us-east-3"); + private static FlagsTarget devUsEast1CfgTarget = createConfigserverTarget(Environment.dev, "us-east-1"); + + private static FlagsTarget createConfigserverTarget(Environment environment, String region) { + return new ConfigserverFlagsTarget( + SYSTEM, + ZoneId.from(environment, RegionName.from(region)), + URI.create("https://cfg-" + region), + new AthenzService("vespa.cfg-" + region)); + } + + @Test + public void can_serialize_and_deserialize_archive() throws IOException { + File tempFile = temporaryFolder.newFile("serialized-flags-archive"); + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) { + var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); + archive.toZip(out); + } + try (InputStream in = new BufferedInputStream(new FileInputStream(tempFile))) { + SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromZip(in); + assertArchiveReturnsCorrectDataForTarget(archive); + } + } + + @Test + public void retrieves_correct_flag_data_for_target() { + var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); + assertArchiveReturnsCorrectDataForTarget(archive); + } + + private static void assertArchiveReturnsCorrectDataForTarget(SystemFlagsDataArchive archive) { + assertFlagDataHasValue(archive, mainControllerTarget, "main.controller"); + assertFlagDataHasValue(archive, prodUsWestCfgTarget, "main.prod.us-west-1.json"); + assertFlagDataHasValue(archive, prodUsEast3CfgTarget, "main.prod"); + assertFlagDataHasValue(archive, devUsEast1CfgTarget, "main"); + } + + private static void assertFlagDataHasValue(SystemFlagsDataArchive archive, FlagsTarget target, String value) { + Set data = archive.flagData(target); + assertThat(data).hasSize(1); + FlagData flagData = data.iterator().next(); + RawFlag rawFlag = flagData.resolve(FetchVector.fromMap(Map.of())).get(); + assertThat(rawFlag.asJson()).isEqualTo(String.format("\"%s\"", value)); + } + +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json new file mode 100644 index 00000000000..2860c833533 --- /dev/null +++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "main.controller" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json new file mode 100644 index 00000000000..d94390cd2a4 --- /dev/null +++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "main" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json new file mode 100644 index 00000000000..28d2f068160 --- /dev/null +++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "main.prod" + } + ] +} \ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json new file mode 100644 index 00000000000..87b435cdab1 --- /dev/null +++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "main.prod.us-west-1.json" + } + ] +} \ No newline at end of file -- cgit v1.2.3