Merge pull request #11238 from vespa-engine/bjorncs/system-flags-handler

Bjorncs/system flags handler

27 files changed, 1226 insertions, 0 deletions

diff --git a/controller-api/pom.xml b/controller-api/pom.xml
index 6b7c01a863c..680ad35cace 100644
--- a/controller-api/pom.xml
+++ b/controller-api/pom.xml
@@ -83,6 +83,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ConfigServerFlagsTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ConfigServerFlagsTarget.java
new file mode 100644
index 00000000000..0304d1a5949
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ConfigServerFlagsTarget.java
@@ -0,0 +1,51 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
+
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.defaultFile;
+import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.environmentFile;
+import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.systemFile;
+import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.zoneFile;
+
+/**
+ * @author bjorncs
+ */
+class ConfigServerFlagsTarget implements FlagsTarget {
+    private final SystemName system;
+    private final ZoneId zone;
+    private final URI endpoint;
+    private final AthenzIdentity identity;
+
+    ConfigServerFlagsTarget(SystemName system, ZoneId zone, URI endpoint, AthenzIdentity identity) {
+        this.system = Objects.requireNonNull(system);
+        this.zone = Objects.requireNonNull(zone);
+        this.endpoint = Objects.requireNonNull(endpoint);
+        this.identity = Objects.requireNonNull(identity);
+    }
+
+    @Override public List<String> flagDataFilesPrioritized() { return List.of(zoneFile(system, zone), environmentFile(system, zone.environment()), systemFile(system), defaultFile()); }
+    @Override public URI endpoint() { return endpoint; }
+    @Override public Optional<AthenzIdentity> athenzHttpsIdentity() { return Optional.of(identity); }
+    @Override public String asString() { return String.format("%s.%s", system.value(), zone.value()); }
+
+    @Override public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ConfigServerFlagsTarget that = (ConfigServerFlagsTarget) o;
+        return system == that.system &&
+                Objects.equals(zone, that.zone) &&
+                Objects.equals(endpoint, that.endpoint) &&
+                Objects.equals(identity, that.identity);
+    }
+
+    @Override public int hashCode() { return Objects.hash(system, zone, endpoint, identity); }
+}
+
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java
new file mode 100644
index 00000000000..a22a9cc63de
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java
@@ -0,0 +1,38 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
+
+/**
+ * @author bjorncs
+ */
+
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.controllerFile;
+import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.defaultFile;
+import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.systemFile;
+
+class ControllerFlagsTarget implements FlagsTarget {
+    private final SystemName system;
+
+    ControllerFlagsTarget(SystemName system) { this.system = Objects.requireNonNull(system); }
+
+    @Override public List<String> flagDataFilesPrioritized() { return List.of(controllerFile(system), systemFile(system), defaultFile()); }
+    @Override public URI endpoint() { return URI.create("https://localhost:4443/"); } // Note: Cannot use VIPs for controllers due to network configuration on AWS
+    @Override public Optional<AthenzIdentity> athenzHttpsIdentity() { return Optional.empty(); }
+    @Override public String asString() { return String.format("%s.controller", system.value()); }
+
+    @Override public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ControllerFlagsTarget that = (ControllerFlagsTarget) o;
+        return system == that.system;
+    }
+
+    @Override public int hashCode() { return Objects.hash(system); }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java
new file mode 100644
index 00000000000..1b7f84d1ec7
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java
@@ -0,0 +1,68 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+
+import java.net.URI;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Represents either configservers in a zone or controllers in a system.
+ *
+ * Defines the location and precedence of the flags data files for the given target.
+ *
+ * Naming rules for flags data files:
+ * <ul>
+ *  <li>zone specific: {@code <system>.<environment>.<region>.json}</li>
+ *  <li>controller specific: {@code <system>.controller.json}</li>
+ *  <li>environment specific: {@code <system>.<environment>.json}</li>
+ *  <li>system specific: {@code <system>.json}</li>
+ *  <li>global default: {@code default.json}</li>
+ * </ul>
+ *
+ * @author bjorncs
+ */
+public interface FlagsTarget {
+
+    List<String> flagDataFilesPrioritized();
+    URI endpoint();
+    Optional<AthenzIdentity> athenzHttpsIdentity();
+    String asString();
+
+    static Set<FlagsTarget> getAllTargetsInSystem(ZoneRegistry registry) {
+        SystemName system = registry.system();
+        Set<FlagsTarget> targets = new HashSet<>();
+        for (ZoneApi zone : registry.zones().reachable().zones()) {
+            targets.add(forConfigServer(registry, zone.getId()));
+        }
+        targets.add(forController(system));
+        return targets;
+    }
+
+    static FlagsTarget forController(SystemName systemName) {
+        return new ControllerFlagsTarget(systemName);
+    }
+
+    static FlagsTarget forConfigServer(ZoneRegistry registry, ZoneId zoneId) {
+        return new ConfigServerFlagsTarget(
+                registry.system(), zoneId, registry.getConfigServerVipUri(zoneId), registry.getConfigServerHttpsIdentity(zoneId));
+    }
+
+    static String defaultFile() { return jsonFile("default"); }
+    static String systemFile(SystemName system) { return jsonFile(system.value()); }
+    static String environmentFile(SystemName system, Environment environment) { return jsonFile(system.value() + "." + environment); }
+    static String zoneFile(SystemName system, ZoneId zone) { return jsonFile(system.value() + "." + zone.environment().value() + "." + zone.region().value()); }
+    static String controllerFile(SystemName system) { return jsonFile(system.value() + ".controller"); }
+
+    private static String jsonFile(String nameWithoutExtension) { return nameWithoutExtension + ".json"; }
+}
+
+
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
new file mode 100644
index 00000000000..ec14dcb7123
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
@@ -0,0 +1,138 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
+
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Represents a hierarchy of flag data files. See {@link FlagsTarget} for file naming convention.
+ *
+ * The flag files must reside in a 'flags/' root directory containing a directory for each flag name:
+ * {@code ./flags/<flag-id>/*.json}
+ *
+ * @author bjorncs
+ */
+public class SystemFlagsDataArchive {
+
+    private final Map<FlagId, Map<String, FlagData>> files;
+
+    private SystemFlagsDataArchive(Map<FlagId, Map<String, FlagData>> files) {
+        this.files = files;
+    }
+
+    public static SystemFlagsDataArchive fromZip(InputStream rawIn) {
+        Builder builder = new Builder();
+        try (ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(rawIn))) {
+            ZipEntry entry;
+            while ((entry = zipIn.getNextEntry()) != null) {
+                String name = entry.getName();
+                if (!entry.isDirectory() && name.startsWith("flags/") && name.endsWith(".json")) {
+                    Path filePath = Paths.get(name);
+                    String filename = filePath.getFileName().toString();
+                    FlagData flagData = FlagData.deserializeUtf8Json(zipIn.readAllBytes());
+                    verifyFlagDataMatchesDirectoryName(filePath, flagData);
+                    builder.addFile(filename, flagData);
+                }
+            }
+            return builder.build();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static SystemFlagsDataArchive fromDirectory(Path directory) {
+        Path root = directory.toAbsolutePath();
+        try (Stream<Path> directoryStream = Files.walk(root)) {
+            Builder builder = new Builder();
+            directoryStream.forEach(absolutePath -> {
+                Path relativePath = root.relativize(absolutePath);
+                if (!Files.isDirectory(absolutePath) && relativePath.startsWith("flags")) {
+                    String filename = relativePath.getFileName().toString();
+                    if (filename.endsWith(".json")) {
+                        FlagData flagData = FlagData.deserializeUtf8Json(uncheck(() -> Files.readAllBytes(absolutePath)));
+                        verifyFlagDataMatchesDirectoryName(relativePath, flagData);
+                        builder.addFile(filename, flagData);
+                    }
+                }
+            });
+            return builder.build();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public void toZip(OutputStream out) {
+        ZipOutputStream zipOut = new ZipOutputStream(out);
+        files.forEach((flagId, fileMap) -> {
+            fileMap.forEach((filename, flagData) -> {
+                uncheck(() -> {
+                    zipOut.putNextEntry(new ZipEntry("flags/" + flagId.toString() + "/" + filename));
+                    zipOut.write(flagData.serializeToUtf8Json());
+                    zipOut.closeEntry();
+                });
+            });
+        });
+        uncheck(zipOut::flush);
+    }
+
+    public Set<FlagData> flagData(FlagsTarget target) {
+        List<String> filenames = target.flagDataFilesPrioritized();
+        Set<FlagData> targetData = new HashSet<>();
+        files.forEach((flagId, fileMap) -> {
+            for (String filename : filenames) {
+                FlagData data = fileMap.get(filename);
+                if (data != null) {
+                    targetData.add(data);
+                    break;
+                }
+            }
+        });
+        return targetData;
+    }
+
+    private static void verifyFlagDataMatchesDirectoryName(Path filePath, FlagData flagData) {
+        String flagDirectoryName = filePath.getName(1).toString();
+        if (!flagDirectoryName.equals(flagData.id().toString())) {
+            throw new IllegalArgumentException(
+                    String.format("Flag data file with flag id '%s' in directory for '%s'", flagData.id(), flagDirectoryName));
+        }
+    }
+
+    public static class Builder {
+        private final Map<FlagId, Map<String, FlagData>> files = new TreeMap<>();
+
+        public Builder() {}
+
+        public Builder addFile(String filename, FlagData data) {
+            files.computeIfAbsent(data.id(), k -> new TreeMap<>()).put(filename, data);
+            return this;
+        }
+
+        public SystemFlagsDataArchive build() {
+            Map<FlagId, Map<String, FlagData>> copy = new TreeMap<>();
+            files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map)));
+            return new SystemFlagsDataArchive(copy);
+        }
+
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/package-info.java
new file mode 100644
index 00000000000..0fe377db08c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bjorncs
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
+
+import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/SystemFlagsV1Api.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/SystemFlagsV1Api.java
new file mode 100644
index 00000000000..1107b70c01c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/SystemFlagsV1Api.java
@@ -0,0 +1,29 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import java.io.InputStream;
+
+/**
+ * @author bjorncs
+ */
+@Path("/system-flags/v1")
+public interface SystemFlagsV1Api {
+
+    @PUT
+    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes("application/zip")
+    @Path("/deploy")
+    WireSystemFlagsDeployResult deploy(InputStream inputStream);
+
+    @PUT
+    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes("application/zip")
+    @Path("/dryrun")
+    WireSystemFlagsDeployResult dryrun(InputStream inputStream);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireErrorResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireErrorResponse.java
new file mode 100644
index 00000000000..996f0ac6cdd
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireErrorResponse.java
@@ -0,0 +1,18 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class WireErrorResponse {
+    @JsonProperty("message")
+    public String message;
+    @JsonProperty("error-code")
+    public String errorCode;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java
new file mode 100644
index 00000000000..bd54fd15d15
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java
@@ -0,0 +1,32 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.flags.json.wire.WireFlagData;
+
+import java.util.List;
+
+/**
+ *
+ *
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class WireSystemFlagsDeployResult {
+    @JsonProperty("changes") public List<WireFlagDataChange> changes;
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public static class WireFlagDataChange {
+        @JsonProperty("flag-id") public String flagId;
+        @JsonProperty("targets") public List<String> targets;
+        @JsonProperty("operation") public String operation;
+        @JsonProperty("data") public WireFlagData data;
+        @JsonProperty("previous-data") public WireFlagData previousData;
+    }
+}
+
+
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/package-info.java
new file mode 100644
index 00000000000..305cf91014b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bjorncs
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire;
+
+import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
new file mode 100644
index 00000000000..35fec04e4c5
--- /dev/null
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
@@ -0,0 +1,89 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.athenz.api.AthenzService;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.RawFlag;
+import com.yahoo.vespa.flags.json.FlagData;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author bjorncs
+ */
+public class SystemFlagsDataArchiveTest {
+
+    private static final SystemName SYSTEM = SystemName.main;
+
+    @Rule
+    public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private static FlagsTarget mainControllerTarget = FlagsTarget.forController(SYSTEM);
+    private static FlagsTarget prodUsWestCfgTarget = createConfigserverTarget(Environment.prod, "us-west-1");
+    private static FlagsTarget prodUsEast3CfgTarget = createConfigserverTarget(Environment.prod, "us-east-3");
+    private static FlagsTarget devUsEast1CfgTarget = createConfigserverTarget(Environment.dev, "us-east-1");
+
+    private static FlagsTarget createConfigserverTarget(Environment environment, String region) {
+        return new ConfigServerFlagsTarget(
+                SYSTEM,
+                ZoneId.from(environment, RegionName.from(region)),
+                URI.create("https://cfg-" + region),
+                new AthenzService("vespa.cfg-" + region));
+    }
+
+    @Test
+    public void can_serialize_and_deserialize_archive() throws IOException {
+        File tempFile = temporaryFolder.newFile("serialized-flags-archive");
+        try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) {
+            var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/"));
+            archive.toZip(out);
+        }
+        try (InputStream in = new BufferedInputStream(new FileInputStream(tempFile))) {
+            SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromZip(in);
+            assertArchiveReturnsCorrectDataForTarget(archive);
+        }
+    }
+
+    @Test
+    public void retrieves_correct_flag_data_for_target() {
+        var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/"));
+        assertArchiveReturnsCorrectDataForTarget(archive);
+    }
+
+    private static void assertArchiveReturnsCorrectDataForTarget(SystemFlagsDataArchive archive) {
+        assertFlagDataHasValue(archive, mainControllerTarget, "main.controller");
+        assertFlagDataHasValue(archive, prodUsWestCfgTarget, "main.prod.us-west-1.json");
+        assertFlagDataHasValue(archive, prodUsEast3CfgTarget, "main.prod");
+        assertFlagDataHasValue(archive, devUsEast1CfgTarget, "main");
+    }
+
+    private static void assertFlagDataHasValue(SystemFlagsDataArchive archive, FlagsTarget target, String value) {
+        Set<FlagData> data = archive.flagData(target);
+        assertThat(data).hasSize(1);
+        FlagData flagData = data.iterator().next();
+        RawFlag rawFlag = flagData.resolve(FetchVector.fromMap(Map.of())).get();
+        assertThat(rawFlag.asJson()).isEqualTo(String.format("\"%s\"", value));
+    }
+
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json
new file mode 100644
index 00000000000..5924eb860c0
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/default.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "default"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json
new file mode 100644
index 00000000000..2860c833533
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.controller.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "main.controller"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json
new file mode 100644
index 00000000000..d94390cd2a4
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "main"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json
new file mode 100644
index 00000000000..28d2f068160
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "main.prod"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json
new file mode 100644
index 00000000000..87b435cdab1
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags/flags/my-test-flag/main.prod.us-west-1.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "main.prod.us-west-1.json"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-server/pom.xml b/controller-server/pom.xml
index ae756eae1fb..0c545020449 100644
--- a/controller-server/pom.xml
+++ b/controller-server/pom.xml
@@ -172,6 +172,18 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java
new file mode 100644
index 00000000000..d11e17ce634
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java
@@ -0,0 +1,181 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.systemflags;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
+import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireErrorResponse;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLSession;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.util.List;
+import java.util.Set;
+
+import static java.util.stream.Collectors.toSet;
+
+/**
+ * A client for /flags/v1 rest api on configserver and controller.
+ *
+ * @author bjorncs
+ */
+class FlagsClient {
+
+    private static final String FLAGS_V1_PATH = "/flags/v1";
+
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    private final CloseableHttpClient client;
+
+    FlagsClient(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) {
+        this.client = createClient(identityProvider, targets);
+    }
+
+    List<FlagData> listFlagData(FlagsTarget target) throws FlagsException, UncheckedIOException {
+        HttpGet request = new HttpGet(createUri(target, "/data"));
+        return executeRequest(request, response -> {
+            verifySuccess(response, target, null);
+            return FlagData.deserializeList(EntityUtils.toByteArray(response.getEntity()));
+        });
+    }
+
+    void putFlagData(FlagsTarget target, FlagData flagData) throws FlagsException, UncheckedIOException {
+        HttpPut request = new HttpPut(createUri(target, "/data/" + flagData.id().toString()));
+        request.setEntity(jsonContent(flagData.serializeToJson()));
+        executeRequest(request, response -> {
+            verifySuccess(response, target, flagData.id());
+            return null;
+        });
+    }
+
+    void deleteFlagData(FlagsTarget target, FlagId flagId) throws FlagsException, UncheckedIOException {
+        HttpDelete request = new HttpDelete(createUri(target, "/data/" + flagId.toString()));
+        executeRequest(request, response -> {
+            verifySuccess(response, target, flagId);
+            return null;
+        });
+    }
+
+
+    private static CloseableHttpClient createClient(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) {
+        return HttpClientBuilder.create()
+                .setUserAgent("controller-flags-v1-client")
+                .setRetryHandler(new DefaultHttpRequestRetryHandler(5, /*retry on non-idempotent requests*/true))
+                .setSslcontext(identityProvider.getIdentitySslContext())
+                .setSSLHostnameVerifier(new FlagTargetsHostnameVerifier(targets))
+                .setDefaultRequestConfig(RequestConfig.custom()
+                                                 .setConnectTimeout((int) Duration.ofSeconds(10).toMillis())
+                                                 .setConnectionRequestTimeout((int) Duration.ofSeconds(10).toMillis())
+                                                 .setSocketTimeout((int) Duration.ofSeconds(20).toMillis())
+                                                 .build())
+                .setMaxConnPerRoute(2)
+                .setMaxConnTotal(100)
+                .build();
+    }
+
+    private <T> T executeRequest(HttpUriRequest request, ResponseHandler<T> handler) {
+        try {
+            return client.execute(request, handler);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static URI createUri(FlagsTarget target, String subPath) {
+        try {
+            return new URIBuilder(target.endpoint()).setPath(FLAGS_V1_PATH + subPath).build();
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e); // should never happen
+        }
+    }
+
+    private static void verifySuccess(HttpResponse response, FlagsTarget target, FlagId flagId) throws IOException {
+        if (!success(response)) {
+            throw createFlagsException(response, target, flagId);
+        }
+    }
+
+    private static FlagsException createFlagsException(HttpResponse response, FlagsTarget target, FlagId flagId) throws IOException {
+        HttpEntity entity = response.getEntity();
+        String content = EntityUtils.toString(entity);
+        int statusCode = response.getStatusLine().getStatusCode();
+        if (ContentType.get(entity).getMimeType().equals(ContentType.APPLICATION_JSON.getMimeType())) {
+            WireErrorResponse error = mapper.readValue(content, WireErrorResponse.class);
+            return new FlagsException(statusCode, target, flagId, error.errorCode, error.message);
+        } else {
+            return new FlagsException(statusCode, target, flagId, null, content);
+        }
+    }
+
+    private static boolean success(HttpResponse response) {
+        return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
+    }
+
+    private static StringEntity jsonContent(String json) {
+        return new StringEntity(json, ContentType.APPLICATION_JSON);
+    }
+
+    private static class FlagTargetsHostnameVerifier implements HostnameVerifier {
+
+        private final AthenzIdentityVerifier athenzVerifier;
+
+        FlagTargetsHostnameVerifier(Set<FlagsTarget> targets) {
+            this.athenzVerifier = createAthenzIdentityVerifier(targets);
+        }
+
+        private static AthenzIdentityVerifier createAthenzIdentityVerifier(Set<FlagsTarget> targets) {
+            Set<AthenzIdentity> identities = targets.stream()
+                    .flatMap(target -> target.athenzHttpsIdentity().stream())
+                    .collect(toSet());
+            return new AthenzIdentityVerifier(identities);
+        }
+
+        @Override
+        public boolean verify(String hostname, SSLSession session) {
+            return "localhost".equals(hostname) /* for controllers */ || athenzVerifier.verify(hostname, session);
+        }
+    }
+
+    static class FlagsException extends RuntimeException {
+
+        private FlagsException(int statusCode, FlagsTarget target, FlagId flagId, String errorCode, String errorMessage) {
+            super(createErrorMessage(statusCode, target, flagId, errorCode, errorMessage));
+        }
+
+        private static String createErrorMessage(int statusCode, FlagsTarget target, FlagId flagId, String errorCode, String errorMessage) {
+            StringBuilder builder = new StringBuilder().append("Received ").append(statusCode);
+            if (errorCode != null) {
+                builder.append('/').append(errorCode);
+            }
+            builder.append(" from '").append(target.endpoint().getHost()).append("'");
+            if (flagId != null) {
+                builder.append("' for flag '").append(flagId).append("'");
+            }
+            return builder.append(": ").append(errorMessage).toString();
+        }
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClientException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClientException.java
new file mode 100644
index 00000000000..864ad332696
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClientException.java
@@ -0,0 +1,26 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.systemflags;
+
+import java.util.OptionalInt;
+
+/**
+ * @author bjorncs
+ */
+class FlagsClientException extends RuntimeException {
+
+    private final int responseCode;
+
+    FlagsClientException(int responseCode, String message) {
+        super(message);
+        this.responseCode = responseCode;
+    }
+
+    FlagsClientException(String message, Throwable cause) {
+        super(message, cause);
+        this.responseCode = -1;
+    }
+
+    OptionalInt responseCode() {
+        return responseCode > 0 ? OptionalInt.of(responseCode) : OptionalInt.empty();
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java
new file mode 100644
index 00000000000..ae1cb6321bd
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java
@@ -0,0 +1,200 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.systemflags;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult.WireFlagDataChange;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * @author bjorncs
+ */
+class SystemFlagsDeployResult {
+
+    private final List<FlagDataChange> flagChanges;
+
+    SystemFlagsDeployResult(List<FlagDataChange> flagChanges) { this.flagChanges = flagChanges; }
+
+    List<FlagDataChange> flagChanges() {
+        return flagChanges;
+    }
+
+    static SystemFlagsDeployResult merge(List<SystemFlagsDeployResult> results) {
+        Map<FlagDataOperation, Set<FlagsTarget>> targetsForOperation = new HashMap<>();
+
+        for (SystemFlagsDeployResult result : results) {
+            for (FlagDataChange change : result.flagChanges()) {
+                FlagDataOperation operation = new FlagDataOperation(change);
+                targetsForOperation.computeIfAbsent(operation, k -> new HashSet<>())
+                        .addAll(change.targets());
+            }
+        }
+
+        List<FlagDataChange> mergedResult = new ArrayList<>();
+        targetsForOperation.forEach(
+                (operation, targets) -> mergedResult.add(operation.toFlagDataChange(targets)));
+        return new SystemFlagsDeployResult(mergedResult);
+    }
+
+    WireSystemFlagsDeployResult toWire() {
+        var wireResult = new WireSystemFlagsDeployResult();
+        wireResult.changes = new ArrayList<>();
+        for (FlagDataChange change : flagChanges) {
+            var wireChange = new WireFlagDataChange();
+            wireChange.flagId = change.flagId().toString();
+            wireChange.operation = change.operation().asString();
+            wireChange.targets = change.targets().stream().map(FlagsTarget::asString).collect(toList());
+            wireChange.data = change.data().map(FlagData::toWire).orElse(null);
+            wireChange.previousData = change.previousData().map(FlagData::toWire).orElse(null);
+        }
+        return wireResult;
+    }
+
+    static class FlagDataChange {
+
+        private final FlagId flagId;
+        private final Set<FlagsTarget> targets;
+        private final OperationType operationType;
+        private final FlagData data;
+        private final FlagData previousData;
+
+        private FlagDataChange(
+                FlagId flagId, Set<FlagsTarget> targets, OperationType operationType, FlagData data, FlagData previousData) {
+            this.flagId = flagId;
+            this.targets = targets;
+            this.operationType = operationType;
+            this.data = data;
+            this.previousData = previousData;
+        }
+
+        static FlagDataChange created(FlagId flagId, Set<FlagsTarget> targets, FlagData data) {
+            return new FlagDataChange(flagId, targets, OperationType.CREATE, data, null);
+        }
+
+        static FlagDataChange deleted(FlagId flagId, Set<FlagsTarget> targets) {
+            return new FlagDataChange(flagId, targets, OperationType.DELETE, null, null);
+        }
+
+        static FlagDataChange updated(FlagId flagId, Set<FlagsTarget> targets, FlagData data, FlagData previousData) {
+            return new FlagDataChange(flagId, targets, OperationType.UPDATE, data, previousData);
+        }
+
+        FlagId flagId() {
+            return flagId;
+        }
+
+        Set<FlagsTarget> targets() {
+            return targets;
+        }
+
+        OperationType operation() {
+            return operationType;
+        }
+
+        Optional<FlagData> data() {
+            return Optional.ofNullable(data);
+        }
+
+        Optional<FlagData> previousData() {
+            return Optional.ofNullable(previousData);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            FlagDataChange that = (FlagDataChange) o;
+            return Objects.equals(flagId, that.flagId) &&
+                    Objects.equals(targets, that.targets) &&
+                    operationType == that.operationType &&
+                    Objects.equals(data, that.data) &&
+                    Objects.equals(previousData, that.previousData);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(flagId, targets, operationType, data, previousData);
+        }
+
+        @Override
+        public String toString() {
+            return "FlagDataChange{" +
+                    "flagId=" + flagId +
+                    ", targets=" + targets +
+                    ", operationType=" + operationType +
+                    ", data=" + data +
+                    ", previousData=" + previousData +
+                    '}';
+        }
+    }
+
+    enum OperationType {
+        CREATE("create"), DELETE("delete"), UPDATE("update");
+
+        private final String stringValue;
+
+        OperationType(String stringValue) { this.stringValue = stringValue; }
+
+        String asString() { return stringValue; }
+
+        static OperationType fromString(String stringValue) {
+            return Arrays.stream(values())
+                    .filter(v -> v.stringValue.equals(stringValue))
+                    .findAny()
+                    .orElseThrow(() -> new IllegalArgumentException("Unknown string value: " + stringValue));
+        }
+    }
+
+    private static class FlagDataOperation {
+        final FlagId flagId;
+        final OperationType operationType;
+        final FlagData data;
+        final FlagData previousData;
+        final JsonNode jsonData; // needed for FlagData equality check
+        final JsonNode jsonPreviousData; // needed for FlagData equality check
+
+
+        FlagDataOperation(FlagDataChange change) {
+            this.flagId = change.flagId();
+            this.operationType = change.operation();
+            this.data = change.data().orElse(null);
+            this.previousData = change.previousData().orElse(null);
+            this.jsonData = Optional.ofNullable(data).map(FlagData::toJsonNode).orElse(null);
+            this.jsonPreviousData = Optional.ofNullable(previousData).map(FlagData::toJsonNode).orElse(null);
+        }
+
+        FlagDataChange toFlagDataChange(Set<FlagsTarget> targets) {
+            return new FlagDataChange(flagId, targets, operationType, data, previousData);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            FlagDataOperation that = (FlagDataOperation) o;
+            return Objects.equals(flagId, that.flagId) &&
+                    operationType == that.operationType &&
+                    Objects.equals(jsonData, that.jsonData) &&
+                    Objects.equals(jsonPreviousData, that.jsonPreviousData);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(flagId, operationType, jsonData, jsonPreviousData);
+        }
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java
new file mode 100644
index 00000000000..2e783d5fcb3
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java
@@ -0,0 +1,106 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.systemflags;
+
+import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.FlagDataChange;
+
+/**
+ * Deploy a flags data archive to all targets in a given system
+ *
+ * @author bjorncs
+ */
+class SystemFlagsDeployer  {
+
+    private final FlagsClient client;
+    private final Set<FlagsTarget> targets;
+    private final ExecutorCompletionService<SystemFlagsDeployResult> completionService =
+            new ExecutorCompletionService<>(Executors.newCachedThreadPool(new DaemonThreadFactory("system-flags-deployer-")));
+
+
+    SystemFlagsDeployer(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) {
+        this(new FlagsClient(identityProvider, targets), targets);
+    }
+
+    SystemFlagsDeployer(FlagsClient client, Set<FlagsTarget> targets) {
+        this.client = client;
+        this.targets = targets;
+    }
+
+    SystemFlagsDeployResult deployFlags(SystemFlagsDataArchive archive, boolean dryRun) {
+        for (FlagsTarget target : targets) {
+            completionService.submit(() -> deployFlags(target, archive.flagData(target), dryRun));
+        }
+        List<SystemFlagsDeployResult> results = new ArrayList<>();
+        Future<SystemFlagsDeployResult> future;
+        try {
+            while (results.size() < targets.size() && (future = completionService.take()) != null) {
+                try {
+                    results.add(future.get());
+                } catch (ExecutionException e) {
+                    // TODO Handle errors
+                    throw new RuntimeException(e);
+                }
+            }
+            return SystemFlagsDeployResult.merge(results);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    // TODO Handle http status code 4xx/5xx (e.g for unknown flag id)
+    private SystemFlagsDeployResult deployFlags(FlagsTarget target, Set<FlagData> flagData, boolean dryRun) {
+        Map<FlagId, FlagData> wantedFlagData = lookupTable(flagData);
+        Map<FlagId, FlagData> currentFlagData = lookupTable(client.listFlagData(target));
+
+        List<FlagDataChange> result = new ArrayList<>();
+
+        wantedFlagData.forEach((id, data) -> {
+            if (currentFlagData.containsKey(id)) {
+                FlagData currentData = currentFlagData.get(id);
+                if (currentData.toJsonNode().equals(data.toJsonNode())) {
+                    return; // noop
+                }
+                result.add(FlagDataChange.updated(id, Set.of(target), data, currentData));
+            } else {
+                result.add(FlagDataChange.created(id, Set.of(target), data));
+            }
+            if (!dryRun) {
+                client.putFlagData(target, data);
+            }
+        });
+
+        currentFlagData.forEach((id, data) -> {
+            if (!wantedFlagData.containsKey(id)) {
+                if (!dryRun) {
+                    client.deleteFlagData(target, id);
+                }
+                result.add(FlagDataChange.deleted(id, Set.of(target)));
+            }
+        });
+
+        return new SystemFlagsDeployResult(result);
+    }
+
+    private static Map<FlagId, FlagData> lookupTable(Collection<FlagData> data) {
+        return data.stream().collect(Collectors.toMap(FlagData::id, Function.identity()));
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java
new file mode 100644
index 00000000000..08bb7628080
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java
@@ -0,0 +1,68 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.systemflags;
+
+import com.google.inject.Inject;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.restapi.ErrorResponse;
+import com.yahoo.restapi.JacksonJsonResponse;
+import com.yahoo.restapi.Path;
+import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive;
+
+import java.util.concurrent.Executor;
+
+/**
+ * Handler implementation for '/system-flags/v1', an API for controlling system-wide feature flags
+ *
+ * @author bjorncs
+ */
+@SuppressWarnings("unused") // Request handler listed in controller's services.xml
+class SystemFlagsHandler extends LoggingRequestHandler {
+
+    private static final String API_PREFIX = "/system-flags/v1";
+
+    private final SystemFlagsDeployer deployer;
+
+    @Inject
+    public SystemFlagsHandler(ZoneRegistry zoneRegistry,
+                              ServiceIdentityProvider identityProvider,
+                              Executor executor,
+                              AccessLog accessLog) {
+        super(executor, accessLog);
+        this.deployer = new SystemFlagsDeployer(identityProvider, FlagsTarget.getAllTargetsInSystem(zoneRegistry));
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        switch (request.getMethod()) {
+            case PUT:
+                return put(request);
+            default:
+                return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported");
+        }
+    }
+
+    private HttpResponse put(HttpRequest request) {
+        Path path = new Path(request.getUri());
+        if (path.matches(API_PREFIX + "/deploy")) return deploy(request, /*dryRun*/false);
+        if (path.matches(API_PREFIX + "/dryrun")) return deploy(request, /*dryRun*/true);
+        return ErrorResponse.notFoundError("Nothing at " + path);
+    }
+
+    private HttpResponse deploy(HttpRequest request, boolean dryRun) {
+        // TODO Error handling
+        String contentType = request.getHeader("Content-Type");
+        if (!contentType.equalsIgnoreCase("application/zip")) {
+            return ErrorResponse.badRequest("Invalid content type: " + contentType);
+        }
+        SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromZip(request.getData());
+        SystemFlagsDeployResult result = deployer.deployFlags(archive, dryRun);
+        return new JacksonJsonResponse<>(200, result.toWire());
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java
new file mode 100644
index 00000000000..42e33bc2f8f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java
@@ -0,0 +1,76 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.systemflags;
+
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import static com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.FlagDataChange;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author bjorncs
+ */
+public class SystemFlagsDeployerTest {
+
+    private static final SystemName SYSTEM = SystemName.main;
+
+    @Test
+    public void deploys_flag_data_to_targets() throws IOException {
+        ZoneApiMock prodUsWest1Zone = ZoneApiMock.fromId("prod.us-west-1");
+        ZoneApiMock prodUsEast3Zone = ZoneApiMock.fromId("prod.us-east-3");
+        ZoneRegistryMock registry = new ZoneRegistryMock(SYSTEM).setZones(prodUsWest1Zone, prodUsEast3Zone);
+
+        FlagsTarget controllerTarget = FlagsTarget.forController(SYSTEM);
+        FlagsTarget prodUsWest1Target = FlagsTarget.forConfigServer(registry, prodUsWest1Zone.getId());
+        FlagsTarget prodUsEast3Target = FlagsTarget.forConfigServer(registry, prodUsEast3Zone.getId());
+
+        FlagsClient flagsClient = mock(FlagsClient.class);
+        when(flagsClient.listFlagData(controllerTarget)).thenReturn(List.of());
+        when(flagsClient.listFlagData(prodUsWest1Target)).thenReturn(List.of(flagData("existing-prod.us-west-1.json")));
+        FlagData existingProdUsEast3Data = flagData("existing-prod.us-east-3.json");
+        when(flagsClient.listFlagData(prodUsEast3Target)).thenReturn(List.of(existingProdUsEast3Data));
+
+        FlagData defaultData = flagData("flags/my-flag/main.json");
+        FlagData prodUsEast3Data = flagData("flags/my-flag/main.prod.us-east-3.json");
+        SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder()
+                .addFile("main.json", defaultData)
+                .addFile("main.prod.us-east-3.json", prodUsEast3Data)
+                .build();
+
+        SystemFlagsDeployer deployer =
+                new SystemFlagsDeployer(flagsClient, Set.of(controllerTarget, prodUsWest1Target, prodUsEast3Target));
+
+
+        SystemFlagsDeployResult result = deployer.deployFlags(archive, false);
+
+        verify(flagsClient).putFlagData(controllerTarget, defaultData);
+        verify(flagsClient).putFlagData(prodUsEast3Target, prodUsEast3Data);
+        verify(flagsClient, never()).putFlagData(prodUsWest1Target, defaultData);
+        List<FlagDataChange> changes = result.flagChanges();
+        FlagId flagId = new FlagId("my-flag");
+        assertThat(changes).containsOnly(
+                FlagDataChange.created(flagId, Set.of(controllerTarget), defaultData),
+                FlagDataChange.updated(flagId, Set.of(prodUsEast3Target), prodUsEast3Data, existingProdUsEast3Data));
+
+    }
+
+    private static FlagData flagData(String filename) throws IOException {
+        return FlagData.deserializeUtf8Json(
+                SystemFlagsDeployerTest.class.getResourceAsStream("/system-flags/" + filename).readAllBytes());
+    }
+
+}
\ No newline at end of file
diff --git a/controller-server/src/test/resources/system-flags/existing-prod.us-east-3.json b/controller-server/src/test/resources/system-flags/existing-prod.us-east-3.json
new file mode 100644
index 00000000000..8db6c423e4d
--- /dev/null
+++ b/controller-server/src/test/resources/system-flags/existing-prod.us-east-3.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-flag",
+  "rules" : [
+    {
+      "value" : "prod.us-east-3.original"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-server/src/test/resources/system-flags/existing-prod.us-west-1.json b/controller-server/src/test/resources/system-flags/existing-prod.us-west-1.json
new file mode 100644
index 00000000000..70fa0624a21
--- /dev/null
+++ b/controller-server/src/test/resources/system-flags/existing-prod.us-west-1.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-flag",
+  "rules" : [
+    {
+      "value" : "default"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-server/src/test/resources/system-flags/flags/my-flag/main.json b/controller-server/src/test/resources/system-flags/flags/my-flag/main.json
new file mode 100644
index 00000000000..70fa0624a21
--- /dev/null
+++ b/controller-server/src/test/resources/system-flags/flags/my-flag/main.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-flag",
+  "rules" : [
+    {
+      "value" : "default"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-server/src/test/resources/system-flags/flags/my-flag/main.prod.us-east-3.json b/controller-server/src/test/resources/system-flags/flags/my-flag/main.prod.us-east-3.json
new file mode 100644
index 00000000000..e7e8a318a6f
--- /dev/null
+++ b/controller-server/src/test/resources/system-flags/flags/my-flag/main.prod.us-east-3.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-flag",
+  "rules" : [
+    {
+      "value" : "us-east-3"
+    }
+  ]
+}
\ No newline at end of file