Support feature flags in node repository

This implements feature flags for the node repository. A feature flag can be
toggled on/off for the following dimensions:

1) The node repository (entire zone)
2) A specific node
3) A specific application

Flags must be declared in the `FlagId` enum, this is typically done when
implementing the feature that should be guarded by a flag.

Flag status is stored in ZooKeeper. Inspecting and toggling flag status is done
through a REST API, see `RestApiTest#test_flags()`.

14 files changed, 589 insertions, 20 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index a9ae0c34028..8710b47b914 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -15,6 +15,7 @@ import com.yahoo.config.provisioning.NodeRepositoryConfig;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.hosted.provision.flag.Flags;
 import com.yahoo.vespa.hosted.provision.maintenance.PeriodicApplicationMaintainer;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.NodeAcl;
@@ -80,6 +81,7 @@ public class NodeRepository extends AbstractComponent {
     private final NameResolver nameResolver;
     private final DockerImage dockerImage;
     private final OsVersions osVersions;
+    private final Flags flags;
 
     /**
      * Creates a node repository from a zookeeper provider.
@@ -103,6 +105,7 @@ public class NodeRepository extends AbstractComponent {
         this.nameResolver = nameResolver;
         this.dockerImage = dockerImage;
         this.osVersions = new OsVersions(this.db);
+        this.flags = new Flags(this.db);
 
         // read and write all nodes to make sure they are stored in the latest version of the serialized format
         for (Node.State state : Node.State.values())
@@ -121,6 +124,11 @@ public class NodeRepository extends AbstractComponent {
     /** Returns the OS versions to use for nodes in this */
     public OsVersions osVersions() { return osVersions; }
 
+    /** Returns feature flags of this node repository */
+    public Flags flags() {
+        return flags;
+    }
+
     // ---------------- Query API ----------------------------------------------------------------
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java
new file mode 100644
index 00000000000..635b1b012da
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java
@@ -0,0 +1,120 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.flag;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Represents a feature flag and its status. Use {@link Flags#get(FlagId)} to lookup status for a specific flag.
+ *
+ * @author mpolden
+ */
+public class Flag {
+
+    private final FlagId id;
+    private final boolean enabled;
+    private final Set<String> hostnames;
+    private final Set<ApplicationId> applications;
+
+    public Flag(FlagId id, boolean enabled, Set<String> hostnames, Set<ApplicationId> applications) {
+        this.id = Objects.requireNonNull(id, "id must be non-null");
+        this.enabled = enabled;
+        this.hostnames = ImmutableSet.copyOf(Objects.requireNonNull(hostnames, "hostnames must be non-null"));
+        this.applications = ImmutableSet.copyOf(Objects.requireNonNull(applications, "applications must be non-null"));
+    }
+
+    public FlagId id() {
+        return id;
+    }
+
+    /** The hostnames this flag should apply to */
+    public Set<String> hostnames() {
+        return hostnames;
+    }
+
+    /** The applications this flag should apply to */
+    public Set<ApplicationId> applications() {
+        return applications;
+    }
+
+    /**
+     * Returns whether this flag is enabled for all dimensions. Note: More specific dimensions always return true when
+     * this is true
+     */
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    /** Returns whether this flag is enabled for given hostname */
+    public boolean isEnabled(HostName hostname) {
+        return enabled || hostnames.contains(hostname.value());
+    }
+
+    /** Returns whether this flag is enabled for given application */
+    public boolean isEnabled(ApplicationId application) {
+        return enabled || applications.contains(application);
+    }
+
+    /** Returns a copy of this with this flag enabled for all dimensions */
+    public Flag enable() {
+        return new Flag(id, true, hostnames, applications);
+    }
+
+    /** Returns a copy of this with flag enabled for given hostname */
+    public Flag enable(HostName hostname) {
+        Set<String> hostnames = new LinkedHashSet<>(this.hostnames);
+        hostnames.add(hostname.value());
+        return new Flag(id, enabled, hostnames, applications);
+    }
+
+    /** Returns a copy of this with flag enabled for given application */
+    public Flag enable(ApplicationId application) {
+        Set<ApplicationId> applications = new LinkedHashSet<>(this.applications);
+        applications.add(application);
+        return new Flag(id, enabled, hostnames, applications);
+    }
+
+    /** Returns a copy of this with flag disabled for given hostname */
+    public Flag disable(HostName hostname) {
+        Set<String> hostnames = new LinkedHashSet<>(this.hostnames);
+        hostnames.remove(hostname.value());
+        return new Flag(id, enabled, hostnames, applications);
+    }
+
+    /** Returns a copy of this with flag disabled for given application */
+    public Flag disable(ApplicationId application) {
+        Set<ApplicationId> applications = new LinkedHashSet<>(this.applications);
+        applications.remove(application);
+        return new Flag(id, enabled, hostnames, applications);
+    }
+
+    /** Returns a copy of this with this flag disabled in all dimensions */
+    public Flag disable() {
+        return new Flag(id, false, hostnames, applications);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Flag flag = (Flag) o;
+        return id == flag.id;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
+    /** Create a flag for given feature that is disabled for all dimensions */
+    public static Flag disabled(FlagId id) {
+        return new Flag(id, false, Collections.emptySet(), Collections.emptySet());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java
new file mode 100644
index 00000000000..1b798c14588
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java
@@ -0,0 +1,34 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.flag;
+
+import java.util.Arrays;
+
+/**
+ * Features of this node repository that can be toggled.
+ *
+ * @author mpolden
+ */
+public enum FlagId {
+
+    /** Indicates whether a exclusive load balancer should be provisioned */
+    exclusiveLoadBalancer("exclusive-load-balancer");
+
+    private final String serializedValue;
+
+    FlagId(String serializedValue) {
+        this.serializedValue = serializedValue;
+    }
+
+    public String serializedValue() {
+        return serializedValue;
+    }
+
+    public static FlagId fromSerializedForm(String value) {
+        return Arrays.stream(FlagId.values())
+                     .filter(f -> f.serializedValue().equals(value))
+                     .findFirst()
+                     .orElseThrow(() -> new IllegalArgumentException("Could not find flag ID by serialized value '" +
+                                                                     value + "'"));
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java
new file mode 100644
index 00000000000..31344d24c9e
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java
@@ -0,0 +1,82 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.flag;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * This class provides feature flags for the node repository. A feature flag can be toggled for the following
+ * dimensions:
+ *
+ * 1) The node repository (entire zone)
+ * 2) A specific node
+ * 3) A specific application
+ *
+ * Code which needs to consider feature flags can access them through {@link NodeRepository#flags()}.
+ *
+ * @author mpolden
+ */
+public class Flags {
+
+    private final CuratorDatabaseClient db;
+
+    public Flags(CuratorDatabaseClient db) {
+        this.db = Objects.requireNonNull(db, "db must be non-null");
+    }
+
+    /** Get status for given feature flag */
+    public Flag get(FlagId id) {
+        return db.readFlag(id).orElseGet(() -> Flag.disabled(id));
+    }
+
+    /** Get all known feature flags */
+    public List<Flag> list() {
+        return Arrays.stream(FlagId.values())
+                     .map(this::get)
+                     .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+    }
+
+    /** Enable feature flag in this node repository */
+    public void setEnabled(FlagId flag, boolean enabled) {
+        if (enabled) {
+            write(flag, Flag::enable);
+        } else {
+            write(flag, Flag::disable);
+        }
+    }
+
+    /** Enable feature flag for given application */
+    public void setEnabled(FlagId flag, ApplicationId application, boolean enabled) {
+        if (enabled) {
+            write(flag, (f) -> f.enable(application));
+        } else {
+            write(flag, (f) -> f.disable(application));
+        }
+    }
+
+    /** Enable feature flag for given node */
+    public void setEnabled(FlagId flag, HostName hostname, boolean enabled) {
+        if (enabled) {
+            write(flag, (f) -> f.enable(hostname));
+        } else {
+            write(flag, (f) -> f.disable(hostname));
+        }
+    }
+
+    private void write(FlagId id, Function<Flag, Flag> updateFunc) {
+        try (Lock lock = db.lockFlags()) {
+            db.writeFlag(updateFunc.apply(get(id)));
+        }
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index d824f9fa53b..0fc5626241f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -16,6 +16,8 @@ import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.curator.transaction.CuratorOperations;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.flag.Flag;
+import com.yahoo.vespa.hosted.provision.flag.FlagId;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
 import com.yahoo.vespa.hosted.provision.node.Agent;
@@ -55,6 +57,7 @@ public class CuratorDatabaseClient {
     private static final Path root = Path.fromString("/provision/v1");
     private static final Path lockRoot = root.append("locks");
     private static final Path loadBalancersRoot = root.append("loadBalancers");
+    private static final Path flagsRoot = root.append("flags");
     private static final Duration defaultLockTimeout = Duration.ofMinutes(2);
 
     private final NodeSerializer nodeSerializer;
@@ -453,4 +456,26 @@ public class CuratorDatabaseClient {
         return loadBalancersRoot.append(id.serializedForm());
     }
 
+    public void writeFlag(Flag flag) {
+        Path path = flagPath(flag.id());
+        curatorDatabase.create(path);
+        NestedTransaction transaction = new NestedTransaction();
+        CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction);
+        curatorTransaction.add(CuratorOperations.setData(path.getAbsolute(),
+                                                         FlagSerializer.toJson(flag)));
+        transaction.commit();
+    }
+
+    public Optional<Flag> readFlag(FlagId id) {
+        return read(flagPath(id), FlagSerializer::fromJson);
+    }
+
+    public Lock lockFlags() {
+        return lock(lockRoot.append("flagsLock"), defaultLockTimeout);
+    }
+
+    private Path flagPath(FlagId id) {
+        return flagsRoot.append(id.serializedValue());
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java
new file mode 100644
index 00000000000..431aa92a513
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java
@@ -0,0 +1,65 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.provision.flag.Flag;
+import com.yahoo.vespa.hosted.provision.flag.FlagId;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * @author mpolden
+ */
+public class FlagSerializer {
+
+    private static final String featureField = "feature";
+    private static final String enabledField = "enabled";
+    private static final String hostnamesField = "hostnames";
+    private static final String applicationsField = "applications";
+
+    public static byte[] toJson(Flag flag) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+
+        root.setString(featureField, flag.id().serializedValue());
+        root.setBool(enabledField, flag.isEnabled());
+
+        Cursor nodeArray = root.setArray(hostnamesField);
+        flag.hostnames().forEach(nodeArray::addString);
+
+        Cursor applicationArray = root.setArray(applicationsField);
+        flag.applications().forEach(application -> applicationArray.addString(application.serializedForm()));
+
+        try {
+            return SlimeUtils.toJsonBytes(slime);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static Flag fromJson(byte[] data) {
+        Inspector inspect = SlimeUtils.jsonToSlime(data).get();
+
+        Set<String> hostnames = new LinkedHashSet<>();
+        inspect.field(hostnamesField).traverse((ArrayTraverser) (i, hostname) -> hostnames.add(hostname.asString()));
+
+        Set<ApplicationId> applications = new LinkedHashSet<>();
+        inspect.field(applicationsField).traverse((ArrayTraverser) (i, application) -> {
+            applications.add(ApplicationId.fromSerializedForm(application.asString()));
+        });
+
+        return new Flag(FlagId.fromSerializedForm(inspect.field(featureField).asString()),
+                        inspect.field(enabledField).asBool(),
+                        hostnames,
+                        applications);
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index f8067cb8661..09f8fc41239 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -19,6 +19,7 @@ import com.yahoo.log.LogLevel;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.flag.FlagId;
 import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
 
@@ -104,6 +105,9 @@ public class NodeRepositoryProvisioner implements Provisioner {
     public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) {
         validate(hosts);
         activator.activate(application, hosts, transaction);
+        if (nodeRepository.flags().get(FlagId.exclusiveLoadBalancer).isEnabled(application)) {
+            // TODO: Provision load balancer
+        }
     }
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java
new file mode 100644
index 00000000000..1bc016bcda2
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java
@@ -0,0 +1,51 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi.v2;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.provision.flag.Flag;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+
+/**
+ * @author mpolden
+ */
+public class FlagsResponse extends HttpResponse {
+
+    private final List<Flag> flags;
+
+    public FlagsResponse(List<Flag> flags) {
+        super(200);
+        this.flags = flags;
+    }
+
+    @Override
+    public void render(OutputStream out) throws IOException {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Cursor flagArray = root.setArray("flags");
+        flags.forEach(flag -> {
+            Cursor flagObject = flagArray.addObject();
+            flagObject.setString("id", flag.id().serializedValue());
+            flagObject.setBool("enabled", flag.isEnabled());
+            Cursor nodeArray = flagObject.setArray("enabledHostnames");
+            flag.hostnames().forEach(nodeArray::addString);
+            Cursor applicationArray = flagObject.setArray("enabledApplications");
+            flag.applications().stream()
+                .map(ApplicationId::serializedForm)
+                .forEach(applicationArray::addString);
+        });
+        new JsonFormat(true).encode(out, slime);
+    }
+
+    @Override
+    public String getContentType() {
+        return "application/json";
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index df6bd7d0f15..418a3b16e2d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -2,13 +2,16 @@
 package com.yahoo.vespa.hosted.provision.restapi.v2;
 
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.io.IOUtils;
+import com.yahoo.restapi.Path;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
@@ -16,6 +19,7 @@ import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.NoSuchNodeException;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.flag.FlagId;
 import com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
@@ -103,6 +107,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
         if (path.equals(    "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot");
         if (path.equals(    "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl());
         if (path.equals(    "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions(), nodeRepository.osVersions());
+        if (path.equals(    "/nodes/v2/flags/")) return new FlagsResponse(nodeRepository.flags().list());
         throw new NotFoundException("Nothing at path '" + path + "'");
     }
 
@@ -149,40 +154,38 @@ public class NodesApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse handlePOST(HttpRequest request) {
-        String path = request.getUri().getPath();
-        if (path.equals("/nodes/v2/command/restart")) {
+        Path path = new Path(request.getUri().getPath());
+        if (path.matches("/nodes/v2/command/restart")) {
             int restartCount = nodeRepository.restart(toNodeFilter(request)).size();
             return new MessageResponse("Scheduled restart of " + restartCount + " matching nodes");
         }
-        else if (path.equals("/nodes/v2/command/reboot")) {
+        if (path.matches("/nodes/v2/command/reboot")) {
             int rebootCount = nodeRepository.reboot(toNodeFilter(request)).size();
             return new MessageResponse("Scheduled reboot of " + rebootCount + " matching nodes");
         }
-        else if (path.equals("/nodes/v2/node")) {
+        if (path.matches("/nodes/v2/node")) {
             int addedNodes = addNodes(request.getData());
             return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state");
         }
-        else if (path.startsWith("/nodes/v2/maintenance/inactive/")) {
-            return setActive(lastElement(path), false);
-        }
-        else {
-            throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'");
-        }
+        if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), false);
+        if (path.matches("/nodes/v2/flags/{flag}")) return setFlag(path.get("flag"), true, "", "");
+        if (path.matches("/nodes/v2/flags/{flag}/{dimension}/{value}")) return setFlag(path.get("flag"), true, path.get("dimension"), path.get("value"));
+
+        throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'");
     }
 
     private HttpResponse handleDELETE(HttpRequest request) {
-        String path = request.getUri().getPath();
-        if (path.startsWith("/nodes/v2/node/")) {
-            String hostname = lastElement(path);
+        Path path = new Path(request.getUri().getPath());
+        if (path.matches("/nodes/v2/node/{hostname}")) {
+            String hostname = path.get("hostname");
             List<Node> removedNodes = nodeRepository.removeRecursively(hostname);
             return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", ")));
         }
-        else if (path.startsWith("/nodes/v2/maintenance/inactive/")) {
-            return setActive(lastElement(path), true);
-        }
-        else {
-            throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'");
-        }
+        if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), true);
+        if (path.matches("/nodes/v2/flags/{flag}")) return setFlag(path.get("flag"), false, "", "");
+        if (path.matches("/nodes/v2/flags/{flag}/{dimension}/{value}")) return setFlag(path.get("flag"), false, path.get("dimension"), path.get("value"));
+
+        throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'");
     }
 
     private Node nodeFromRequest(HttpRequest request) {
@@ -268,13 +271,31 @@ public class NodesApiHandler extends LoggingRequestHandler {
         return false;
     }
 
-    private MessageResponse setActive(String jobName, boolean active) {
+    private MessageResponse setJobActive(String jobName, boolean active) {
         if ( ! maintenance.jobControl().jobs().contains(jobName))
             throw new NotFoundException("No job named '" + jobName + "'");
         maintenance.jobControl().setActive(jobName, active);
         return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'");
     }
 
+    private HttpResponse setFlag(String flag, boolean enabled, String dimension, String value) {
+        FlagId flagId = FlagId.fromSerializedForm(flag);
+        switch (dimension) {
+            case "application":
+                nodeRepository.flags().setEnabled(flagId, ApplicationId.fromSerializedForm(value), enabled);
+                break;
+            case "node":
+                nodeRepository.flags().setEnabled(flagId, HostName.from(value), enabled);
+                break;
+            case "":
+                nodeRepository.flags().setEnabled(flagId, enabled);
+                break;
+            default: throw new IllegalArgumentException("Unknown flag dimension '" + dimension + "'");
+        }
+        return new MessageResponse((enabled ? "Enabled" : "Disabled") + " feature " + flagId +
+                                   (!value.isEmpty() ? " for " + dimension + " '" + value + "'" : ""));
+    }
+
     private MessageResponse setTargetVersions(HttpRequest request) {
         NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase());
         Inspector inspector = toSlime(request.getData()).get();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java
new file mode 100644
index 00000000000..5018b18c491
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java
@@ -0,0 +1,64 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.flag;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors;
+import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository;
+import org.junit.Test;
+
+import java.util.function.Supplier;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class FlagsTest {
+
+    @Test
+    public void test_flag_toggling() {
+        NodeRepository nodeRepository = new MockNodeRepository(new MockCurator(), new MockNodeFlavors());
+        Flags flags = nodeRepository.flags();
+        Supplier<Flag> flag = () -> flags.get(FlagId.exclusiveLoadBalancer);
+
+        // Flag is disabled by default
+        assertFalse(flag.get().isEnabled());
+
+        // Toggle flag for a node
+        {
+            HostName node1 = HostName.from("host1");
+            flags.setEnabled(FlagId.exclusiveLoadBalancer, node1, true);
+            assertTrue(flag.get().isEnabled(node1));
+            assertFalse(flag.get().isEnabled());
+            flags.setEnabled(FlagId.exclusiveLoadBalancer, node1, false);
+            assertFalse(flag.get().isEnabled(node1));
+        }
+
+        // Toggle flag for an application
+        {
+            ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default");
+            flags.setEnabled(FlagId.exclusiveLoadBalancer, app1, true);
+            assertTrue(flag.get().isEnabled(app1));
+            assertFalse(flag.get().isEnabled());
+            flags.setEnabled(FlagId.exclusiveLoadBalancer, app1, false);
+            assertFalse(flag.get().isEnabled(app1));
+        }
+
+        // Toggle flag globally
+        {
+            flags.setEnabled(FlagId.exclusiveLoadBalancer, true);
+            assertTrue(flag.get().isEnabled());
+            // Flag is implicitly enabled for all dimensions
+            assertTrue(flag.get().isEnabled(HostName.from("host1")));
+            assertTrue(flag.get().isEnabled(ApplicationId.from("tenant1", "application1", "default")));
+            flags.setEnabled(FlagId.exclusiveLoadBalancer, false);
+            assertFalse(flag.get().isEnabled());
+        }
+    }
+
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java
new file mode 100644
index 00000000000..15f2289d340
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java
@@ -0,0 +1,33 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.persistence;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.provision.flag.Flag;
+import com.yahoo.vespa.hosted.provision.flag.FlagId;
+import org.junit.Test;
+
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class FlagSerializerTest {
+
+    @Test
+    public void test_serialization() {
+        Flag flag = new Flag(FlagId.exclusiveLoadBalancer, true,
+                             ImmutableSet.of("host1", "host2"),
+                             Collections.singleton(
+                                     ApplicationId.from("tenant1", "application1", "default")
+                             ));
+        Flag serialized = FlagSerializer.fromJson(FlagSerializer.toJson(flag));
+        assertEquals(flag.id(), serialized.id());
+        assertEquals(flag.isEnabled(), serialized.isEnabled());
+        assertEquals(flag.hostnames(), serialized.hostnames());
+        assertEquals(flag.applications(), serialized.applications());
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index 2ff1e403e35..ae7f3f14975 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -659,6 +659,44 @@ public class RestApiTest {
                        "]}");
     }
 
+    @Test
+    public void test_flags() throws Exception {
+        assertFile(new Request("http://localhost:8080/nodes/v2/flags/"), "flags1.json");
+
+        // Enable flag for application
+        assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/application/foo:bar:default",
+                                   new byte[0], Request.Method.POST),
+                       "{\"message\":\"Enabled feature exclusiveLoadBalancer for application 'foo:bar:default'\"}");
+
+        // Enable flag for node
+        assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/node/host1",
+                                   new byte[0], Request.Method.POST),
+                       "{\"message\":\"Enabled feature exclusiveLoadBalancer for node 'host1'\"}");
+
+        assertFile(new Request("http://localhost:8080/nodes/v2/flags/"), "flags2.json");
+
+        // Enable flag for entire repository
+        assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer",
+                                   new byte[0], Request.Method.POST),
+                       "{\"message\":\"Enabled feature exclusiveLoadBalancer\"}");
+
+        // Disable flag for application
+        assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/application/foo:bar:default",
+                                   new byte[0], Request.Method.DELETE),
+                       "{\"message\":\"Disabled feature exclusiveLoadBalancer for application 'foo:bar:default'\"}");
+
+        // Disable flag for node
+        assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/node/host1",
+                                   new byte[0], Request.Method.DELETE),
+                       "{\"message\":\"Disabled feature exclusiveLoadBalancer for node 'host1'\"}");
+
+        // Disable flag for entire repository
+        assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer",
+                                   new byte[0], Request.Method.DELETE),
+                       "{\"message\":\"Disabled feature exclusiveLoadBalancer\"}");
+
+    }
+
     /** Tests the rendering of each node separately to make it easier to find errors */
     @Test
     public void test_single_node_rendering() throws Exception {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json
new file mode 100644
index 00000000000..8fd09b4a274
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json
@@ -0,0 +1,10 @@
+{
+  "flags": [
+    {
+      "id": "exclusive-load-balancer",
+      "enabled": false,
+      "enabledHostnames": [],
+      "enabledApplications": []
+    }
+  ]
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json
new file mode 100644
index 00000000000..78de52e4e85
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json
@@ -0,0 +1,14 @@
+{
+  "flags": [
+    {
+      "id": "exclusive-load-balancer",
+      "enabled": false,
+      "enabledHostnames": [
+        "host1"
+      ],
+      "enabledApplications": [
+        "foo:bar:default"
+      ]
+    }
+  ]
+}