Write operator actions to audit log

All mutable requests to the following paths will be written to the audit log:

* `/controller/v1/`
* `/os/v1/`
* `/zone/v2/`

Those paths cover the most important operator actions, such as:

* (De)activation of maintenance jobs on controller
* Version confidence overrides
* Scheduling OS or firmware upgrades
* Proxied calls to config servers: Feature flag changes, (de)activation of
  node repository maintenance jobs, node state changes, scheduling of node reboots etc.

We can also consider adding node (de)provisioning (`/provision/v2/`), but that
needs to be changed in internal code.

Future handlers that require audit logging can simply extend
`AuditLoggingRequestHandler`.

19 files changed, 644 insertions, 32 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 0b5cb199994..2f7ff97ce26 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerato
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.OsVersion;
@@ -80,6 +81,7 @@ public class Controller extends AbstractComponent {
     private final Chef chef;
     private final ZmsClientFacade zmsClient;
     private final Mailer mailer;
+    private final AuditLogger auditLogger;
 
     /**
      * Creates a controller 
@@ -134,6 +136,7 @@ public class Controller extends AbstractComponent {
                                                           Objects.requireNonNull(buildService, "BuildService cannot be null"),
                                                           clock);
         tenantController = new TenantController(this, curator, athenzClientFactory);
+        auditLogger = new AuditLogger(curator, clock);
 
         // Record the version of this controller
         curator().writeControllerVersion(this.hostname(), Vtag.currentVersion);
@@ -298,6 +301,10 @@ public class Controller extends AbstractComponent {
         return curator;
     }
 
+    public AuditLogger auditLogger() {
+        return auditLogger;
+    }
+
     private Set<CloudName> clouds() {
         return zoneRegistry.zones().all().ids().stream()
                            .map(ZoneId::cloud)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java
new file mode 100644
index 00000000000..8302be5c61e
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java
@@ -0,0 +1,122 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.auditlog;
+
+import com.google.common.collect.Ordering;
+import org.jetbrains.annotations.NotNull;
+
+import java.net.URI;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Predicate;
+
+/**
+ * This represents the audit log of a hosted Vespa system. The audit log contains manual actions performed through
+ * operator APIs served by the controller.
+ *
+ * @author mpolden
+ */
+public class AuditLog {
+
+    public static final AuditLog empty = new AuditLog(List.of());
+
+    private final List<Entry> entries;
+
+    /** DO NOT USE. Public for serialization purposes */
+    public AuditLog(List<Entry> entries) {
+        this.entries = Ordering.natural().immutableSortedCopy(entries);
+    }
+
+    /** Returns a new audit log without entries older than given instant */
+    public AuditLog pruneBefore(Instant instant) {
+        List<Entry> entries = new ArrayList<>(this.entries);
+        entries.removeIf(entry -> entry.at().isBefore(instant));
+        return new AuditLog(entries);
+    }
+
+    /** Returns an new audit log with given entry added */
+    public AuditLog with(Entry entry) {
+        List<Entry> entries = new ArrayList<>(this.entries);
+        entries.add(entry);
+        return new AuditLog(entries);
+    }
+
+    /** Returns all entries in this. Entries are sorted descending by their timestamp */
+    public List<Entry> entries() {
+        return entries;
+    }
+
+    /** An entry in the audit log. This describes an HTTP request */
+    public static class Entry implements Comparable<Entry> {
+
+        private final static int maxDataLength = 1024;
+        private final static Comparator<Entry> comparator = Comparator.comparing(Entry::at).reversed();
+
+        private final Instant at;
+        private final String principal;
+        private final Method method;
+        private final URI url;
+        private final Optional<String> data;
+
+        public Entry(Instant at, String principal, Method method, URI url, Optional<String> data) {
+            this.at = Objects.requireNonNull(at, "at must be non-null");
+            this.principal = Objects.requireNonNull(principal, "principal must be non-null");
+            this.method = Objects.requireNonNull(method, "method must be non-null");
+            this.url = Objects.requireNonNull(url, "url must be non-null");
+            this.data = truncateData(data);
+        }
+
+        /** Time of the request */
+        public Instant at() {
+            return at;
+        }
+
+        /** The principal performing the request */
+        public String principal() {
+            return principal;
+        }
+
+        /** Request method */
+        public Method method() {
+            return method;
+        }
+
+        /** Request URL */
+        public URI url() {
+            return url;
+        }
+
+        /** Request data. This may be truncated if request data logged in this entry was too large */
+        public Optional<String> data() {
+            return data;
+        }
+
+        @Override
+        public int compareTo(@NotNull Entry that) {
+            return comparator.compare(this, that);
+        }
+
+        /** HTTP methods that should be logged */
+        public enum Method {
+            POST,
+            PATCH,
+            DELETE
+        }
+
+        private static Optional<String> truncateData(Optional<String> data) {
+            Objects.requireNonNull(data, "data must be non-null");
+            return data.filter(Predicate.not(String::isBlank))
+                       .map(v -> {
+                           if (v.length() > maxDataLength) {
+                               return v.substring(0, maxDataLength);
+                           }
+                           return v;
+                       });
+        }
+
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java
new file mode 100644
index 00000000000..1a91fe9ada1
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java
@@ -0,0 +1,91 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.auditlog;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.security.Principal;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * This provides read and write operations for the audit log.
+ *
+ * @author mpolden
+ */
+public class AuditLogger {
+
+    /** The TTL of log entries. Entries older than this will be removed when the log is updated */
+    private static final Duration entryTtl = Duration.ofDays(14);
+
+    private final CuratorDb db;
+    private final Clock clock;
+
+    public AuditLogger(CuratorDb db, Clock clock) {
+        this.db = Objects.requireNonNull(db, "db must be non-null");
+        this.clock = Objects.requireNonNull(clock, "clock must be non-null");
+    }
+
+    /** Read the current audit log */
+    public AuditLog get() {
+        return db.readAuditLog();
+    }
+
+    /**
+     * Write a log entry for given request to the audit log.
+     *
+     * Note that data contained in the given request may be consumed. Callers should use the returned HttpRequest for
+     * further processing.
+     */
+    public HttpRequest log(HttpRequest request) {
+        Optional<AuditLog.Entry.Method> method = auditableMethod(request);
+        if (method.isEmpty()) return request; // Nothing to audit, e.g. a GET request
+
+        Principal principal = request.getJDiscRequest().getUserPrincipal();
+        if (principal == null) {
+            throw new IllegalStateException("Cannot audit " + request.getMethod() + " " + request.getUri() +
+                                            " as no principal was found in the request. This is likely caused by a " +
+                                            "misconfiguration and should not happen");
+        }
+
+        byte[] data = new byte[0];
+        try {
+            if (request.getData() != null) {
+                data = request.getData().readAllBytes();
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        Instant now = clock.instant();
+        AuditLog.Entry entry = new AuditLog.Entry(now, principal.getName(), method.get(), request.getUri(),
+                                                  Optional.of(new String(data, StandardCharsets.UTF_8)));
+        try (Lock lock = db.lockAuditLog()) {
+            AuditLog auditLog = db.readAuditLog()
+                                  .pruneBefore(now.minus(entryTtl))
+                                  .with(entry);
+            db.writeAuditLog(auditLog);
+        }
+
+        // Create a new input stream to allow callers to consume request body
+        return new HttpRequest(request.getJDiscRequest(), new ByteArrayInputStream(data), request.propertyMap());
+    }
+
+    /** Returns the auditable method of given request, if any */
+    private static Optional<AuditLog.Entry.Method> auditableMethod(HttpRequest request) {
+        try {
+            return Optional.of(AuditLog.Entry.Method.valueOf(request.getMethod().name()));
+        } catch (IllegalArgumentException e) {
+            return Optional.empty();
+        }
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java
new file mode 100644
index 00000000000..7eb38fed7ee
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggingRequestHandler.java
@@ -0,0 +1,30 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.auditlog;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+
+/**
+ * A handler that logs requests to the audit log. Handlers that need audit logging should extend this and implement
+ * {@link AuditLoggingRequestHandler#auditAndHandle(HttpRequest)}.
+ *
+ * @author mpolden
+ */
+public abstract class AuditLoggingRequestHandler extends LoggingRequestHandler {
+
+    private final AuditLogger auditLogger;
+
+    public AuditLoggingRequestHandler(Context ctx, AuditLogger auditLogger) {
+        super(ctx);
+        this.auditLogger = auditLogger;
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        return auditAndHandle(auditLogger.log(request));
+    }
+
+    public abstract HttpResponse auditAndHandle(HttpRequest request);
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/package-info.java
new file mode 100644
index 00000000000..e51cb6f7134
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author mpolden
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.auditlog;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java
new file mode 100644
index 00000000000..0b89c9158e4
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java
@@ -0,0 +1,83 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
+
+import java.net.URI;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+
+/**
+ * Slime serializer for the audit log.
+ *
+ * @author mpolden
+ */
+public class AuditLogSerializer {
+
+    private static final String entriesField = "entries";
+    private static final String atField = "at";
+    private static final String principalField = "principal";
+    private static final String methodField = "method";
+    private static final String urlField = "url";
+    private static final String dataField = "data";
+
+    public Slime toSlime(AuditLog log) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Cursor entryArray = root.setArray(entriesField);
+        log.entries().forEach(entry -> {
+            Cursor entryObject = entryArray.addObject();
+            entryObject.setLong(atField, entry.at().toEpochMilli());
+            entryObject.setString(principalField, entry.principal());
+            entryObject.setString(methodField, asString(entry.method()));
+            entryObject.setString(urlField, entry.url().toString());
+            entry.data().ifPresent(data -> entryObject.setString(dataField, data));
+        });
+        return slime;
+    }
+
+    public AuditLog fromSlime(Slime slime) {
+        List<AuditLog.Entry> entries = new ArrayList<>();
+        Cursor root = slime.get();
+        root.field(entriesField).traverse((ArrayTraverser) (i, entryObject) -> {
+            entries.add(new AuditLog.Entry(
+                    Instant.ofEpochMilli(entryObject.field(atField).asLong()),
+                    entryObject.field(principalField).asString(),
+                    methodFrom(entryObject.field(methodField)),
+                    URI.create(entryObject.field(urlField).asString()),
+                    optionalField(entryObject.field(dataField), Function.identity())
+            ));
+        });
+        return new AuditLog(entries);
+    }
+
+    private static String asString(AuditLog.Entry.Method method) {
+        switch (method) {
+            case POST: return "POST";
+            case PATCH: return "PATCH";
+            case DELETE: return "DELETE";
+            default: throw new IllegalArgumentException("No serialization defined for method " + method);
+        }
+    }
+
+    private static AuditLog.Entry.Method methodFrom(Inspector field) {
+        switch (field.asString()) {
+            case "POST": return AuditLog.Entry.Method.POST;
+            case "PATCH": return AuditLog.Entry.Method.PATCH;
+            case "DELETE": return AuditLog.Entry.Method.DELETE;
+            default: throw new IllegalArgumentException("Unknown serialized value '" + field.asString() + "'");
+        }
+    }
+
+    private static <T> Optional<T> optionalField(Inspector field, Function<String, T> fieldMapper) {
+        return Optional.of(field).filter(Inspector::valid).map(Inspector::asString).map(fieldMapper);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 0c79793e6a9..1648040fc2b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
@@ -86,6 +87,7 @@ public class CuratorDb {
     private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer();
     private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer);
     private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer();
+    private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer();
 
     private final Curator curator;
     private final Duration tryLockTimeout;
@@ -187,6 +189,11 @@ public class CuratorDb {
     public Lock lockRoutingPolicies() {
         return lock(lockRoot.append("routingPolicies"), defaultLockTimeout);
     }
+
+    public Lock lockAuditLog() {
+        return lock(lockRoot.append("auditLog"), defaultLockTimeout);
+    }
+
     // -------------- Helpers ------------------------------------------
 
     /** Try locking with a low timeout, meaning it is OK to fail lock acquisition.
@@ -429,6 +436,17 @@ public class CuratorDb {
                       .sorted();
     }
 
+    // -------------- Audit log -----------------------------------------------
+
+    public AuditLog readAuditLog() {
+        return readSlime(auditLogPath()).map(auditLogSerializer::fromSlime)
+                                        .orElse(AuditLog.empty);
+    }
+
+    public void writeAuditLog(AuditLog log) {
+        curator.set(auditLogPath(), asJson(auditLogSerializer.toSlime(log)));
+    }
+
     // -------------- Provisioning (called by internal code) ------------------
 
     @SuppressWarnings("unused")
@@ -551,6 +569,10 @@ public class CuratorDb {
         return routingPoliciesRoot.append(application.serializedForm());
     }
 
+    private static Path auditLogPath() {
+        return root.append("auditLog");
+    }
+
     private static Path provisionStatePath() {
         return root.append("provisioning").append("states");
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java
new file mode 100644
index 00000000000..f24ddbfec94
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/AuditLogResponse.java
@@ -0,0 +1,33 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.controller;
+
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
+import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
+
+/**
+ * @author mpolden
+ */
+public class AuditLogResponse extends SlimeJsonResponse {
+
+    public AuditLogResponse(AuditLog log) {
+        super(toSlime(log));
+    }
+
+    private static Slime toSlime(AuditLog log) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Cursor entryArray = root.setArray("entries");
+        log.entries().forEach(entry -> {
+            Cursor entryObject = entryArray.addObject();
+            entryObject.setString("time", entry.at().toString());
+            entryObject.setString("user", entry.principal());
+            entryObject.setString("method", entry.method().name());
+            entryObject.setString("url", entry.url().toString());
+            entry.data().ifPresent(data -> entryObject.setString("data", data));
+        });
+        return slime;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
index 5fd1ccf4f96..05fc691fefb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
@@ -6,13 +6,15 @@ import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.io.IOUtils;
+import com.yahoo.restapi.Path;
 import com.yahoo.slime.Inspector;
 import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
 import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance;
 import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
-import com.yahoo.restapi.Path;
 import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
 import com.yahoo.yolean.Exceptions;
@@ -31,17 +33,19 @@ import java.util.logging.Level;
  * @author bratseth
  */
 @SuppressWarnings("unused") // Created by injection
-public class ControllerApiHandler extends LoggingRequestHandler {
+public class ControllerApiHandler extends AuditLoggingRequestHandler {
 
     private final ControllerMaintenance maintenance;
+    private final Controller controller;
 
-    public ControllerApiHandler(LoggingRequestHandler.Context parentCtx, ControllerMaintenance maintenance) {
-        super(parentCtx);
+    public ControllerApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller, ControllerMaintenance maintenance) {
+        super(parentCtx, controller.auditLogger());
+        this.controller = controller;
         this.maintenance = maintenance;
     }
 
     @Override
-    public HttpResponse handle(HttpRequest request) {
+    public HttpResponse auditAndHandle(HttpRequest request) {
         try {
             switch (request.getMethod()) {
                 case GET: return get(request);
@@ -63,6 +67,7 @@ public class ControllerApiHandler extends LoggingRequestHandler {
     private HttpResponse get(HttpRequest request) {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/controller/v1/")) return root(request);
+        if (path.matches("/controller/v1/auditlog/")) return new AuditLogResponse(controller.auditLogger().get());
         if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl());
         if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader());
         return notFound(path);
@@ -91,7 +96,7 @@ public class ControllerApiHandler extends LoggingRequestHandler {
     private HttpResponse notFound(Path path) { return ErrorResponse.notFoundError("Nothing at " + path); }
 
     private HttpResponse root(HttpRequest request) {
-        return new ResourceResponse(request, "jobs/upgrader", "maintenance");
+        return new ResourceResponse(request, "auditlog", "jobs/upgrader", "maintenance");
     }
 
     private HttpResponse setActive(String jobName, boolean active) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java
index aa3f5f9f909..3c0f515aa8a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java
@@ -7,7 +7,6 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.io.IOUtils;
 import com.yahoo.restapi.Path;
 import com.yahoo.slime.Cursor;
@@ -17,6 +16,7 @@ import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
 import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
@@ -38,17 +38,17 @@ import java.util.logging.Level;
  * @author mpolden
  */
 @SuppressWarnings("unused") // Injected
-public class OsApiHandler extends LoggingRequestHandler {
+public class OsApiHandler extends AuditLoggingRequestHandler {
 
     private final Controller controller;
 
     public OsApiHandler(Context ctx, Controller controller) {
-        super(ctx);
+        super(ctx, controller.auditLogger());
         this.controller = controller;
     }
 
     @Override
-    public HttpResponse handle(HttpRequest request) {
+    public HttpResponse auditAndHandle(HttpRequest request) {
         try {
             switch (request.getMethod()) {
                 case GET: return get(request);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java
index 11c1e5ec6df..6f014e5661f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java
@@ -21,7 +21,7 @@ import java.util.logging.Level;
 import java.util.stream.Collectors;
 
 /**
- * REST API that provides information about zones in hosted Vespa (version 1)
+ * Read-only REST API that provides information about zones in hosted Vespa (version 1)
  *
  * @author mpolden
  */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java
index 3d86d5da262..377a57fbb91 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java
@@ -1,19 +1,21 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.zone.v2;
 
-import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.restapi.Path;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
 import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor;
 import com.yahoo.vespa.hosted.controller.proxy.ProxyException;
 import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
-import com.yahoo.restapi.Path;
 import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
 import com.yahoo.yolean.Exceptions;
 
@@ -28,20 +30,20 @@ import java.util.logging.Level;
  * @author mpolden
  */
 @SuppressWarnings("unused")
-public class ZoneApiHandler extends LoggingRequestHandler {
+public class ZoneApiHandler extends AuditLoggingRequestHandler {
 
     private final ZoneRegistry zoneRegistry;
     private final ConfigServerRestExecutor proxy;
 
     public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ZoneRegistry zoneRegistry,
-                          ConfigServerRestExecutor proxy) {
-        super(parentCtx);
+                          ConfigServerRestExecutor proxy, Controller controller) {
+        super(parentCtx, controller.auditLogger());
         this.zoneRegistry = zoneRegistry;
         this.proxy = proxy;
     }
 
     @Override
-    public HttpResponse handle(HttpRequest request) {
+    public HttpResponse auditAndHandle(HttpRequest request) {
         try {
             switch (request.getMethod()) {
                 case GET:
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggerTest.java
new file mode 100644
index 00000000000..7579b2a02a9
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggerTest.java
@@ -0,0 +1,83 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.auditlog;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLog.Entry;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.function.Supplier;
+
+import static java.time.temporal.ChronoUnit.MILLIS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class AuditLoggerTest {
+
+    private final ControllerTester tester = new ControllerTester();
+
+    @Test
+    public void test_logging() {
+        Supplier<AuditLog> log = () -> tester.controller().auditLogger().get();
+
+        { // GET request is ignored
+            HttpRequest request = testRequest(Method.GET, URI.create("http://localhost:8080/os/v1/"), "");
+            tester.controller().auditLogger().log(request);
+            assertTrue("Not logged", log.get().entries().isEmpty());
+        }
+
+        { // PATCH request is logged in audit log
+            URI url = URI.create("http://localhost:8080/os/v1/");
+            String data = "{\"cloud\":\"cloud9\",\"version\":\"42.0\"}";
+            HttpRequest request = testRequest(Method.PATCH, url, data);
+            tester.controller().auditLogger().log(request);
+
+            assertEquals(instant(), log.get().entries().get(0).at());
+            assertEquals("user", log.get().entries().get(0).principal());
+            assertEquals(Entry.Method.PATCH, log.get().entries().get(0).method());
+            assertEquals(data, log.get().entries().get(0).data().get());
+        }
+
+        { // Another PATCH request is logged
+            tester.clock().advance(Duration.ofDays(1));
+            HttpRequest request = testRequest(Method.PATCH, URI.create("http://localhost:8080/os/v1/"),
+                                              "{\"cloud\":\"cloud9\",\"version\":\"43.0\"}");
+            tester.controller().auditLogger().log(request);
+            assertEquals(2, log.get().entries().size());
+            assertEquals(instant(), log.get().entries().get(0).at());
+        }
+
+        { // 14 days pass and another PATCH request is logged. Older entries are removed due to expiry
+            tester.clock().advance(Duration.ofDays(14));
+            HttpRequest request = testRequest(Method.PATCH, URI.create("http://localhost:8080/os/v1/"),
+                                              "{\"cloud\":\"cloud9\",\"version\":\"44.0\"}");
+            tester.controller().auditLogger().log(request);
+            assertEquals(1, log.get().entries().size());
+            assertEquals(instant(), log.get().entries().get(0).at());
+        }
+    }
+
+    private Instant instant() {
+        return tester.clock().instant().truncatedTo(MILLIS);
+    }
+
+    private static HttpRequest testRequest(Method method, URI url, String data) {
+        HttpRequest request = HttpRequest.createTestRequest(
+                url.toString(),
+                method,
+                new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8))
+        );
+        request.getJDiscRequest().setUserPrincipal(() -> "user");
+        return request;
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java
new file mode 100644
index 00000000000..00ad83f0a83
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java
@@ -0,0 +1,59 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
+import org.junit.Test;
+
+import java.net.URI;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+import static java.time.temporal.ChronoUnit.MILLIS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class AuditLogSerializerTest {
+
+    @Test
+    public void test_serialization() {
+        Instant i1 = Instant.now();
+        Instant i2 = i1.minus(Duration.ofHours(1));
+        Instant i3 = i1.minus(Duration.ofHours(2));
+
+        AuditLog log = new AuditLog(List.of(
+                new AuditLog.Entry(i1, "bar", AuditLog.Entry.Method.POST,
+                                   URI.create("http://localhost/bar/baz/"),
+                                   Optional.of("0".repeat(2048))),
+                new AuditLog.Entry(i2, "foo", AuditLog.Entry.Method.POST,
+                                   URI.create("http://localhost/foo/bar/"),
+                                   Optional.of("{\"foo\":\"bar\"}")),
+                new AuditLog.Entry(i3, "baz", AuditLog.Entry.Method.POST,
+                                   URI.create("http://localhost/foo/baz/"),
+                                   Optional.of(""))
+        ));
+
+        AuditLogSerializer serializer = new AuditLogSerializer();
+        AuditLog serialized = serializer.fromSlime(serializer.toSlime(log));
+        assertEquals(log.entries().size(), serialized.entries().size());
+
+        for (int i = 0; i < log.entries().size(); i++) {
+            AuditLog.Entry entry = log.entries().get(i);
+            AuditLog.Entry serializedEntry = serialized.entries().get(i);
+
+            assertEquals(entry.at().truncatedTo(MILLIS), serializedEntry.at());
+            assertEquals(entry.principal(), serializedEntry.principal());
+            assertEquals(entry.method(), serializedEntry.method());
+            assertEquals(entry.url(), serializedEntry.url());
+            assertEquals(entry.data(), serializedEntry.data());
+        }
+
+        assertEquals(1024, log.entries().get(0).data().get().length());
+        assertTrue(log.entries().get(2).data().isEmpty());
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index a3673b0cfc9..5f0d074fdd8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -2,49 +2,63 @@
 package com.yahoo.vespa.hosted.controller.restapi.controller;
 
 import com.yahoo.application.container.handler.Request;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzUser;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+
+import static org.junit.Assert.assertFalse;
 
 /**
  * @author bratseth
  */
 public class ControllerApiTest extends ControllerContainerTest {
 
-    private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/";
+    private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/";
     private static final AthenzIdentity HOSTED_VESPA_OPERATOR = AthenzUser.fromUserId("johnoperator");
 
+    private ContainerControllerTester tester;
+
+    @Before
+    public void before() {
+        addUserToHostedOperatorRole(HOSTED_VESPA_OPERATOR);
+        tester = new ContainerControllerTester(container, responseFiles);
+    }
+
     @Test
     public void testControllerApi() {
-        ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
-
         tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/", new byte[0], Request.Method.GET), new File("root.json"));
 
         // POST deactivation of a maintenance job
-        assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
-                                            new byte[0], Request.Method.POST),
-                       200,
-                       "{\"message\":\"Deactivated job 'DeploymentExpirer'\"}");
+        tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
+                                            "", Request.Method.POST),
+                       "{\"message\":\"Deactivated job 'DeploymentExpirer'\"}", 200);
+
         // GET a list of all maintenance jobs
         tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/", new byte[0], Request.Method.GET),
                               new File("maintenance.json"));
         // DELETE deactivation of a maintenance job
-        assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
-                                            new byte[0], Request.Method.DELETE),
-                       200,
-                       "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}");
+        tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
+                                            "", Request.Method.DELETE),
+                       "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}",
+                              200);
+
+        assertFalse("Actions are logged to audit log", tester.controller().auditLogger().get().entries().isEmpty());
     }
 
     @Test
     public void testUpgraderApi() {
-        addUserToHostedOperatorRole(HOSTED_VESPA_OPERATOR);
-
-        ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
-
         // Get current configuration
         tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/jobs/upgrader", new byte[0], Request.Method.GET),
                               "{\"upgradesPerMinute\":0.125,\"confidenceOverrides\":[]}",
@@ -97,6 +111,34 @@ public class ControllerApiTest extends ControllerContainerTest {
                 hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "", Request.Method.DELETE),
                 "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.43\":\"broken\"}]}",
                 200);
+
+        assertFalse("Actions are logged to audit log", tester.controller().auditLogger().get().entries().isEmpty());
+    }
+
+    @Test
+    public void testAuditLogApi() {
+        ManualClock clock = new ManualClock(Instant.parse("2019-03-01T12:13:14.00Z"));
+        AuditLogger logger = new AuditLogger(tester.controller().curator(), clock);
+
+        // Log some operator actions
+        HttpRequest req1 = HttpRequest.createTestRequest(
+                "http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
+                com.yahoo.jdisc.http.HttpRequest.Method.POST
+        );
+        req1.getJDiscRequest().setUserPrincipal(() -> "operator1");
+        logger.log(req1);
+
+        clock.advance(Duration.ofHours(2));
+        HttpRequest req2 = HttpRequest.createTestRequest(
+                "http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42",
+                com.yahoo.jdisc.http.HttpRequest.Method.POST,
+                new ByteArrayInputStream("broken".getBytes(StandardCharsets.UTF_8))
+        );
+        req2.getJDiscRequest().setUserPrincipal(() -> "operator2");
+        logger.log(req2);
+
+        // Verify log
+        tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/auditlog/"), new File("auditlog.json"));
     }
 
     private static Request hostedOperatorRequest(String uri, String body, Request.Method method) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/auditlog.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/auditlog.json
new file mode 100644
index 00000000000..3eff75feb86
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/auditlog.json
@@ -0,0 +1,17 @@
+{
+  "entries": [
+    {
+      "time": "2019-03-01T14:13:14Z",
+      "user": "operator2",
+      "method": "POST",
+      "url": "http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42",
+      "data": "broken"
+    },
+    {
+      "time": "2019-03-01T12:13:14Z",
+      "user": "operator1",
+      "method": "POST",
+      "url": "http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json
index 33e5b40249d..0ff5a8bf443 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json
@@ -1,6 +1,9 @@
 {
   "resources": [
     {
+      "url": "http://localhost:8080/controller/v1/auditlog/"
+    },
+    {
       "url": "http://localhost:8080/controller/v1/jobs/upgrader/"
     },
     {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
index 62aa157f1ab..5ad2833456a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
@@ -29,6 +29,8 @@ import java.io.File;
 import java.time.Duration;
 import java.util.List;
 
+import static org.junit.Assert.assertFalse;
+
 /**
  * @author mpolden
  */
@@ -127,6 +129,7 @@ public class OsApiTest extends ControllerContainerTest {
         assertResponse(new Request("http://localhost:8080/os/v1/firmware/dev/", "", Request.Method.DELETE),
                        "{\"error-code\":\"NOT_FOUND\",\"message\":\"No zones at path '/os/v1/firmware/dev'\"}", 404);
 
+        assertFalse("Actions are logged to audit log", tester.controller().auditLogger().get().entries().isEmpty());
     }
 
     private void upgradeAndUpdateStatus() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
index f62f6e21910..5f9f4748676 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
@@ -107,6 +107,8 @@ public class ZoneApiTest extends ControllerContainerTest {
         assertEquals("/nodes/v2/node/node1", proxy.lastReceived().get().getConfigServerRequest());
         assertEquals("PATCH", proxy.lastReceived().get().getMethod());
         assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get());
+
+        assertFalse("Actions are logged to audit log", tester.controller().auditLogger().get().entries().isEmpty());
     }
 
     @Test