Flag repository cfg client and flag directory

 - Makes new FlagRepository config server client to retrieve all flag data.
 - Makes WireFlagDataList to be used for creating the HTTP response in the
   config server, and parse the HTTP response in host admin.
 - Fixes problem with URL generation for controller: when port is not present
   in the request, remove ":-1" port specification in the url.
 - Makes a new FlagDirectory class, responsible for reading flags from
   /opt/vespa/var/vespa/flags (in FlagData JSON format), and reversely,
   sync that directory to exactly match a set of FlagData.
 - No longer have 'State state()' as a default method in interface.

7 files changed, 285 insertions, 3 deletions

diff --git a/flags/pom.xml b/flags/pom.xml
index 99fde3faebb..fc38676ff20 100644
--- a/flags/pom.xml
+++ b/flags/pom.xml
@@ -38,6 +38,12 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespalog</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
             <scope>provided</scope>
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java
index f4e23144449..1a31ecd713e 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java
@@ -1,7 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.flags;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.inject.Inject;
 import com.yahoo.vespa.flags.json.FlagData;
 import com.yahoo.vespa.flags.json.Rule;
@@ -21,8 +20,6 @@ import java.util.Optional;
  * @author hakonhall
  */
 public class FileFlagSource implements FlagSource {
-    private static final ObjectMapper mapper = new ObjectMapper();
-
     static final String FLAGS_DIRECTORY = "/etc/vespa/flags";
 
     private final Path flagsDirectory;
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java
index da8e6b29cab..182ab85858c 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java
@@ -4,8 +4,11 @@ package com.yahoo.vespa.flags;
 import java.util.Optional;
 
 /**
+ * A source of raw flag values that can be converted to typed flag values elsewhere.
+ *
  * @author hakonhall
  */
 public interface FlagSource {
+    /** Get raw flag for the given vector (specifying hostname, application id, etc). */
     Optional<RawFlag> fetch(FlagId id, FetchVector vector);
 }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java
new file mode 100644
index 00000000000..7dd61a18ad1
--- /dev/null
+++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java
@@ -0,0 +1,156 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.flags.file;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.NotDirectoryException;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Java API for a local file-based flag repository.
+ *
+ * @author hakonhall
+ */
+public class FlagDirectory {
+    private static final Logger logger = Logger.getLogger(FlagDirectory.class.getName());
+
+    private final Path flagDirectory;
+
+    public FlagDirectory() {
+        this(FileSystems.getDefault());
+    }
+
+    FlagDirectory(FileSystem fileSystem) {
+        this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flags"));
+    }
+
+    public FlagDirectory(Path flagDirectory) {
+        this.flagDirectory = flagDirectory;
+    }
+
+    public Path getPath() {
+        return flagDirectory;
+    }
+
+    public Map<FlagId, FlagData> read() {
+        return getAllRegularFilesStream()
+                .map(path -> {
+                    FlagId flagId = new FlagId(getFilenameOf(path));
+                    Optional<FlagData> flagData = readFlagData(flagId);
+                    if (!flagData.isPresent()) return null;
+                    if (!Objects.equals(flagData.get().id(), flagId)) {
+                        logger.log(LogLevel.WARNING, "Flag file " + path + " contains conflicting id " +
+                                flagData.get().id() + ", ignoring flag");
+                        return null;
+                    }
+                    return flagData.get();
+                })
+                .filter(Objects::nonNull)
+                .collect(Collectors.toMap(FlagData::id, Function.identity()));
+    }
+
+    public Optional<FlagData> readFlagData(FlagId flagId) {
+        return readUtf8File(getPathFor(flagId)).map(FlagData::deserialize);
+    }
+
+    /**
+     * Modify the flag directory as necessary, such that a later {@link #read()} will return {@code flagData}.
+     *
+     * @return true if any modifications were done.
+     */
+    public boolean sync(Map<FlagId, FlagData> flagData) {
+        boolean modified = false;
+
+        Set<Path> pathsToDelete = getAllRegularFilesStream().collect(Collectors.toCollection(HashSet::new));
+
+        uncheck(() -> Files.createDirectories(flagDirectory));
+        for (Map.Entry<FlagId, FlagData> entry : flagData.entrySet()) {
+            FlagId flagId = entry.getKey();
+            FlagData data = entry.getValue();
+            Path path = getPathFor(flagId);
+
+            pathsToDelete.remove(path);
+
+            String serializedData = data.serializeToJson();
+            Optional<String> fileContent = readUtf8File(path);
+            if (fileContent.isPresent()) {
+                if (!Objects.equals(fileContent.get(), serializedData)) {
+                    logger.log(LogLevel.INFO, "Updating flag " + flagId + " from " + fileContent.get() +
+                            " to " + serializedData);
+                    writeUtf8File(path, serializedData);
+                    modified = true;
+                }
+            } else {
+                logger.log(LogLevel.INFO, "New flag " + flagId + ": " + serializedData);
+                writeUtf8File(path, serializedData);
+                modified = true;
+            }
+        }
+
+        for (Path path : pathsToDelete) {
+            logger.log(LogLevel.INFO, "Removing flag file " + path);
+            uncheck(() -> Files.deleteIfExists(path));
+            modified = true;
+        }
+
+        return modified;
+    }
+
+    private Stream<Path> getAllRegularFilesStream() {
+        try {
+            return Files.list(flagDirectory).filter(Files::isRegularFile);
+        } catch (NotDirectoryException | NoSuchFileException e) {
+            return Stream.empty();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static String getFilenameOf(Path path) {
+        return path.getName(path.getNameCount() - 1).toString();
+    }
+
+    private Path getPathFor(FlagId flagId) {
+        return flagDirectory.resolve(flagId.toString());
+    }
+
+    private Optional<String> readUtf8File(Path path) {
+        try {
+            return Optional.of(new String(Files.readAllBytes(path), StandardCharsets.UTF_8));
+        } catch (NoSuchFileException e) {
+            return Optional.empty();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private void writeUtf8File(Path path, String content) {
+        uncheck(() -> Files.write(path, content.getBytes(StandardCharsets.UTF_8)));
+    }
+
+    @Override
+    public String toString() {
+        return "FlagDirectory{" + flagDirectory + '}';
+    }
+}
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java
new file mode 100644
index 00000000000..27ad44f938e
--- /dev/null
+++ b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.flags.file;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java
new file mode 100644
index 00000000000..09a7f635dd4
--- /dev/null
+++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java
@@ -0,0 +1,29 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.flags.json.wire;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author hakonhall
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class WireFlagDataList {
+    @JsonProperty("flags")
+    public List<WireFlagData> flags = new ArrayList<>();
+
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    public void serializeToOutputStream(OutputStream outputStream) {
+        uncheck(() -> mapper.writeValue(outputStream, this));
+    }
+}
diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java
new file mode 100644
index 00000000000..fec98fab164
--- /dev/null
+++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java
@@ -0,0 +1,86 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.flags.file;
+
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.FlagData;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.hamcrest.collection.IsMapContaining;
+import org.hamcrest.collection.IsMapWithSize;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.Map;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * @author hakonhall
+ */
+public class FlagDirectoryTest {
+    private final FileSystem fileSystem = TestFileSystem.create();
+    private final FlagDirectory flagDirectory = new FlagDirectory(fileSystem);
+
+    @Test
+    public void testReadingOnly() {
+        Map<FlagId, FlagData> data = flagDirectory.read();
+        assertThat(data, IsMapWithSize.anEmptyMap());
+
+        FlagId id1 = new FlagId("id1");
+        String json1 = "{\"id\":\"id1\"}";
+        writeUtf8FlagFile(id1.toString(), json1);
+        data = flagDirectory.read();
+        assertThat(data, IsMapWithSize.aMapWithSize(1));
+        assertThat(data, IsMapContaining.hasKey(id1));
+        assertThat(data.get(id1).id(), equalTo(id1));
+        assertThat(data.get(id1).serializeToJson(), equalTo(json1));
+    }
+
+    @Test
+    public void testSync() {
+        Map<FlagId, FlagData> dataMap = new HashMap<>();
+        FlagId id1 = new FlagId("id1");
+        FlagData data1 = new FlagData(id1, new FetchVector());
+        dataMap.put(id1, data1);
+        FlagId id2 = new FlagId("id2");
+        FlagData data2 = new FlagData(id2, new FetchVector());
+        dataMap.put(id2, data2);
+
+        // Non-existing directory => empty map
+        assertThat(flagDirectory.read(), IsMapWithSize.anEmptyMap());
+
+        // sync() will create directory with map content
+        assertThat(flagDirectory.sync(dataMap), equalTo(true));
+        Map<FlagId, FlagData> readDataMap = flagDirectory.read();
+        assertThat(readDataMap, IsMapWithSize.aMapWithSize(2));
+        assertThat(readDataMap, IsMapContaining.hasKey(id1));
+        assertThat(readDataMap, IsMapContaining.hasKey(id2));
+
+        // another sync with the same data is a no-op
+        assertThat(flagDirectory.sync(dataMap), equalTo(false));
+
+        // Changing value of id1, removing id2, adding id3
+        dataMap.remove(id2);
+        FlagData newData1 = new FlagData(id1, new FetchVector().with(FetchVector.Dimension.HOSTNAME, "h1"));
+        dataMap.put(id1, newData1);
+        FlagId id3 = new FlagId("id3");
+        FlagData data3 = new FlagData(id3, new FetchVector());
+        dataMap.put(id3, data3);
+        assertThat(flagDirectory.sync(dataMap), equalTo(true));
+        Map<FlagId, FlagData> anotherReadDataMap = flagDirectory.read();
+        assertThat(anotherReadDataMap, IsMapWithSize.aMapWithSize(2));
+        assertThat(anotherReadDataMap, IsMapContaining.hasKey(id1));
+        assertThat(anotherReadDataMap, IsMapContaining.hasKey(id3));
+        assertThat(anotherReadDataMap.get(id1).serializeToJson(), equalTo("{\"id\":\"id1\",\"attributes\":{\"hostname\":\"h1\"}}"));
+    }
+
+    private void writeUtf8FlagFile(String flagIdAkaFilename, String content) {
+        uncheck(() -> Files.createDirectories(flagDirectory.getPath()));
+        uncheck(() -> Files.write(flagDirectory.getPath().resolve(flagIdAkaFilename), content.getBytes(StandardCharsets.UTF_8)));
+    }
+}
\ No newline at end of file