diff options
author | Håkon Hallingstad <hakon@oath.com> | 2019-01-09 17:31:34 +0100 |
---|---|---|
committer | Håkon Hallingstad <hakon@oath.com> | 2019-01-09 17:31:34 +0100 |
commit | 27ad82c5122b28e886d04904a73cd17e9bbcd105 (patch) | |
tree | 94e577720b0ac5ce7546d1667a571e2f0b036734 /flags | |
parent | 68cbb82330cb610ff3e3462a7ec704f78e49a31c (diff) |
Store flags locally in one file
Diffstat (limited to 'flags')
-rw-r--r-- | flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java | 3 | ||||
-rw-r--r-- | flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java | 111 | ||||
-rw-r--r-- | flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java | 156 | ||||
-rw-r--r-- | flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java | 19 | ||||
-rw-r--r-- | flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java | 8 | ||||
-rw-r--r-- | flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java (renamed from flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java) | 44 |
6 files changed, 156 insertions, 185 deletions
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index 581ec599aab..7d84efa52b2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -17,8 +17,7 @@ import java.util.function.Consumer; * @author hakonhall */ @Immutable -public -class FetchVector { +public class FetchVector { public enum Dimension { /** Value from ZoneId::value */ ZONE_ID, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java new file mode 100644 index 00000000000..abe8d407ab0 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java @@ -0,0 +1,111 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.file; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Java API for a flag database stored in a single file + * + * @author hakonhall + */ +public class FlagDbFile { + private static final Logger logger = Logger.getLogger(FlagDbFile.class.getName()); + + private final Path path; + + public FlagDbFile() { + this(FileSystems.getDefault()); + } + + public FlagDbFile(FileSystem fileSystem) { + this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flag.db")); + } + + public FlagDbFile(Path path) { + this.path = path; + } + + public Path getPath() { + return path; + } + + public Map<FlagId, FlagData> read() { + Optional<byte[]> bytes = readFile(); + if (!bytes.isPresent()) return Collections.emptyMap(); + return FlagData.deserializeList(bytes.get()).stream().collect(Collectors.toMap(FlagData::id, Function.identity())); + } + + public boolean sync(Map<FlagId, FlagData> flagData) { + boolean modified = false; + Map<FlagId, FlagData> currentFlagData = read(); + Set<FlagId> flagIdsToBeRemoved = new HashSet<>(currentFlagData.keySet()); + List<FlagData> flagDataList = new ArrayList<>(flagData.values()); + + for (FlagData data : flagDataList) { + flagIdsToBeRemoved.remove(data.id()); + + FlagData existingFlagData = currentFlagData.get(data.id()); + if (existingFlagData == null) { + logger.log(LogLevel.INFO, "New flag " + data.id() + ": " + data.serializeToJson()); + modified = true; + + // Could also consider testing with FlagData::equals, but that would be too fragile? + } else if (!Objects.equals(data.serializeToJson(), existingFlagData.serializeToJson())){ + logger.log(LogLevel.INFO, "Updating flag " + data.id() + " from " + + existingFlagData.serializeToJson() + " to " + data.serializeToJson()); + modified = true; + } + } + + if (!flagIdsToBeRemoved.isEmpty()) { + String flagIdsString = flagIdsToBeRemoved.stream().map(FlagId::toString).collect(Collectors.joining(", ")); + logger.log(LogLevel.INFO, "Removing flags " + flagIdsString); + modified = true; + } + + if (!modified) return false; + + writeFile(FlagData.serializeListToUtf8Json(flagDataList)); + + return modified; + } + + private Optional<byte[]> readFile() { + try { + return Optional.of(Files.readAllBytes(path)); + } catch (NoSuchFileException e) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void writeFile(byte[] bytes) { + uncheck(() -> Files.createDirectories(path.getParent())); + uncheck(() -> Files.write(path, bytes)); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java deleted file mode 100644 index 7dd61a18ad1..00000000000 --- a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags.file; - -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.flags.FlagId; -import com.yahoo.vespa.flags.json.FlagData; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.NotDirectoryException; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * Java API for a local file-based flag repository. - * - * @author hakonhall - */ -public class FlagDirectory { - private static final Logger logger = Logger.getLogger(FlagDirectory.class.getName()); - - private final Path flagDirectory; - - public FlagDirectory() { - this(FileSystems.getDefault()); - } - - FlagDirectory(FileSystem fileSystem) { - this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flags")); - } - - public FlagDirectory(Path flagDirectory) { - this.flagDirectory = flagDirectory; - } - - public Path getPath() { - return flagDirectory; - } - - public Map<FlagId, FlagData> read() { - return getAllRegularFilesStream() - .map(path -> { - FlagId flagId = new FlagId(getFilenameOf(path)); - Optional<FlagData> flagData = readFlagData(flagId); - if (!flagData.isPresent()) return null; - if (!Objects.equals(flagData.get().id(), flagId)) { - logger.log(LogLevel.WARNING, "Flag file " + path + " contains conflicting id " + - flagData.get().id() + ", ignoring flag"); - return null; - } - return flagData.get(); - }) - .filter(Objects::nonNull) - .collect(Collectors.toMap(FlagData::id, Function.identity())); - } - - public Optional<FlagData> readFlagData(FlagId flagId) { - return readUtf8File(getPathFor(flagId)).map(FlagData::deserialize); - } - - /** - * Modify the flag directory as necessary, such that a later {@link #read()} will return {@code flagData}. - * - * @return true if any modifications were done. - */ - public boolean sync(Map<FlagId, FlagData> flagData) { - boolean modified = false; - - Set<Path> pathsToDelete = getAllRegularFilesStream().collect(Collectors.toCollection(HashSet::new)); - - uncheck(() -> Files.createDirectories(flagDirectory)); - for (Map.Entry<FlagId, FlagData> entry : flagData.entrySet()) { - FlagId flagId = entry.getKey(); - FlagData data = entry.getValue(); - Path path = getPathFor(flagId); - - pathsToDelete.remove(path); - - String serializedData = data.serializeToJson(); - Optional<String> fileContent = readUtf8File(path); - if (fileContent.isPresent()) { - if (!Objects.equals(fileContent.get(), serializedData)) { - logger.log(LogLevel.INFO, "Updating flag " + flagId + " from " + fileContent.get() + - " to " + serializedData); - writeUtf8File(path, serializedData); - modified = true; - } - } else { - logger.log(LogLevel.INFO, "New flag " + flagId + ": " + serializedData); - writeUtf8File(path, serializedData); - modified = true; - } - } - - for (Path path : pathsToDelete) { - logger.log(LogLevel.INFO, "Removing flag file " + path); - uncheck(() -> Files.deleteIfExists(path)); - modified = true; - } - - return modified; - } - - private Stream<Path> getAllRegularFilesStream() { - try { - return Files.list(flagDirectory).filter(Files::isRegularFile); - } catch (NotDirectoryException | NoSuchFileException e) { - return Stream.empty(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static String getFilenameOf(Path path) { - return path.getName(path.getNameCount() - 1).toString(); - } - - private Path getPathFor(FlagId flagId) { - return flagDirectory.resolve(flagId.toString()); - } - - private Optional<String> readUtf8File(Path path) { - try { - return Optional.of(new String(Files.readAllBytes(path), StandardCharsets.UTF_8)); - } catch (NoSuchFileException e) { - return Optional.empty(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private void writeUtf8File(Path path, String content) { - uncheck(() -> Files.write(path, content.getBytes(StandardCharsets.UTF_8))); - } - - @Override - public String toString() { - return "FlagDirectory{" + flagDirectory + '}'; - } -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java index 572ec511607..64c4bbe7616 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.RawFlag; import com.yahoo.vespa.flags.json.wire.WireFlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagDataList; import com.yahoo.vespa.flags.json.wire.WireRule; import javax.annotation.concurrent.Immutable; @@ -114,6 +115,24 @@ public class FlagData { ); } + public static byte[] serializeListToUtf8Json(List<FlagData> list) { + return listToWire(list).serializeToBytes(); + } + + public static List<FlagData> deserializeList(byte[] bytes) { + return listFromWire(WireFlagDataList.deserializeFrom(bytes)); + } + + public static WireFlagDataList listToWire(List<FlagData> list) { + WireFlagDataList wireList = new WireFlagDataList(); + wireList.flags = list.stream().map(FlagData::toWire).collect(Collectors.toList()); + return wireList; + } + + public static List<FlagData> listFromWire(WireFlagDataList wireList) { + return wireList.flags.stream().map(FlagData::fromWire).collect(Collectors.toList()); + } + private static List<Rule> rulesFromWire(List<WireRule> wireRules) { if (wireRules == null) return Collections.emptyList(); return wireRules.stream().map(Rule::fromWire).collect(Collectors.toList()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java index 09a7f635dd4..60b35d9b69e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java @@ -26,4 +26,12 @@ public class WireFlagDataList { public void serializeToOutputStream(OutputStream outputStream) { uncheck(() -> mapper.writeValue(outputStream, this)); } + + public byte[] serializeToBytes() { + return uncheck(() -> mapper.writeValueAsBytes(this)); + } + + public static WireFlagDataList deserializeFrom(byte[] bytes) { + return uncheck(() -> mapper.readValue(bytes, WireFlagDataList.class)); + } } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java index fec98fab164..fd1a71e4b4a 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -22,27 +23,12 @@ import static org.hamcrest.Matchers.equalTo; /** * @author hakonhall */ -public class FlagDirectoryTest { +public class FlagDbFileTest { private final FileSystem fileSystem = TestFileSystem.create(); - private final FlagDirectory flagDirectory = new FlagDirectory(fileSystem); + private final FlagDbFile flagDb = new FlagDbFile(fileSystem); @Test - public void testReadingOnly() { - Map<FlagId, FlagData> data = flagDirectory.read(); - assertThat(data, IsMapWithSize.anEmptyMap()); - - FlagId id1 = new FlagId("id1"); - String json1 = "{\"id\":\"id1\"}"; - writeUtf8FlagFile(id1.toString(), json1); - data = flagDirectory.read(); - assertThat(data, IsMapWithSize.aMapWithSize(1)); - assertThat(data, IsMapContaining.hasKey(id1)); - assertThat(data.get(id1).id(), equalTo(id1)); - assertThat(data.get(id1).serializeToJson(), equalTo(json1)); - } - - @Test - public void testSync() { + public void test() { Map<FlagId, FlagData> dataMap = new HashMap<>(); FlagId id1 = new FlagId("id1"); FlagData data1 = new FlagData(id1, new FetchVector()); @@ -52,17 +38,19 @@ public class FlagDirectoryTest { dataMap.put(id2, data2); // Non-existing directory => empty map - assertThat(flagDirectory.read(), IsMapWithSize.anEmptyMap()); + assertThat(flagDb.read(), IsMapWithSize.anEmptyMap()); // sync() will create directory with map content - assertThat(flagDirectory.sync(dataMap), equalTo(true)); - Map<FlagId, FlagData> readDataMap = flagDirectory.read(); + assertThat(flagDb.sync(dataMap), equalTo(true)); + Map<FlagId, FlagData> readDataMap = flagDb.read(); assertThat(readDataMap, IsMapWithSize.aMapWithSize(2)); assertThat(readDataMap, IsMapContaining.hasKey(id1)); assertThat(readDataMap, IsMapContaining.hasKey(id2)); + assertThat(getDbContent(), equalTo("{\"flags\":[{\"id\":\"id1\"},{\"id\":\"id2\"}]}")); + // another sync with the same data is a no-op - assertThat(flagDirectory.sync(dataMap), equalTo(false)); + assertThat(flagDb.sync(dataMap), equalTo(false)); // Changing value of id1, removing id2, adding id3 dataMap.remove(id2); @@ -71,16 +59,18 @@ public class FlagDirectoryTest { FlagId id3 = new FlagId("id3"); FlagData data3 = new FlagData(id3, new FetchVector()); dataMap.put(id3, data3); - assertThat(flagDirectory.sync(dataMap), equalTo(true)); - Map<FlagId, FlagData> anotherReadDataMap = flagDirectory.read(); + assertThat(flagDb.sync(dataMap), equalTo(true)); + Map<FlagId, FlagData> anotherReadDataMap = flagDb.read(); assertThat(anotherReadDataMap, IsMapWithSize.aMapWithSize(2)); assertThat(anotherReadDataMap, IsMapContaining.hasKey(id1)); assertThat(anotherReadDataMap, IsMapContaining.hasKey(id3)); assertThat(anotherReadDataMap.get(id1).serializeToJson(), equalTo("{\"id\":\"id1\",\"attributes\":{\"hostname\":\"h1\"}}")); + + assertThat(flagDb.sync(Collections.emptyMap()), equalTo(true)); + assertThat(getDbContent(), equalTo("{\"flags\":[]}")); } - private void writeUtf8FlagFile(String flagIdAkaFilename, String content) { - uncheck(() -> Files.createDirectories(flagDirectory.getPath())); - uncheck(() -> Files.write(flagDirectory.getPath().resolve(flagIdAkaFilename), content.getBytes(StandardCharsets.UTF_8))); + public String getDbContent() { + return uncheck(() -> new String(Files.readAllBytes(flagDb.getPath()), StandardCharsets.UTF_8)); } }
\ No newline at end of file |