diff options
author | HÃ¥kon Hallingstad <hakon@oath.com> | 2019-01-10 09:36:27 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-01-10 09:36:27 +0100 |
commit | e87ecf42627da4fc1b38c6d8eeceadd1784558bc (patch) | |
tree | 80d5833da22f4dafce8770be5d59a7f127863522 /flags | |
parent | bc78c77b555be488b9d7be155b6190a4bfe2283f (diff) | |
parent | 27ad82c5122b28e886d04904a73cd17e9bbcd105 (diff) |
Merge pull request #8071 from vespa-engine/hakonhall/flag-repository-cfg-client-and-flag-directory
Flag repository cfg client and flag directory
Diffstat (limited to 'flags')
9 files changed, 258 insertions, 5 deletions
diff --git a/flags/pom.xml b/flags/pom.xml index 99fde3faebb..fc38676ff20 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -38,6 +38,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> <scope>provided</scope> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index 581ec599aab..7d84efa52b2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -17,8 +17,7 @@ import java.util.function.Consumer; * @author hakonhall */ @Immutable -public -class FetchVector { +public class FetchVector { public enum Dimension { /** Value from ZoneId::value */ ZONE_ID, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java index f4e23144449..1a31ecd713e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.flags.json.Rule; @@ -21,8 +20,6 @@ import java.util.Optional; * @author hakonhall */ public class FileFlagSource implements FlagSource { - private static final ObjectMapper mapper = new ObjectMapper(); - static final String FLAGS_DIRECTORY = "/etc/vespa/flags"; private final Path flagsDirectory; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java index da8e6b29cab..182ab85858c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java @@ -4,8 +4,11 @@ package com.yahoo.vespa.flags; import java.util.Optional; /** + * A source of raw flag values that can be converted to typed flag values elsewhere. + * * @author hakonhall */ public interface FlagSource { + /** Get raw flag for the given vector (specifying hostname, application id, etc). */ Optional<RawFlag> fetch(FlagId id, FetchVector vector); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java new file mode 100644 index 00000000000..abe8d407ab0 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java @@ -0,0 +1,111 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.file; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Java API for a flag database stored in a single file + * + * @author hakonhall + */ +public class FlagDbFile { + private static final Logger logger = Logger.getLogger(FlagDbFile.class.getName()); + + private final Path path; + + public FlagDbFile() { + this(FileSystems.getDefault()); + } + + public FlagDbFile(FileSystem fileSystem) { + this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flag.db")); + } + + public FlagDbFile(Path path) { + this.path = path; + } + + public Path getPath() { + return path; + } + + public Map<FlagId, FlagData> read() { + Optional<byte[]> bytes = readFile(); + if (!bytes.isPresent()) return Collections.emptyMap(); + return FlagData.deserializeList(bytes.get()).stream().collect(Collectors.toMap(FlagData::id, Function.identity())); + } + + public boolean sync(Map<FlagId, FlagData> flagData) { + boolean modified = false; + Map<FlagId, FlagData> currentFlagData = read(); + Set<FlagId> flagIdsToBeRemoved = new HashSet<>(currentFlagData.keySet()); + List<FlagData> flagDataList = new ArrayList<>(flagData.values()); + + for (FlagData data : flagDataList) { + flagIdsToBeRemoved.remove(data.id()); + + FlagData existingFlagData = currentFlagData.get(data.id()); + if (existingFlagData == null) { + logger.log(LogLevel.INFO, "New flag " + data.id() + ": " + data.serializeToJson()); + modified = true; + + // Could also consider testing with FlagData::equals, but that would be too fragile? + } else if (!Objects.equals(data.serializeToJson(), existingFlagData.serializeToJson())){ + logger.log(LogLevel.INFO, "Updating flag " + data.id() + " from " + + existingFlagData.serializeToJson() + " to " + data.serializeToJson()); + modified = true; + } + } + + if (!flagIdsToBeRemoved.isEmpty()) { + String flagIdsString = flagIdsToBeRemoved.stream().map(FlagId::toString).collect(Collectors.joining(", ")); + logger.log(LogLevel.INFO, "Removing flags " + flagIdsString); + modified = true; + } + + if (!modified) return false; + + writeFile(FlagData.serializeListToUtf8Json(flagDataList)); + + return modified; + } + + private Optional<byte[]> readFile() { + try { + return Optional.of(Files.readAllBytes(path)); + } catch (NoSuchFileException e) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void writeFile(byte[] bytes) { + uncheck(() -> Files.createDirectories(path.getParent())); + uncheck(() -> Files.write(path, bytes)); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java new file mode 100644 index 00000000000..27ad44f938e --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.flags.file; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java index 572ec511607..64c4bbe7616 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.RawFlag; import com.yahoo.vespa.flags.json.wire.WireFlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagDataList; import com.yahoo.vespa.flags.json.wire.WireRule; import javax.annotation.concurrent.Immutable; @@ -114,6 +115,24 @@ public class FlagData { ); } + public static byte[] serializeListToUtf8Json(List<FlagData> list) { + return listToWire(list).serializeToBytes(); + } + + public static List<FlagData> deserializeList(byte[] bytes) { + return listFromWire(WireFlagDataList.deserializeFrom(bytes)); + } + + public static WireFlagDataList listToWire(List<FlagData> list) { + WireFlagDataList wireList = new WireFlagDataList(); + wireList.flags = list.stream().map(FlagData::toWire).collect(Collectors.toList()); + return wireList; + } + + public static List<FlagData> listFromWire(WireFlagDataList wireList) { + return wireList.flags.stream().map(FlagData::fromWire).collect(Collectors.toList()); + } + private static List<Rule> rulesFromWire(List<WireRule> wireRules) { if (wireRules == null) return Collections.emptyList(); return wireRules.stream().map(Rule::fromWire).collect(Collectors.toList()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java new file mode 100644 index 00000000000..60b35d9b69e --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java @@ -0,0 +1,37 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json.wire; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author hakonhall + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireFlagDataList { + @JsonProperty("flags") + public List<WireFlagData> flags = new ArrayList<>(); + + private static final ObjectMapper mapper = new ObjectMapper(); + + public void serializeToOutputStream(OutputStream outputStream) { + uncheck(() -> mapper.writeValue(outputStream, this)); + } + + public byte[] serializeToBytes() { + return uncheck(() -> mapper.writeValueAsBytes(this)); + } + + public static WireFlagDataList deserializeFrom(byte[] bytes) { + return uncheck(() -> mapper.readValue(bytes, WireFlagDataList.class)); + } +} diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java new file mode 100644 index 00000000000..fd1a71e4b4a --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java @@ -0,0 +1,76 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.file; + +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.hamcrest.collection.IsMapContaining; +import org.hamcrest.collection.IsMapWithSize; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + * @author hakonhall + */ +public class FlagDbFileTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final FlagDbFile flagDb = new FlagDbFile(fileSystem); + + @Test + public void test() { + Map<FlagId, FlagData> dataMap = new HashMap<>(); + FlagId id1 = new FlagId("id1"); + FlagData data1 = new FlagData(id1, new FetchVector()); + dataMap.put(id1, data1); + FlagId id2 = new FlagId("id2"); + FlagData data2 = new FlagData(id2, new FetchVector()); + dataMap.put(id2, data2); + + // Non-existing directory => empty map + assertThat(flagDb.read(), IsMapWithSize.anEmptyMap()); + + // sync() will create directory with map content + assertThat(flagDb.sync(dataMap), equalTo(true)); + Map<FlagId, FlagData> readDataMap = flagDb.read(); + assertThat(readDataMap, IsMapWithSize.aMapWithSize(2)); + assertThat(readDataMap, IsMapContaining.hasKey(id1)); + assertThat(readDataMap, IsMapContaining.hasKey(id2)); + + assertThat(getDbContent(), equalTo("{\"flags\":[{\"id\":\"id1\"},{\"id\":\"id2\"}]}")); + + // another sync with the same data is a no-op + assertThat(flagDb.sync(dataMap), equalTo(false)); + + // Changing value of id1, removing id2, adding id3 + dataMap.remove(id2); + FlagData newData1 = new FlagData(id1, new FetchVector().with(FetchVector.Dimension.HOSTNAME, "h1")); + dataMap.put(id1, newData1); + FlagId id3 = new FlagId("id3"); + FlagData data3 = new FlagData(id3, new FetchVector()); + dataMap.put(id3, data3); + assertThat(flagDb.sync(dataMap), equalTo(true)); + Map<FlagId, FlagData> anotherReadDataMap = flagDb.read(); + assertThat(anotherReadDataMap, IsMapWithSize.aMapWithSize(2)); + assertThat(anotherReadDataMap, IsMapContaining.hasKey(id1)); + assertThat(anotherReadDataMap, IsMapContaining.hasKey(id3)); + assertThat(anotherReadDataMap.get(id1).serializeToJson(), equalTo("{\"id\":\"id1\",\"attributes\":{\"hostname\":\"h1\"}}")); + + assertThat(flagDb.sync(Collections.emptyMap()), equalTo(true)); + assertThat(getDbContent(), equalTo("{\"flags\":[]}")); + } + + public String getDbContent() { + return uncheck(() -> new String(Files.readAllBytes(flagDb.getPath()), StandardCharsets.UTF_8)); + } +}
\ No newline at end of file |