diff options
author | Håkon Hallingstad <hakon@oath.com> | 2019-01-09 12:08:53 +0100 |
---|---|---|
committer | Håkon Hallingstad <hakon@oath.com> | 2019-01-09 12:08:53 +0100 |
commit | 68cbb82330cb610ff3e3462a7ec704f78e49a31c (patch) | |
tree | 6c3a996789700f24037a2c6a4d4d37f7f643ea60 /flags | |
parent | 8bec9bc0719af7ee27cda0f0d6d6b3627d155180 (diff) |
Flag repository cfg client and flag directory
- Makes new FlagRepository config server client to retrieve all flag data.
- Makes WireFlagDataList to be used for creating the HTTP response in the
config server, and parse the HTTP response in host admin.
- Fixes problem with URL generation for controller: when port is not present
in the request, remove ":-1" port specification in the url.
- Makes a new FlagDirectory class, responsible for reading flags from
/opt/vespa/var/vespa/flags (in FlagData JSON format), and reversely,
sync that directory to exactly match a set of FlagData.
- No longer have 'State state()' as a default method in interface.
Diffstat (limited to 'flags')
7 files changed, 285 insertions, 3 deletions
diff --git a/flags/pom.xml b/flags/pom.xml index 99fde3faebb..fc38676ff20 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -38,6 +38,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> <scope>provided</scope> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java index f4e23144449..1a31ecd713e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.flags.json.Rule; @@ -21,8 +20,6 @@ import java.util.Optional; * @author hakonhall */ public class FileFlagSource implements FlagSource { - private static final ObjectMapper mapper = new ObjectMapper(); - static final String FLAGS_DIRECTORY = "/etc/vespa/flags"; private final Path flagsDirectory; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java index da8e6b29cab..182ab85858c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java @@ -4,8 +4,11 @@ package com.yahoo.vespa.flags; import java.util.Optional; /** + * A source of raw flag values that can be converted to typed flag values elsewhere. + * * @author hakonhall */ public interface FlagSource { + /** Get raw flag for the given vector (specifying hostname, application id, etc). */ Optional<RawFlag> fetch(FlagId id, FetchVector vector); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java new file mode 100644 index 00000000000..7dd61a18ad1 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDirectory.java @@ -0,0 +1,156 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.file; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.NotDirectoryException; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Java API for a local file-based flag repository. + * + * @author hakonhall + */ +public class FlagDirectory { + private static final Logger logger = Logger.getLogger(FlagDirectory.class.getName()); + + private final Path flagDirectory; + + public FlagDirectory() { + this(FileSystems.getDefault()); + } + + FlagDirectory(FileSystem fileSystem) { + this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flags")); + } + + public FlagDirectory(Path flagDirectory) { + this.flagDirectory = flagDirectory; + } + + public Path getPath() { + return flagDirectory; + } + + public Map<FlagId, FlagData> read() { + return getAllRegularFilesStream() + .map(path -> { + FlagId flagId = new FlagId(getFilenameOf(path)); + Optional<FlagData> flagData = readFlagData(flagId); + if (!flagData.isPresent()) return null; + if (!Objects.equals(flagData.get().id(), flagId)) { + logger.log(LogLevel.WARNING, "Flag file " + path + " contains conflicting id " + + flagData.get().id() + ", ignoring flag"); + return null; + } + return flagData.get(); + }) + .filter(Objects::nonNull) + .collect(Collectors.toMap(FlagData::id, Function.identity())); + } + + public Optional<FlagData> readFlagData(FlagId flagId) { + return readUtf8File(getPathFor(flagId)).map(FlagData::deserialize); + } + + /** + * Modify the flag directory as necessary, such that a later {@link #read()} will return {@code flagData}. + * + * @return true if any modifications were done. + */ + public boolean sync(Map<FlagId, FlagData> flagData) { + boolean modified = false; + + Set<Path> pathsToDelete = getAllRegularFilesStream().collect(Collectors.toCollection(HashSet::new)); + + uncheck(() -> Files.createDirectories(flagDirectory)); + for (Map.Entry<FlagId, FlagData> entry : flagData.entrySet()) { + FlagId flagId = entry.getKey(); + FlagData data = entry.getValue(); + Path path = getPathFor(flagId); + + pathsToDelete.remove(path); + + String serializedData = data.serializeToJson(); + Optional<String> fileContent = readUtf8File(path); + if (fileContent.isPresent()) { + if (!Objects.equals(fileContent.get(), serializedData)) { + logger.log(LogLevel.INFO, "Updating flag " + flagId + " from " + fileContent.get() + + " to " + serializedData); + writeUtf8File(path, serializedData); + modified = true; + } + } else { + logger.log(LogLevel.INFO, "New flag " + flagId + ": " + serializedData); + writeUtf8File(path, serializedData); + modified = true; + } + } + + for (Path path : pathsToDelete) { + logger.log(LogLevel.INFO, "Removing flag file " + path); + uncheck(() -> Files.deleteIfExists(path)); + modified = true; + } + + return modified; + } + + private Stream<Path> getAllRegularFilesStream() { + try { + return Files.list(flagDirectory).filter(Files::isRegularFile); + } catch (NotDirectoryException | NoSuchFileException e) { + return Stream.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String getFilenameOf(Path path) { + return path.getName(path.getNameCount() - 1).toString(); + } + + private Path getPathFor(FlagId flagId) { + return flagDirectory.resolve(flagId.toString()); + } + + private Optional<String> readUtf8File(Path path) { + try { + return Optional.of(new String(Files.readAllBytes(path), StandardCharsets.UTF_8)); + } catch (NoSuchFileException e) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void writeUtf8File(Path path, String content) { + uncheck(() -> Files.write(path, content.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public String toString() { + return "FlagDirectory{" + flagDirectory + '}'; + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java new file mode 100644 index 00000000000..27ad44f938e --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.flags.file; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java new file mode 100644 index 00000000000..09a7f635dd4 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java @@ -0,0 +1,29 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json.wire; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author hakonhall + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireFlagDataList { + @JsonProperty("flags") + public List<WireFlagData> flags = new ArrayList<>(); + + private static final ObjectMapper mapper = new ObjectMapper(); + + public void serializeToOutputStream(OutputStream outputStream) { + uncheck(() -> mapper.writeValue(outputStream, this)); + } +} diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java new file mode 100644 index 00000000000..fec98fab164 --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDirectoryTest.java @@ -0,0 +1,86 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.file; + +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.hamcrest.collection.IsMapContaining; +import org.hamcrest.collection.IsMapWithSize; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + * @author hakonhall + */ +public class FlagDirectoryTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final FlagDirectory flagDirectory = new FlagDirectory(fileSystem); + + @Test + public void testReadingOnly() { + Map<FlagId, FlagData> data = flagDirectory.read(); + assertThat(data, IsMapWithSize.anEmptyMap()); + + FlagId id1 = new FlagId("id1"); + String json1 = "{\"id\":\"id1\"}"; + writeUtf8FlagFile(id1.toString(), json1); + data = flagDirectory.read(); + assertThat(data, IsMapWithSize.aMapWithSize(1)); + assertThat(data, IsMapContaining.hasKey(id1)); + assertThat(data.get(id1).id(), equalTo(id1)); + assertThat(data.get(id1).serializeToJson(), equalTo(json1)); + } + + @Test + public void testSync() { + Map<FlagId, FlagData> dataMap = new HashMap<>(); + FlagId id1 = new FlagId("id1"); + FlagData data1 = new FlagData(id1, new FetchVector()); + dataMap.put(id1, data1); + FlagId id2 = new FlagId("id2"); + FlagData data2 = new FlagData(id2, new FetchVector()); + dataMap.put(id2, data2); + + // Non-existing directory => empty map + assertThat(flagDirectory.read(), IsMapWithSize.anEmptyMap()); + + // sync() will create directory with map content + assertThat(flagDirectory.sync(dataMap), equalTo(true)); + Map<FlagId, FlagData> readDataMap = flagDirectory.read(); + assertThat(readDataMap, IsMapWithSize.aMapWithSize(2)); + assertThat(readDataMap, IsMapContaining.hasKey(id1)); + assertThat(readDataMap, IsMapContaining.hasKey(id2)); + + // another sync with the same data is a no-op + assertThat(flagDirectory.sync(dataMap), equalTo(false)); + + // Changing value of id1, removing id2, adding id3 + dataMap.remove(id2); + FlagData newData1 = new FlagData(id1, new FetchVector().with(FetchVector.Dimension.HOSTNAME, "h1")); + dataMap.put(id1, newData1); + FlagId id3 = new FlagId("id3"); + FlagData data3 = new FlagData(id3, new FetchVector()); + dataMap.put(id3, data3); + assertThat(flagDirectory.sync(dataMap), equalTo(true)); + Map<FlagId, FlagData> anotherReadDataMap = flagDirectory.read(); + assertThat(anotherReadDataMap, IsMapWithSize.aMapWithSize(2)); + assertThat(anotherReadDataMap, IsMapContaining.hasKey(id1)); + assertThat(anotherReadDataMap, IsMapContaining.hasKey(id3)); + assertThat(anotherReadDataMap.get(id1).serializeToJson(), equalTo("{\"id\":\"id1\",\"attributes\":{\"hostname\":\"h1\"}}")); + } + + private void writeUtf8FlagFile(String flagIdAkaFilename, String content) { + uncheck(() -> Files.createDirectories(flagDirectory.getPath())); + uncheck(() -> Files.write(flagDirectory.getPath().resolve(flagIdAkaFilename), content.getBytes(StandardCharsets.UTF_8))); + } +}
\ No newline at end of file |