diff options
Diffstat (limited to 'controller-api/src/main')
9 files changed, 390 insertions, 0 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ConfigServerFlagsTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ConfigServerFlagsTarget.java new file mode 100644 index 00000000000..0304d1a5949 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ConfigServerFlagsTarget.java @@ -0,0 +1,51 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.athenz.api.AthenzIdentity; + +import java.net.URI; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.defaultFile; +import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.environmentFile; +import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.systemFile; +import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.zoneFile; + +/** + * @author bjorncs + */ +class ConfigServerFlagsTarget implements FlagsTarget { + private final SystemName system; + private final ZoneId zone; + private final URI endpoint; + private final AthenzIdentity identity; + + ConfigServerFlagsTarget(SystemName system, ZoneId zone, URI endpoint, AthenzIdentity identity) { + this.system = Objects.requireNonNull(system); + this.zone = Objects.requireNonNull(zone); + this.endpoint = Objects.requireNonNull(endpoint); + this.identity = Objects.requireNonNull(identity); + } + + @Override public List<String> flagDataFilesPrioritized() { return List.of(zoneFile(system, zone), environmentFile(system, zone.environment()), systemFile(system), defaultFile()); } + @Override public URI endpoint() { return endpoint; } + @Override public Optional<AthenzIdentity> athenzHttpsIdentity() { return Optional.of(identity); } + @Override public String asString() { return String.format("%s.%s", system.value(), zone.value()); } + + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConfigServerFlagsTarget that = (ConfigServerFlagsTarget) o; + return system == that.system && + Objects.equals(zone, that.zone) && + Objects.equals(endpoint, that.endpoint) && + Objects.equals(identity, that.identity); + } + + @Override public int hashCode() { return Objects.hash(system, zone, endpoint, identity); } +} + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java new file mode 100644 index 00000000000..a22a9cc63de --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java @@ -0,0 +1,38 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +/** + * @author bjorncs + */ + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.athenz.api.AthenzIdentity; + +import java.net.URI; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.controllerFile; +import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.defaultFile; +import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.systemFile; + +class ControllerFlagsTarget implements FlagsTarget { + private final SystemName system; + + ControllerFlagsTarget(SystemName system) { this.system = Objects.requireNonNull(system); } + + @Override public List<String> flagDataFilesPrioritized() { return List.of(controllerFile(system), systemFile(system), defaultFile()); } + @Override public URI endpoint() { return URI.create("https://localhost:4443/"); } // Note: Cannot use VIPs for controllers due to network configuration on AWS + @Override public Optional<AthenzIdentity> athenzHttpsIdentity() { return Optional.empty(); } + @Override public String asString() { return String.format("%s.controller", system.value()); } + + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ControllerFlagsTarget that = (ControllerFlagsTarget) o; + return system == that.system; + } + + @Override public int hashCode() { return Objects.hash(system); } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java new file mode 100644 index 00000000000..1b7f84d1ec7 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java @@ -0,0 +1,68 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; + +import java.net.URI; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * Represents either configservers in a zone or controllers in a system. + * + * Defines the location and precedence of the flags data files for the given target. + * + * Naming rules for flags data files: + * <ul> + * <li>zone specific: {@code <system>.<environment>.<region>.json}</li> + * <li>controller specific: {@code <system>.controller.json}</li> + * <li>environment specific: {@code <system>.<environment>.json}</li> + * <li>system specific: {@code <system>.json}</li> + * <li>global default: {@code default.json}</li> + * </ul> + * + * @author bjorncs + */ +public interface FlagsTarget { + + List<String> flagDataFilesPrioritized(); + URI endpoint(); + Optional<AthenzIdentity> athenzHttpsIdentity(); + String asString(); + + static Set<FlagsTarget> getAllTargetsInSystem(ZoneRegistry registry) { + SystemName system = registry.system(); + Set<FlagsTarget> targets = new HashSet<>(); + for (ZoneApi zone : registry.zones().reachable().zones()) { + targets.add(forConfigServer(registry, zone.getId())); + } + targets.add(forController(system)); + return targets; + } + + static FlagsTarget forController(SystemName systemName) { + return new ControllerFlagsTarget(systemName); + } + + static FlagsTarget forConfigServer(ZoneRegistry registry, ZoneId zoneId) { + return new ConfigServerFlagsTarget( + registry.system(), zoneId, registry.getConfigServerVipUri(zoneId), registry.getConfigServerHttpsIdentity(zoneId)); + } + + static String defaultFile() { return jsonFile("default"); } + static String systemFile(SystemName system) { return jsonFile(system.value()); } + static String environmentFile(SystemName system, Environment environment) { return jsonFile(system.value() + "." + environment); } + static String zoneFile(SystemName system, ZoneId zone) { return jsonFile(system.value() + "." + zone.environment().value() + "." + zone.region().value()); } + static String controllerFile(SystemName system) { return jsonFile(system.value() + ".controller"); } + + private static String jsonFile(String nameWithoutExtension) { return nameWithoutExtension + ".json"; } +} + + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java new file mode 100644 index 00000000000..ec14dcb7123 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -0,0 +1,138 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Represents a hierarchy of flag data files. See {@link FlagsTarget} for file naming convention. + * + * The flag files must reside in a 'flags/' root directory containing a directory for each flag name: + * {@code ./flags/<flag-id>/*.json} + * + * @author bjorncs + */ +public class SystemFlagsDataArchive { + + private final Map<FlagId, Map<String, FlagData>> files; + + private SystemFlagsDataArchive(Map<FlagId, Map<String, FlagData>> files) { + this.files = files; + } + + public static SystemFlagsDataArchive fromZip(InputStream rawIn) { + Builder builder = new Builder(); + try (ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(rawIn))) { + ZipEntry entry; + while ((entry = zipIn.getNextEntry()) != null) { + String name = entry.getName(); + if (!entry.isDirectory() && name.startsWith("flags/") && name.endsWith(".json")) { + Path filePath = Paths.get(name); + String filename = filePath.getFileName().toString(); + FlagData flagData = FlagData.deserializeUtf8Json(zipIn.readAllBytes()); + verifyFlagDataMatchesDirectoryName(filePath, flagData); + builder.addFile(filename, flagData); + } + } + return builder.build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static SystemFlagsDataArchive fromDirectory(Path directory) { + Path root = directory.toAbsolutePath(); + try (Stream<Path> directoryStream = Files.walk(root)) { + Builder builder = new Builder(); + directoryStream.forEach(absolutePath -> { + Path relativePath = root.relativize(absolutePath); + if (!Files.isDirectory(absolutePath) && relativePath.startsWith("flags")) { + String filename = relativePath.getFileName().toString(); + if (filename.endsWith(".json")) { + FlagData flagData = FlagData.deserializeUtf8Json(uncheck(() -> Files.readAllBytes(absolutePath))); + verifyFlagDataMatchesDirectoryName(relativePath, flagData); + builder.addFile(filename, flagData); + } + } + }); + return builder.build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void toZip(OutputStream out) { + ZipOutputStream zipOut = new ZipOutputStream(out); + files.forEach((flagId, fileMap) -> { + fileMap.forEach((filename, flagData) -> { + uncheck(() -> { + zipOut.putNextEntry(new ZipEntry("flags/" + flagId.toString() + "/" + filename)); + zipOut.write(flagData.serializeToUtf8Json()); + zipOut.closeEntry(); + }); + }); + }); + uncheck(zipOut::flush); + } + + public Set<FlagData> flagData(FlagsTarget target) { + List<String> filenames = target.flagDataFilesPrioritized(); + Set<FlagData> targetData = new HashSet<>(); + files.forEach((flagId, fileMap) -> { + for (String filename : filenames) { + FlagData data = fileMap.get(filename); + if (data != null) { + targetData.add(data); + break; + } + } + }); + return targetData; + } + + private static void verifyFlagDataMatchesDirectoryName(Path filePath, FlagData flagData) { + String flagDirectoryName = filePath.getName(1).toString(); + if (!flagDirectoryName.equals(flagData.id().toString())) { + throw new IllegalArgumentException( + String.format("Flag data file with flag id '%s' in directory for '%s'", flagData.id(), flagDirectoryName)); + } + } + + public static class Builder { + private final Map<FlagId, Map<String, FlagData>> files = new TreeMap<>(); + + public Builder() {} + + public Builder addFile(String filename, FlagData data) { + files.computeIfAbsent(data.id(), k -> new TreeMap<>()).put(filename, data); + return this; + } + + public SystemFlagsDataArchive build() { + Map<FlagId, Map<String, FlagData>> copy = new TreeMap<>(); + files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map))); + return new SystemFlagsDataArchive(copy); + } + + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/package-info.java new file mode 100644 index 00000000000..0fe377db08c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/SystemFlagsV1Api.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/SystemFlagsV1Api.java new file mode 100644 index 00000000000..1107b70c01c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/SystemFlagsV1Api.java @@ -0,0 +1,29 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire; + +import javax.ws.rs.Consumes; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.io.InputStream; + +/** + * @author bjorncs + */ +@Path("/system-flags/v1") +public interface SystemFlagsV1Api { + + @PUT + @Produces(MediaType.APPLICATION_JSON) + @Consumes("application/zip") + @Path("/deploy") + WireSystemFlagsDeployResult deploy(InputStream inputStream); + + @PUT + @Produces(MediaType.APPLICATION_JSON) + @Consumes("application/zip") + @Path("/dryrun") + WireSystemFlagsDeployResult dryrun(InputStream inputStream); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireErrorResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireErrorResponse.java new file mode 100644 index 00000000000..996f0ac6cdd --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireErrorResponse.java @@ -0,0 +1,18 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireErrorResponse { + @JsonProperty("message") + public String message; + @JsonProperty("error-code") + public String errorCode; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java new file mode 100644 index 00000000000..bd54fd15d15 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java @@ -0,0 +1,32 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.flags.json.wire.WireFlagData; + +import java.util.List; + +/** + * + * + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireSystemFlagsDeployResult { + @JsonProperty("changes") public List<WireFlagDataChange> changes; + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class WireFlagDataChange { + @JsonProperty("flag-id") public String flagId; + @JsonProperty("targets") public List<String> targets; + @JsonProperty("operation") public String operation; + @JsonProperty("data") public WireFlagData data; + @JsonProperty("previous-data") public WireFlagData previousData; + } +} + + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/package-info.java new file mode 100644 index 00000000000..305cf91014b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file |