diff options
Diffstat (limited to 'config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java')
-rw-r--r-- | config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java | 176 |
1 files changed, 146 insertions, 30 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 8182e697e7e..a7c7ee3aab4 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -15,6 +15,8 @@ import com.yahoo.config.application.api.DeploymentSpec.Steps; import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; import com.yahoo.config.application.api.DeploymentSpec.UpgradeRollout; import com.yahoo.config.application.api.Endpoint; +import com.yahoo.config.application.api.Endpoint.Level; +import com.yahoo.config.application.api.Endpoint.Target; import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.Notifications.Role; import com.yahoo.config.application.api.Notifications.When; @@ -22,10 +24,15 @@ import com.yahoo.config.application.api.TimeWindow; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; +import com.yahoo.config.provision.ZoneEndpoint; +import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; +import com.yahoo.config.provision.ZoneEndpoint.AccessType; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.text.XML; import org.w3c.dom.Element; @@ -41,12 +48,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -123,7 +131,7 @@ public class DeploymentSpecXmlReader { return DeploymentSpec.empty; List<Step> steps = new ArrayList<>(); - List<Endpoint> applicationEndpoints = List.of(); + List<Endpoint> applicationEndpoints = new ArrayList<>(); if ( ! containsTag(instanceTag, root)) { // deployment spec skipping explicit instance -> "default" instance steps.addAll(readInstanceContent("default", root, new HashMap<>(), root)); } @@ -141,7 +149,7 @@ public class DeploymentSpecXmlReader { steps.addAll(readNonInstanceSteps(child, new HashMap<>(), root)); // (No global service id here) } } - applicationEndpoints = readEndpoints(root, Optional.empty(), steps); + readEndpoints(root, Optional.empty(), steps, applicationEndpoints, Map.of()); } return new DeploymentSpec(steps, @@ -190,11 +198,13 @@ public class DeploymentSpecXmlReader { Notifications notifications = readNotifications(instanceElement, parentTag); // Values where there is no default - Tags tags = XML.attribute(tagsTag, instanceElement).map(value -> Tags.fromString(value)).orElse(Tags.empty()); + Tags tags = XML.attribute(tagsTag, instanceElement).map(Tags::fromString).orElse(Tags.empty()); List<Step> steps = new ArrayList<>(); for (Element instanceChild : XML.getChildren(instanceElement)) steps.addAll(readNonInstanceSteps(instanceChild, prodAttributes, instanceChild)); - List<Endpoint> endpoints = readEndpoints(instanceElement, Optional.of(instanceNameString), steps); + List<Endpoint> endpoints = new ArrayList<>(); + Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints = new LinkedHashMap<>(); + readEndpoints(instanceElement, Optional.of(instanceNameString), steps, endpoints, zoneEndpoints); // Build and return instances with these values Instant now = clock.instant(); @@ -214,6 +224,7 @@ public class DeploymentSpecXmlReader { cloudAccount, notifications, endpoints, + zoneEndpoints, now)) .toList(); } @@ -306,19 +317,25 @@ public class DeploymentSpecXmlReader { return Notifications.of(emailAddresses, emailRoles); } - private List<Endpoint> readEndpoints(Element parent, Optional<String> instance, List<Step> steps) { + private void readEndpoints(Element parent, Optional<String> instance, List<Step> steps, List<Endpoint> endpoints, + Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints) { var endpointsElement = XML.getChild(parent, endpointsTag); - if (endpointsElement == null) return List.of(); + if (endpointsElement == null) return; Endpoint.Level level = instance.isEmpty() ? Endpoint.Level.application : Endpoint.Level.instance; - Map<String, Endpoint> endpoints = new LinkedHashMap<>(); + Map<String, Endpoint> endpointsById = new LinkedHashMap<>(); + Map<String, Map<RegionName, List<ZoneEndpoint>>> endpointsByZone = new LinkedHashMap<>(); for (var endpointElement : XML.getChildren(endpointsElement, endpointTag)) { - String endpointId = stringAttribute("id", endpointElement).orElse(Endpoint.DEFAULT_ID); String containerId = requireStringAttribute("container-id", endpointElement); + Optional<String> endpointId = stringAttribute("id", endpointElement); + Optional<String> zoneEndpointType = getZoneEndpointType(endpointElement, level); String msgPrefix = (level == Endpoint.Level.application ? "Application-level" : "Instance-level") + - " endpoint '" + endpointId + "': "; + " endpoint '" + endpointId.orElse(Endpoint.DEFAULT_ID) + "': "; + if (zoneEndpointType.isPresent() && endpointId.isPresent()) + illegal(msgPrefix + "cannot declare 'id' with type 'zone' or 'private'"); String invalidChild = level == Endpoint.Level.application ? "region" : "instance"; - if (!XML.getChildren(endpointElement, invalidChild).isEmpty()) illegal(msgPrefix + "invalid element '" + invalidChild + "'"); + if ( ! XML.getChildren(endpointElement, invalidChild).isEmpty()) + illegal(msgPrefix + "invalid element '" + invalidChild + "'"); List<Endpoint.Target> targets = new ArrayList<>(); if (level == Endpoint.Level.application) { @@ -345,33 +362,132 @@ public class DeploymentSpecXmlReader { if (weightSum == 0) illegal(msgPrefix + "sum of all weights must be positive, got " + weightSum); } else { if (stringAttribute("region", endpointElement).isPresent()) illegal(msgPrefix + "invalid 'region' attribute"); + Set<RegionName> regions = new LinkedHashSet<>(); for (var regionElement : XML.getChildren(endpointElement, "region")) { String region = regionElement.getTextContent(); if (region == null || region.isBlank()) illegal(msgPrefix + "empty 'region' element"); - targets.add(new Endpoint.Target(RegionName.from(region), - InstanceName.from(instance.get()), - 1)); + if ( ! regions.add(RegionName.from(region))) illegal(msgPrefix + "duplicate 'region' element: '" + region + "'"); + } + + if (zoneEndpointType.isPresent()) { + if (regions.isEmpty()) regions.add(null); + ZoneEndpoint endpoint; + Optional<String> enabled = XML.attribute("enabled", endpointElement); + if ("zone".equals(zoneEndpointType.get())) { + switch (enabled.orElse("true")) { + case "true" -> endpoint = new ZoneEndpoint(true, false, List.of()); + case "false" -> endpoint = new ZoneEndpoint(false, false, List.of()); + default -> throw new IllegalArgumentException("Zone endpoint for container-id '" + containerId + "': " + + "invalid 'enabled' value; must be 'true' or 'false'"); + } + } + else { + if (enabled.isPresent()) illegal("Private endpoint for container-id '" + containerId + "': " + + "only endpoints of type 'zone' can specify 'enabled'"); + List<AllowedUrn> allowedUrns = new ArrayList<>(); + for (var allow : XML.getChildren(endpointElement, "allow")) { + switch (requireStringAttribute("with", allow)) { + case "aws-private-link" -> allowedUrns.add(new AllowedUrn(AccessType.awsPrivateLink, requireStringAttribute("arn", allow))); + case "gcp-service-connect" -> allowedUrns.add(new AllowedUrn(AccessType.gcpServiceConnect, requireStringAttribute("project", allow))); + default -> illegal("Private endpoint for container-id '" + containerId + "': " + + "invalid attribute 'with': '" + requireStringAttribute("with", allow) + "'"); + } + } + endpoint = new ZoneEndpoint(true, true, allowedUrns); // Doesn't turn off public visibility. + } + for (RegionName region : regions) endpointsByZone.computeIfAbsent(containerId, __ -> new LinkedHashMap<>()) + .computeIfAbsent(region, __ -> new ArrayList<>()) + .add(endpoint); + } + else { + if (regions.isEmpty()) { + // No explicit targets given for instance level endpoint. Include all declared regions by default + steps.stream() + .filter(step -> step.concerns(Environment.prod)) + .flatMap(step -> step.zones().stream()) + .flatMap(zone -> zone.region().stream()) + .forEach(regions::add); + } + InstanceName instanceName = instance.map(InstanceName::from).get(); + for (RegionName region : regions) targets.add(new Target(region, instanceName, 1)); } - } - if (targets.isEmpty() && level == Endpoint.Level.instance) { - // No explicit targets given for instance level endpoint. Include all declared regions by default - InstanceName instanceName = instance.map(InstanceName::from).get(); - steps.stream() - .filter(step -> step.concerns(Environment.prod)) - .flatMap(step -> step.zones().stream()) - .flatMap(zone -> zone.region().stream()) - .distinct() - .map(region -> new Endpoint.Target(region, instanceName, 1)) - .forEach(targets::add); } - Endpoint endpoint = new Endpoint(endpointId, containerId, level, targets); - if (endpoints.containsKey(endpoint.endpointId())) { - illegal("Endpoint ID '" + endpoint.endpointId() + "' is specified multiple times"); + if (zoneEndpointType.isEmpty()) { + Endpoint endpoint = new Endpoint(endpointId.orElse(Endpoint.DEFAULT_ID), containerId, level, targets); + if (endpointsById.containsKey(endpoint.endpointId())) { + illegal("Endpoint ID '" + endpoint.endpointId() + "' is specified multiple times"); + } + endpointsById.put(endpoint.endpointId(), endpoint); } - endpoints.put(endpoint.endpointId(), endpoint); } - return List.copyOf(endpoints.values()); + endpoints.addAll(endpointsById.values()); + validateAndConsolidate(endpointsByZone, zoneEndpoints); + } + + static void validateAndConsolidate(Map<String, Map<RegionName, List<ZoneEndpoint>>> in, Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> out) { + in.forEach((cluster, regions) -> { + List<ZoneEndpoint> wildcards = regions.remove(null); + ZoneEndpoint wildcardZoneEndpoint = null; + ZoneEndpoint wildcardPrivateEndpoint = null; + if (wildcards != null) for (ZoneEndpoint endpoint : wildcards) { + if (endpoint.isPrivateEndpoint()) { + if (wildcardPrivateEndpoint != null) illegal("Multiple private endpoints (for all regions) declared for " + + "container id '" + cluster + "'"); + wildcardPrivateEndpoint = endpoint; + } + else { + if (wildcardZoneEndpoint != null) illegal("Multiple zone endpoints (for all regions) declared " + + "for container id '" + cluster + "'"); + wildcardZoneEndpoint = endpoint; + } + } + for (RegionName region : regions.keySet()) { + ZoneEndpoint zoneEndpoint = null; + ZoneEndpoint privateEndpoint = null; + for (ZoneEndpoint endpoint : regions.getOrDefault(region, List.of())) { + if (endpoint.isPrivateEndpoint()) { + if (privateEndpoint != null) illegal("Multiple private endpoints declared for " + + "container id '" + cluster + "' in region '" + region + "'"); + privateEndpoint = endpoint; + } + else { + if (zoneEndpoint != null) illegal("Multiple zone endpoints (without regions) declared " + + "for container id '" + cluster + "' in region '" + region + "'"); + zoneEndpoint = endpoint; + } + } + if (wildcardZoneEndpoint != null && zoneEndpoint != null) illegal("Zone endpoint for container id '" + cluster + "' declared " + + "both with region '" + region + "', and for all regions."); + if (wildcardPrivateEndpoint != null && privateEndpoint != null) illegal("Private endpoint for container id '" + cluster + "' declared " + + "both with region '" + region + "', and for all regions."); + + if (zoneEndpoint == null) zoneEndpoint = wildcardZoneEndpoint; + if (privateEndpoint == null) privateEndpoint = wildcardPrivateEndpoint; + + // Gosh, we made it here! Now we'll combine the settings for zone and private types into one ZoneEndpoint to rule them all. + out.computeIfAbsent(ClusterSpec.Id.from(cluster), __ -> new LinkedHashMap<>()) + .put(ZoneId.from(Environment.prod, region), new ZoneEndpoint(zoneEndpoint == null || zoneEndpoint.isPublicEndpoint(), + privateEndpoint != null, + privateEndpoint != null ? privateEndpoint.allowedUrns() : List.of())); + } + out.computeIfAbsent(ClusterSpec.Id.from(cluster), __ -> new LinkedHashMap<>()) + .put(null, new ZoneEndpoint(wildcardZoneEndpoint == null || wildcardZoneEndpoint.isPublicEndpoint(), + wildcardPrivateEndpoint != null, + wildcardPrivateEndpoint != null ? wildcardPrivateEndpoint.allowedUrns() : List.of())); + }); + } + + /** Returns endpoint type if a private or zone type endpoint, throws if invalid, or otherwise returns empty (global, application). */ + static Optional<String> getZoneEndpointType(Element endpoint, Level level) { + Optional<String> type = XML.attribute("type", endpoint); + if (type.isPresent() && ! List.of("zone", "private", "global", "application").contains(type.get())) + illegal("Illegal endpoint type '" + type.get() + "'"); + + String implied = switch (level) { case instance -> "global"; case application -> "application"; }; + if (type.isEmpty() || type.get().equals(implied)) return Optional.empty(); + if (level == Level.instance && (type.get().equals("zone") || type.get().equals("private"))) return type; + throw new IllegalArgumentException("Endpoints at " + level + " level cannot be of type '" + type.get() + "'"); } /** |