Read prod test and steps tags, and some refactoring

6 files changed, 401 insertions, 74 deletions

diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json
index 7920b7bab78..e5635aea4c3 100644
--- a/config-model-api/abi-spec.json
+++ b/config-model-api/abi-spec.json
@@ -188,7 +188,7 @@
     "fields": []
   },
   "com.yahoo.config.application.api.DeploymentInstanceSpec": {
-    "superClass": "com.yahoo.config.application.api.DeploymentSpec$Step",
+    "superClass": "com.yahoo.config.application.api.DeploymentSpec$Steps",
     "interfaces": [],
     "attributes": [
       "public"
@@ -196,15 +196,11 @@
     "methods": [
       "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List)",
       "public com.yahoo.config.provision.InstanceName name()",
-      "public java.time.Duration delay()",
-      "public java.util.List steps()",
       "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()",
       "public java.util.List changeBlocker()",
       "public java.util.Optional globalServiceId()",
       "public boolean canUpgradeAt(java.time.Instant)",
       "public boolean canChangeRevisionAt(java.time.Instant)",
-      "public java.util.List zones()",
-      "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)",
       "public java.util.Optional athenzDomain()",
       "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)",
       "public com.yahoo.config.application.api.Notifications notifications()",
@@ -238,7 +234,7 @@
       "public"
     ],
     "methods": [
-      "public void <init>(com.yahoo.config.provision.RegionName, java.util.Optional)",
+      "public void <init>(com.yahoo.config.provision.RegionName)",
       "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)",
       "public boolean isTest()",
       "public boolean equals(java.lang.Object)",
@@ -292,6 +288,8 @@
     ],
     "methods": [
       "public void <init>(java.util.List)",
+      "public java.time.Duration delay()",
+      "public boolean isOrdered()",
       "public boolean equals(java.lang.Object)",
       "public int hashCode()",
       "public java.lang.String toString()"
@@ -312,7 +310,8 @@
       "public java.util.List zones()",
       "public java.time.Duration delay()",
       "public java.util.List steps()",
-      "public boolean isTest()"
+      "public boolean isTest()",
+      "public boolean isOrdered()"
     ],
     "fields": []
   },
@@ -327,6 +326,7 @@
       "public java.util.List zones()",
       "public java.util.List steps()",
       "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)",
+      "public java.time.Duration delay()",
       "public boolean equals(java.lang.Object)",
       "public int hashCode()",
       "public java.lang.String toString()"
@@ -603,4 +603,4 @@
       "public static final com.yahoo.config.application.api.ValidationOverrides all"
     ]
   }
-}
\ No newline at end of file
+}
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
index 7645f4c4ebe..541e52cf6e9 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
@@ -7,7 +7,6 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.RegionName;
 
-import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -22,12 +21,11 @@ import java.util.stream.Collectors;
  *
  * @author bratseth
  */
-public class DeploymentInstanceSpec extends DeploymentSpec.Step {
+public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
 
     /** The name of the instance this step deploys */
     private final InstanceName name;
 
-    private final List<DeploymentSpec.Step> steps;
     private final DeploymentSpec.UpgradePolicy upgradePolicy;
     private final List<DeploymentSpec.ChangeBlocker> changeBlockers;
     private final Optional<String> globalServiceId;
@@ -45,34 +43,54 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Step {
                                   Optional<AthenzService> athenzService,
                                   Notifications notifications,
                                   List<Endpoint> endpoints) {
+        super(steps);
         this.name = name;
-        this.steps = steps;
         this.upgradePolicy = upgradePolicy;
         this.changeBlockers = changeBlockers;
         this.globalServiceId = globalServiceId;
         this.athenzDomain = athenzDomain;
         this.athenzService = athenzService;
         this.notifications = notifications;
-        this.endpoints = List.copyOf(validateEndpoints(endpoints, this.steps));
-        validateZones(this.steps);
-        validateEndpoints(this.steps, globalServiceId, this.endpoints);
+        this.endpoints = List.copyOf(validateEndpoints(endpoints, steps()));
+        validateZones(new HashSet<>(), this);
+        validateEndpoints(steps(), globalServiceId, this.endpoints);
         validateAthenz();
     }
 
     public InstanceName name() { return name; }
 
-    /** Throw an IllegalArgumentException if any production zone is declared multiple times */
-    private void validateZones(List<DeploymentSpec.Step> steps) {
-        Set<DeploymentSpec.DeclaredZone> zones = new HashSet<>();
-
-        for (DeploymentSpec.Step step : steps)
-            for (DeploymentSpec.DeclaredZone zone : step.zones())
-                ensureUnique(zone, zones);
-    }
+    /**
+     * Throws an IllegalArgumentException if any production deployment or test is declared multiple times,
+     * or if any production test is declared not after its corresponding deployment.
+     *
+     * @param deployments previously seen deployments
+     * @param step step whose members to validate
+     * @return all contained tests
+     */
+    private static Set<RegionName> validateZones(Set<RegionName> deployments, DeploymentSpec.Step step) {
+        if ( ! step.steps().isEmpty()) {
+            Set<RegionName> oldDeployments = Set.copyOf(deployments);
+            Set<RegionName> tests = new HashSet<>();
+            for (DeploymentSpec.Step nested : step.steps()) {
+                for (RegionName test : validateZones(deployments, nested)) {
+                    if ( ! (step.isOrdered() ? deployments : oldDeployments).contains(test))
+                        throw new IllegalArgumentException("tests for prod." + test + " must be after the corresponding deployment in deployment.xml");
+
+                    if ( ! tests.add(test))
+                        throw new IllegalArgumentException("tests for prod." + test + " arelisted twice in deployment.xml");
+                }
+            }
+            return tests;
+        }
+        if (step.concerns(Environment.prod)) {
+            if (step.isTest())
+                return Set.of(((DeploymentSpec.DeclaredTest) step).region());
 
-    private void ensureUnique(DeploymentSpec.DeclaredZone zone, Set<DeploymentSpec.DeclaredZone> zones) {
-        if ( ! zones.add(zone))
-            throw new IllegalArgumentException(zone + " is listed twice in deployment.xml");
+            RegionName region = ((DeploymentSpec.DeclaredZone) step).region().get();
+            if ( ! deployments.add(region))
+                throw new IllegalArgumentException("prod." + region + " is listed twice in deployment.xml");
+        }
+        return Set.of();
     }
 
     /** Validates the endpoints and makes sure default values are respected */
@@ -143,15 +161,6 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Step {
         }
     }
 
-    @Override
-    public Duration delay() {
-        return Duration.ofSeconds(steps.stream().mapToLong(step -> (step.delay().getSeconds())).sum());
-    }
-
-    /** Returns the deployment steps inside this in the order they will be performed */
-    @Override
-    public List<DeploymentSpec.Step> steps() { return steps; }
-
     /** Returns the upgrade policy of this, which is defaultPolicy if none is specified */
     public DeploymentSpec.UpgradePolicy upgradePolicy() { return upgradePolicy; }
 
@@ -173,19 +182,6 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Step {
                              .noneMatch(block -> block.window().includes(instant));
     }
 
-    @Override
-    public List<DeploymentSpec.DeclaredZone> zones() {
-        return steps.stream()
-                    .flatMap(step -> step.zones().stream())
-                    .collect(Collectors.toUnmodifiableList());
-    }
-
-    /** Returns whether this deployment spec specifies the given zone, either implicitly or explicitly */
-    @Override
-    public boolean concerns(Environment environment, Optional<RegionName> region) {
-        return steps.stream().anyMatch(step -> step.concerns(environment, region));
-    }
-
     /** Returns the athenz domain if configured */
     public Optional<AthenzDomain> athenzDomain() { return athenzDomain; }
 
@@ -217,7 +213,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Step {
         return globalServiceId.equals(other.globalServiceId) &&
                upgradePolicy == other.upgradePolicy &&
                changeBlockers.equals(other.changeBlockers) &&
-               steps.equals(other.steps) &&
+               steps().equals(other.steps()) &&
                athenzDomain.equals(other.athenzDomain) &&
                athenzService.equals(other.athenzService) &&
                notifications.equals(other.notifications) &&
@@ -226,7 +222,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Step {
 
     @Override
     public int hashCode() {
-        return Objects.hash(globalServiceId, upgradePolicy, changeBlockers, steps, athenzDomain, athenzService, notifications, endpoints);
+        return Objects.hash(globalServiceId, upgradePolicy, changeBlockers, steps(), athenzDomain, athenzService, notifications, endpoints);
     }
 
     @Override
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
index b37a915ca86..67b3d585881 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
@@ -13,10 +13,12 @@ import java.io.Reader;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Specifies the environments and regions to which an application should be deployed.
@@ -198,11 +200,15 @@ public class DeploymentSpec {
 
     private static List<DeploymentInstanceSpec> instances(List<DeploymentSpec.Step> steps) {
         return steps.stream()
-                    .flatMap(step -> step instanceof Steps ? step.steps().stream() : List.of(step).stream())
-                    .filter(step -> step instanceof DeploymentInstanceSpec).map(DeploymentInstanceSpec.class::cast)
+                    .flatMap(DeploymentSpec::flatten)
                     .collect(Collectors.toList());
     }
 
+    private static Stream<DeploymentInstanceSpec> flatten(Step step) {
+        if (step instanceof DeploymentInstanceSpec) return Stream.of((DeploymentInstanceSpec) step);
+        return step.steps().stream().flatMap(DeploymentSpec::flatten);
+    }
+
     /**
      * Creates a deployment spec from XML.
      *
@@ -265,26 +271,31 @@ public class DeploymentSpec {
     /** A deployment step */
     public abstract static class Step {
 
-        /** Returns whether this step specifies the given environment */
+        /** Returns whether this step specifies the given environment. */
         public final boolean concerns(Environment environment) {
             return concerns(environment, Optional.empty());
         }
 
-        /** Returns whether this step specifies the given environment, and (optionally) region */
+        /** Returns whether this step specifies the given environment, and, optionally, region. */
         public abstract boolean concerns(Environment environment, Optional<RegionName> region);
 
-        /** Returns the zones deployed to in this step */
+        /** Returns the zones deployed to in this step. */
         public List<DeclaredZone> zones() { return Collections.emptyList(); }
 
-        /** The delay introduced by this step (beyond the time it takes to execute the step). Default is zero. */
+        /** The delay introduced by this step (beyond the time it takes to execute the step). */
         public Duration delay() { return Duration.ZERO; }
 
-        /** Returns all the steps nested in this. */
+        /** Returns any steps nested in this. */
         public List<Step> steps() { return List.of(); }
 
-        /** Returns whether this step is a test run. */
+        /** Returns whether this step is a test step. */
         public boolean isTest() { return false; }
 
+        /** Returns whether the nested steps in this, if any, should be performed in declaration order. */
+        public boolean isOrdered() {
+            return true;
+        }
+
     }
 
     /** A deployment step which is to wait for some time before progressing to the next step */
@@ -386,33 +397,35 @@ public class DeploymentSpec {
     public static class DeclaredTest extends Step {
 
         private final RegionName region;
-        private final Optional<String> testerFlavor;
 
-        public DeclaredTest(RegionName region, Optional<String> testerFlavor) {
-            this.region = region;
-            this.testerFlavor = testerFlavor;
+        public DeclaredTest(RegionName region) {
+            this.region = Objects.requireNonNull(region);
         }
 
         @Override
         public boolean concerns(Environment environment, Optional<RegionName> region) {
-            return environment == Environment.prod && Optional.of(this.region).equals(region);
+            return region.map(this.region::equals).orElse(true) && environment == Environment.prod;
         }
 
         @Override
         public boolean isTest() { return true; }
 
+        /** Returns the region this test is for. */
+        public RegionName region() {
+            return region;
+        }
+
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             DeclaredTest that = (DeclaredTest) o;
-            return region.equals(that.region) &&
-                   testerFlavor.equals(that.testerFlavor);
+            return region.equals(that.region);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(region, testerFlavor);
+            return Objects.hash(region);
         }
 
         @Override
@@ -433,10 +446,9 @@ public class DeploymentSpec {
 
         @Override
         public List<DeclaredZone> zones() {
-            return this.steps.stream()
-                             .filter(step -> step instanceof DeclaredZone)
-                             .map(DeclaredZone.class::cast)
-                             .collect(Collectors.toList());
+            return steps.stream()
+                        .flatMap(step -> step.zones().stream())
+                        .collect(Collectors.toUnmodifiableList());
         }
 
         @Override
@@ -448,6 +460,11 @@ public class DeploymentSpec {
         }
 
         @Override
+        public Duration delay() {
+            return steps.stream().map(Step::delay).reduce(Duration.ZERO, Duration::plus);
+        }
+
+        @Override
         public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
@@ -474,6 +491,16 @@ public class DeploymentSpec {
         }
 
         @Override
+        public Duration delay() {
+            return steps().stream().map(Step::delay).max(Comparator.naturalOrder()).orElse(Duration.ZERO);
+        }
+
+        @Override
+        public boolean isOrdered() {
+            return false;
+        }
+
+        @Override
         public boolean equals(Object o) {
             if (this == o) return true;
             if ( ! (o instanceof ParallelSteps)) return false;
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
index b38816d8405..c14e6ce5966 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
@@ -3,10 +3,12 @@ package com.yahoo.config.application.api.xml;
 
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.DeploymentSpec.DeclaredTest;
 import com.yahoo.config.application.api.DeploymentSpec.DeclaredZone;
 import com.yahoo.config.application.api.DeploymentSpec.Delay;
 import com.yahoo.config.application.api.DeploymentSpec.ParallelSteps;
 import com.yahoo.config.application.api.DeploymentSpec.Step;
+import com.yahoo.config.application.api.DeploymentSpec.Steps;
 import com.yahoo.config.application.api.Endpoint;
 import com.yahoo.config.application.api.Notifications;
 import com.yahoo.config.application.api.Notifications.Role;
@@ -20,6 +22,7 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.io.IOUtils;
 import com.yahoo.text.XML;
 import org.w3c.dom.Element;
+import org.w3c.dom.Node;
 
 import java.io.IOException;
 import java.io.Reader;
@@ -32,8 +35,10 @@ import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * @author bratseth
@@ -119,7 +124,7 @@ public class DeploymentSpecXmlReader {
      *
      * @param instanceNameString a comma-separated list of the names of the instances this is for
      * @param instanceTag the element having the content of this instance
-     * @param parentTag the parent of instanceTag (or the same, if this instances is implicitly defined which means instanceTag is the root)
+     * @param parentTag the parent of instanceTag (or the same, if this instance is implicitly defined, which means instanceTag is the root)
      * @return the instances specified, one for each instance name element
      */
     private List<DeploymentInstanceSpec> readInstanceContent(String instanceNameString,
@@ -170,6 +175,7 @@ public class DeploymentSpecXmlReader {
     }
 
     // Consume the given tag as 0-N steps. 0 if it is not a step, >1 if it contains multiple nested steps that should be flattened
+    @SuppressWarnings("fallthrough")
     private List<Step> readNonInstanceSteps(Element stepTag, MutableOptional<String> globalServiceId, Element parentTag) {
         Optional<AthenzService> athenzService = stringAttribute(athenzServiceAttribute, stepTag)
                                                         .or(() -> stringAttribute(athenzServiceAttribute, parentTag))
@@ -183,7 +189,11 @@ public class DeploymentSpecXmlReader {
             throw new IllegalArgumentException("Attribute 'global-service-id' is only valid on 'prod' tag.");
 
         switch (stepTag.getTagName()) {
-            case testTag: case stagingTag:
+            case testTag:
+                if (Stream.iterate(stepTag, Objects::nonNull, Node::getParentNode)
+                          .anyMatch(node -> prodTag.equals(node.getNodeName())))
+                    return List.of(new DeclaredTest(RegionName.from(XML.getValue(stepTag).trim())));
+            case stagingTag:
                 return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor));
             case prodTag: // regions, delay and parallel may be nested within, but we can flatten them
                 return XML.getChildren(stepTag).stream()
@@ -197,6 +207,10 @@ public class DeploymentSpecXmlReader {
                 return List.of(new ParallelSteps(XML.getChildren(stepTag).stream()
                                                     .flatMap(child -> readSteps(child, globalServiceId, parentTag).stream())
                                                     .collect(Collectors.toList())));
+            case stepsTag: // regions and instances may be nested within
+                return List.of(new Steps(XML.getChildren(stepTag).stream()
+                                            .flatMap(child -> readSteps(child, globalServiceId, parentTag).stream())
+                                            .collect(Collectors.toList())));
             case regionTag:
                 return List.of(readDeclaredZone(Environment.prod, athenzService, testerFlavor, stepTag));
             default:
diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
index 9a12655fa2e..584664b6ef1 100644
--- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
+++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.RegionName;
 import org.junit.Test;
 
 import java.io.StringReader;
+import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.util.Collections;
@@ -144,6 +145,84 @@ public class DeploymentSpecTest {
     }
 
     @Test
+    public void productionTests() {
+        StringReader r = new StringReader(
+                "<deployment version='1.0'>" +
+                "   <instance id='default'>" +
+                "      <test/>" +
+                "      <staging/>" +
+                "      <prod>" +
+                "         <region active='false'>us-east-1</region>" +
+                "         <region active='true'>us-west-1</region>" +
+                "         <delay hours='1' />" +
+                "         <test>us-west-1</test>" +
+                "         <test>us-east-1</test>" +
+                "      </prod>" +
+                "   </instance>" +
+                "</deployment>"
+        );
+
+        DeploymentSpec spec = DeploymentSpec.fromXml(r);
+        List<DeploymentSpec.Step> instanceSteps = spec.steps().get(0).steps();
+        assertEquals(7, instanceSteps.size());
+        assertEquals("test", instanceSteps.get(0).toString());
+        assertEquals("staging", instanceSteps.get(1).toString());
+        assertEquals("prod.us-east-1", instanceSteps.get(2).toString());
+        assertEquals("prod.us-west-1", instanceSteps.get(3).toString());
+        assertEquals("delay PT1H", instanceSteps.get(4).toString());
+        assertEquals("tests for prod.us-west-1", instanceSteps.get(5).toString());
+        assertEquals("tests for prod.us-east-1", instanceSteps.get(6).toString());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void duplicateProductionTest() {
+        StringReader r = new StringReader(
+                "<deployment version='1.0'>" +
+                "   <instance id='default'>" +
+                "      <prod>" +
+                "         <region active='true'>us-east1</region>" +
+                "         <test>us-east1</test>" +
+                "         <test>us-east1</test>" +
+                "      </prod>" +
+                "   </instance>" +
+                "</deployment>"
+        );
+        DeploymentSpec.fromXml(r);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void productionTestBeforeDeployment() {
+        StringReader r = new StringReader(
+                "<deployment version='1.0'>" +
+                "   <instance id='default'>" +
+                "      <prod>" +
+                "         <test>us-east1</test>" +
+                "         <region active='true'>us-east1</region>" +
+                "      </prod>" +
+                "   </instance>" +
+                "</deployment>"
+        );
+        DeploymentSpec.fromXml(r);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void productionTestInParallelWithDeployment() {
+        StringReader r = new StringReader(
+                "<deployment version='1.0'>" +
+                "   <instance id='default'>" +
+                "      <prod>" +
+                "         <parallel>" +
+                "            <region active='true'>us-east1</region>" +
+                "            <test>us-east1</test>" +
+                "         </parallel>" +
+                "      </prod>" +
+                "   </instance>" +
+                "</deployment>"
+        );
+         DeploymentSpec.fromXml(r);
+    }
+
+    @Test
     public void maximalProductionSpecMultipleInstances() {
         StringReader r = new StringReader(
                 "<deployment version='1.0'>" +
@@ -418,6 +497,77 @@ public class DeploymentSpecTest {
     }
 
     @Test
+    public void testNestedParallelAndSteps() {
+        StringReader r = new StringReader(
+                "<deployment athenz-domain='domain'>" +
+                "   <instance id='instance' athenz-service='in-service'>" +
+                "      <prod>" +
+                "         <parallel>" +
+                "            <region active='true'>us-west-1</region>" +
+                "            <steps>" +
+                "               <region active='true'>us-east-3</region>" +
+                "               <delay hours='2' />" +
+                "               <region active='true'>eu-west-1</region>" +
+                "               <delay hours='2' />" +
+                "            </steps>" +
+                "            <steps>" +
+                "               <delay hours='3' />" +
+                "               <region active='true'>aws-us-east-1a</region>" +
+                "               <parallel>" +
+                "                  <region active='true' athenz-service='no-service'>ap-northeast-1</region>" +
+                "                  <region active='true'>ap-southeast-2</region>" +
+                "               </parallel>" +
+                "            </steps>" +
+                "            <delay hours='3' minutes='30' />" +
+                "         </parallel>" +
+                "         <region active='true'>us-north-7</region>" +
+                "      </prod>" +
+                "   </instance>" +
+                "</deployment>"
+        );
+
+        DeploymentSpec spec = DeploymentSpec.fromXml(r);
+        List<DeploymentSpec.Step> steps = spec.steps();
+        assertEquals(3, steps.size());
+        assertEquals("test", steps.get(0).toString());
+        assertEquals("staging", steps.get(1).toString());
+        assertEquals("instance 'instance'", steps.get(2).toString());
+        assertEquals(Duration.ofHours(4), steps.get(2).delay());
+
+        List<DeploymentSpec.Step> instanceSteps = steps.get(2).steps();
+        assertEquals(2, instanceSteps.size());
+        assertEquals("4 parallel steps", instanceSteps.get(0).toString());
+        assertEquals("prod.us-north-7", instanceSteps.get(1).toString());
+
+        List<DeploymentSpec.Step> parallelSteps = instanceSteps.get(0).steps();
+        assertEquals(4, parallelSteps.size());
+        assertEquals("prod.us-west-1", parallelSteps.get(0).toString());
+        assertEquals("4 steps", parallelSteps.get(1).toString());
+        assertEquals("3 steps", parallelSteps.get(2).toString());
+        assertEquals("delay PT3H30M", parallelSteps.get(3).toString());
+
+        List<DeploymentSpec.Step> firstSerialSteps = parallelSteps.get(1).steps();
+        assertEquals(4, firstSerialSteps.size());
+        assertEquals("prod.us-east-3", firstSerialSteps.get(0).toString());
+        assertEquals("delay PT2H", firstSerialSteps.get(1).toString());
+        assertEquals("prod.eu-west-1", firstSerialSteps.get(2).toString());
+        assertEquals("delay PT2H", firstSerialSteps.get(3).toString());
+
+        List<DeploymentSpec.Step> secondSerialSteps = parallelSteps.get(2).steps();
+        assertEquals(3, secondSerialSteps.size());
+        assertEquals("delay PT3H", secondSerialSteps.get(0).toString());
+        assertEquals("prod.aws-us-east-1a", secondSerialSteps.get(1).toString());
+        assertEquals("2 parallel steps", secondSerialSteps.get(2).toString());
+
+        List<DeploymentSpec.Step> innerParallelSteps = secondSerialSteps.get(2).steps();
+        assertEquals(2, innerParallelSteps.size());
+        assertEquals("prod.ap-northeast-1", innerParallelSteps.get(0).toString());
+        assertEquals("no-service", spec.requireInstance("instance").athenzService(Environment.prod, RegionName.from("ap-northeast-1")).get().value());
+        assertEquals("prod.ap-southeast-2", innerParallelSteps.get(1).toString());
+        assertEquals("in-service", spec.requireInstance("instance").athenzService(Environment.prod, RegionName.from("ap-southeast-2")).get().value());
+    }
+
+    @Test
     public void testParallelInstances() {
         StringReader r = new StringReader(
                 "<deployment>" +
@@ -485,8 +635,8 @@ public class DeploymentSpecTest {
                 "         <region active='true'>us-west-1</region>" +
                 "         <parallel>" +
                 "            <region active='true'>us-west-1</region>" +
-                "             <region active='true'>us-central-1</region>" +
-                "             <region active='true'>us-east-3</region>" +
+                "            <region active='true'>us-central-1</region>" +
+                "            <region active='true'>us-east-3</region>" +
                 "         </parallel>" +
                 "      </prod>" +
                 "   </instance>" +
diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
index 0d1556bc666..13ec7bf1a8b 100644
--- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
+++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.RegionName;
 import org.junit.Test;
 
 import java.io.StringReader;
+import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.util.Collections;
@@ -156,6 +157,76 @@ public class DeploymentSpecWithoutInstanceTest {
     }
 
     @Test
+    public void productionTests() {
+        StringReader r = new StringReader(
+                "<deployment version='1.0'>" +
+                "   <test/>" +
+                "   <staging/>" +
+                "   <prod>" +
+                "      <region active='false'>us-east-1</region>" +
+                "      <region active='true'>us-west-1</region>" +
+                "      <delay hours='1' />" +
+                "      <test>us-west-1</test>" +
+                "      <test>us-east-1</test>" +
+                "   </prod>" +
+                "</deployment>"
+        );
+
+        DeploymentSpec spec = DeploymentSpec.fromXml(r);
+        List<DeploymentSpec.Step> instanceSteps = spec.steps().get(0).steps();
+        assertEquals(7, instanceSteps.size());
+        assertEquals("test", instanceSteps.get(0).toString());
+        assertEquals("staging", instanceSteps.get(1).toString());
+        assertEquals("prod.us-east-1", instanceSteps.get(2).toString());
+        assertEquals("prod.us-west-1", instanceSteps.get(3).toString());
+        assertEquals("delay PT1H", instanceSteps.get(4).toString());
+        assertEquals("tests for prod.us-west-1", instanceSteps.get(5).toString());
+        assertEquals("tests for prod.us-east-1", instanceSteps.get(6).toString());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void duplicateProductionTests() {
+        StringReader r = new StringReader(
+                "<deployment version='1.0'>" +
+                "   <prod>" +
+                "      <region active='true'>us-east1</region>" +
+                "      <test>us-east1</test>" +
+                "      <test>us-east1</test>" +
+                "   </prod>" +
+                "</deployment>"
+        );
+        DeploymentSpec.fromXml(r);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void productionTestBeforeDeployment() {
+        StringReader r = new StringReader(
+                "<deployment version='1.0'>" +
+                "   <prod>" +
+                "      <test>us-east1</test>" +
+                "      <region active='true'>us-east1</region>" +
+                "   </prod>" +
+                "</deployment>"
+        );
+        DeploymentSpec.fromXml(r);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void productionTestInParallelWithDeployment() {
+        StringReader r = new StringReader(
+                "<deployment version='1.0'>" +
+                "   <prod>" +
+                "      <parallel>" +
+                "         <region active='true'>us-east1</region>" +
+                "         <test>us-east1</test>" +
+                "      </parallel>" +
+                "   </prod>" +
+                "</deployment>"
+        );
+        DeploymentSpec.fromXml(r);
+    }
+
+    @Test
     public void productionSpecWithGlobalServiceId() {
         StringReader r = new StringReader(
             "<deployment version='1.0'>" +
@@ -278,6 +349,75 @@ public class DeploymentSpecWithoutInstanceTest {
     }
 
     @Test
+    public void testNestedParallelAndSteps() {
+        StringReader r = new StringReader(
+                "<deployment athenz-domain='domain' athenz-service='service'>" +
+                "   <prod>" +
+                "      <parallel>" +
+                "         <region active='true'>us-west-1</region>" +
+                "         <steps>" +
+                "            <region active='true'>us-east-3</region>" +
+                "            <delay hours='2' />" +
+                "            <region active='true'>eu-west-1</region>" +
+                "            <delay hours='2' />" +
+                "         </steps>" +
+                "         <steps>" +
+                "            <delay hours='3' />" +
+                "            <region active='true'>aws-us-east-1a</region>" +
+                "            <parallel>" +
+                "               <region active='true' athenz-service='no-service'>ap-northeast-1</region>" +
+                "               <region active='true'>ap-southeast-2</region>" +
+                "            </parallel>" +
+                "         </steps>" +
+                "         <delay hours='3' minutes='30' />" +
+                "      </parallel>" +
+                "      <region active='true'>us-north-7</region>" +
+                "   </prod>" +
+                "</deployment>"
+        );
+
+        DeploymentSpec spec = DeploymentSpec.fromXml(r);
+        List<DeploymentSpec.Step> steps = spec.steps();
+        assertEquals(3, steps.size());
+        assertEquals("test", steps.get(0).toString());
+        assertEquals("staging", steps.get(1).toString());
+        assertEquals("instance 'default'", steps.get(2).toString());
+        assertEquals(Duration.ofHours(4), steps.get(2).delay());
+
+        List<DeploymentSpec.Step> instanceSteps = steps.get(2).steps();
+        assertEquals(2, instanceSteps.size());
+        assertEquals("4 parallel steps", instanceSteps.get(0).toString());
+        assertEquals("prod.us-north-7", instanceSteps.get(1).toString());
+
+        List<DeploymentSpec.Step> parallelSteps = instanceSteps.get(0).steps();
+        assertEquals(4, parallelSteps.size());
+        assertEquals("prod.us-west-1", parallelSteps.get(0).toString());
+        assertEquals("4 steps", parallelSteps.get(1).toString());
+        assertEquals("3 steps", parallelSteps.get(2).toString());
+        assertEquals("delay PT3H30M", parallelSteps.get(3).toString());
+
+        List<DeploymentSpec.Step> firstSerialSteps = parallelSteps.get(1).steps();
+        assertEquals(4, firstSerialSteps.size());
+        assertEquals("prod.us-east-3", firstSerialSteps.get(0).toString());
+        assertEquals("delay PT2H", firstSerialSteps.get(1).toString());
+        assertEquals("prod.eu-west-1", firstSerialSteps.get(2).toString());
+        assertEquals("delay PT2H", firstSerialSteps.get(3).toString());
+
+        List<DeploymentSpec.Step> secondSerialSteps = parallelSteps.get(2).steps();
+        assertEquals(3, secondSerialSteps.size());
+        assertEquals("delay PT3H", secondSerialSteps.get(0).toString());
+        assertEquals("prod.aws-us-east-1a", secondSerialSteps.get(1).toString());
+        assertEquals("2 parallel steps", secondSerialSteps.get(2).toString());
+
+        List<DeploymentSpec.Step> innerParallelSteps = secondSerialSteps.get(2).steps();
+        assertEquals(2, innerParallelSteps.size());
+        assertEquals("prod.ap-northeast-1", innerParallelSteps.get(0).toString());
+        assertEquals("no-service", spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("ap-northeast-1")).get().value());
+        assertEquals("prod.ap-southeast-2", innerParallelSteps.get(1).toString());
+        assertEquals("service", spec.requireInstance("default").athenzService(Environment.prod, RegionName.from("ap-southeast-2")).get().value());
+    }
+
+    @Test
     public void productionSpecWithDuplicateRegions() {
         StringReader r = new StringReader(
                 "<deployment>\n" +