Reduce usage of legacy deployment system in MetricsReporterTest

5 files changed, 120 insertions, 49 deletions

diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index 2c88d122e8f..ce357a68ae2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -6,19 +6,20 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.slime.Slime;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.athenz.api.OktaAccessToken;
-import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
 import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
@@ -26,10 +27,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
@@ -40,9 +40,11 @@ import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec;
 import com.yahoo.vespa.hosted.controller.security.Credentials;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
+import com.yahoo.vespa.hosted.controller.versions.ControllerVersion;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
@@ -181,6 +183,44 @@ public final class ControllerTester {
         createAndDeploy(tenantName, domainName, applicationName, environment, projectId, null);
     }
 
+    /** Upgrade controller to given version */
+    public void upgradeController(Version version, String commitSha, Instant commitDate) {
+        controller().curator().writeControllerVersion(controller().hostname(), new ControllerVersion(version, commitSha, commitDate));
+        computeVersionStatus();
+    }
+
+    public void upgradeController(Version version) {
+        upgradeController(version, "badc0ffee", Instant.EPOCH);
+    }
+
+    /** Upgrade system applications in all zones to given version */
+    public void upgradeSystemApplications(Version version) {
+        upgradeSystemApplications(version, SystemApplication.all());
+    }
+
+    /** Upgrade given system applications in all zones to version */
+    public void upgradeSystemApplications(Version version, List<SystemApplication> systemApplications) {
+        for (ZoneApi zone : zoneRegistry().zones().all().zones()) {
+            for (SystemApplication application : systemApplications) {
+                configServer().setVersion(application.id(), zone.getId(), version);
+                configServer().convergeServices(application.id(), zone.getId());
+            }
+        }
+        computeVersionStatus();
+    }
+
+    /** Upgrade entire system to given version */
+    public void upgradeSystem(Version version) {
+        upgradeController(version);
+        upgradeSystemApplications(version);
+    }
+
+    /** Re-compute and write version status */
+    public void computeVersionStatus() {
+        controller().updateVersionStatus(VersionStatus.compute(controller()));
+    }
+
+
     public ZoneId toZone(Environment environment) {
         switch (environment) {
             case dev: case test:
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index f9639835cf0..881910c1625 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -132,27 +132,28 @@ public class DeploymentTester {
         return controller().applications().requireInstance(application);
     }
 
-    /** Re-compute and write version status */
+    // TODO(mpolden): Change callers to use ControllerTester#computeVersionStatus and remove this
     public void computeVersionStatus() {
         controller().updateVersionStatus(VersionStatus.compute(controller()));
     }
 
+    // TODO(mpolden): Change callers to use ControllerTester#upgradeController and remove this
     public void upgradeController(Version version) {
         upgradeController(version, "badc0ffee", Instant.EPOCH);
     }
 
-    /** Upgrade controller to given version */
+    // TODO(mpolden): Change callers to use ControllerTester#upgradeController and remove this
     public void upgradeController(Version version, String commitSha, Instant commitDate) {
         controller().curator().writeControllerVersion(controller().hostname(), new ControllerVersion(version, commitSha, commitDate));
         computeVersionStatus();
     }
 
-    /** Upgrade system applications in all zones to given version */
+    // TODO(mpolden): Change callers to use ControllerTester#upgradeSystemApplications and remove this
     public void upgradeSystemApplications(Version version) {
         upgradeSystemApplications(version, SystemApplication.all());
     }
 
-    /** Upgrade given system applications in all zones to version */
+    // TODO(mpolden): Change callers to use ControllerTester#upgradeSystemApplications and remove this
     public void upgradeSystemApplications(Version version, List<SystemApplication> systemApplications) {
         for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) {
             for (SystemApplication application : systemApplications) {
@@ -163,7 +164,7 @@ public class DeploymentTester {
         computeVersionStatus();
     }
 
-    /** Upgrade entire system to given version */
+    // TODO(mpolden): Change callers to use ControllerTester#upgradeSystem and remove this
     public void upgradeSystem(Version version) {
         upgradeController(version);
         upgradeSystemApplications(version);
@@ -171,7 +172,7 @@ public class DeploymentTester {
         readyJobTrigger().maintain();
     }
 
-    /** Flush all pending name services requests */
+    // TODO(mpolden): Change callers to use InternalDeploymentTester#flushDnsRequests and remove this
     public void flushDnsRequests() {
         nameServiceDispatcher.run();
         assertTrue("All name service requests dispatched",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
index 5b3833fd811..d252b99f026 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
@@ -14,6 +14,7 @@ import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
+import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
@@ -33,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunnerTest;
+import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher;
 
 import javax.security.auth.x500.X500Principal;
 import java.math.BigInteger;
@@ -45,6 +47,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Logger;
 
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted;
@@ -85,6 +89,11 @@ public class InternalDeploymentTester {
     private final RoutingGeneratorMock routing;
     private final MockTesterCloud cloud;
     private final JobRunner runner;
+    private final NameServiceDispatcher nameServiceDispatcher;
+
+    private final AtomicLong nextPropertyId = new AtomicLong(1);
+    private final AtomicInteger nextProjectId = new AtomicInteger(1);
+    private final AtomicInteger nextDomainId = new AtomicInteger(100);
 
     public DeploymentTester tester() { return tester; }
     public JobController jobs() { return jobs; }
@@ -92,6 +101,7 @@ public class InternalDeploymentTester {
     public MockTesterCloud cloud() { return cloud; }
     public JobRunner runner() { return runner; }
     public ConfigServerMock configServer() { return tester.configServer(); }
+    public Controller controller() { return tester.controller(); }
     public ApplicationController applications() { return tester.applications(); }
     public ManualClock clock() { return tester.clock(); }
     public Application application() { return tester.application(appId); }
@@ -99,15 +109,15 @@ public class InternalDeploymentTester {
 
     public InternalDeploymentTester() {
         tester = new DeploymentTester();
-        tester.controllerTester().createApplication(tester.controllerTester().createTenant(instanceId.tenant().value(), athenzDomain, 1L),
-                                                    instanceId.application().value(),
-                                                    instanceId.instance().value(),
-                                                    1);
+        createApplication(instanceId.tenant().value(), instanceId.application().value(), instanceId.instance().value());
         jobs = tester.controller().jobController();
         routing = tester.controllerTester().serviceRegistry().routingGeneratorMock();
         cloud = (MockTesterCloud) tester.controller().jobController().cloud();
         runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()),
                                JobRunnerTest.inThreadExecutor(), new InternalStepRunner(tester.controller()));
+        this.nameServiceDispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofHours(12),
+                                                               new JobControl(tester.controller().curator()),
+                                                               Integer.MAX_VALUE);
         routing.putEndpoints(new DeploymentId(null, null), Collections.emptyList()); // Turn off default behaviour for the mock.
 
         // Get deployment job logs to stderr.
@@ -120,6 +130,16 @@ public class InternalDeploymentTester {
         domain.services.put(ATHENZ_SERVICE, new AthenzDbMock.Service(true));
     }
 
+    /** Create a new application with given tenant and application name */
+    public Application createApplication(String tenantName, String applicationName, String instanceName) {
+        return tester.controllerTester().createApplication(tester.controllerTester().createTenant(tenantName,
+                                                                                                  athenzDomain + nextDomainId.getAndIncrement(),
+                                                                                                  nextPropertyId.getAndIncrement()),
+                                                           applicationName,
+                                                           instanceName,
+                                                           nextProjectId.getAndIncrement());
+    }
+
     /** Submits a new application, and returns the version of the new submission. */
     public ApplicationVersion newSubmission(TenantAndApplicationId id, ApplicationPackage applicationPackage,
                                             SourceRevision revision, String authorEmail, long projectId) {
@@ -128,7 +148,8 @@ public class InternalDeploymentTester {
 
     /** Submits a new application, and returns the version of the new submission. */
     public ApplicationVersion newSubmission(TenantAndApplicationId id, ApplicationPackage applicationPackage) {
-        return newSubmission(id, applicationPackage, BuildJob.defaultSourceRevision, "a@b", 2);
+        var projectId = tester.application(id).projectId().orElseThrow(() -> new IllegalArgumentException("No project ID set for " + id));
+        return newSubmission(id, applicationPackage, BuildJob.defaultSourceRevision, "a@b", projectId);
     }
 
     /**
@@ -374,7 +395,7 @@ public class InternalDeploymentTester {
         runner.advance(currentRun(job));
         assertTrue(jobs.run(id).get().hasEnded());
         assertFalse(jobs.run(id).get().hasFailed());
-        assertEquals(job.type().isProduction(), instance().deployments().containsKey(zone));
+        assertEquals(job.type().isProduction(), tester.instance(job.application()).deployments().containsKey(zone));
         assertTrue(tester.configServer().nodeRepository().list(zone, TesterId.of(id.application()).id()).isEmpty());
     }
 
@@ -530,4 +551,11 @@ public class InternalDeploymentTester {
         assertTrue(jobs.active().stream().anyMatch(run -> run.id().application().equals(id) && run.id().type() == type));
     }
 
+    /** Flush all pending name services requests */
+    public void flushDnsRequests() {
+        nameServiceDispatcher.run();
+        assertTrue("All name service requests dispatched",
+                   controller().curator().readNameServiceQueue().requests().isEmpty());
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index 81e33b490f3..4e60e13dd51 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -41,8 +41,8 @@ public class MetricsReporterTest {
     private final MetricsMock metrics = new MetricsMock();
 
     @Test
-    public void test_deployment_fail_ratio() {
-        DeploymentTester tester = new DeploymentTester();
+    public void deployment_fail_ratio() {
+        var tester = new InternalDeploymentTester();
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .environment(Environment.prod)
                 .region("us-west-1")
@@ -53,28 +53,30 @@ public class MetricsReporterTest {
         assertEquals(0.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC));
 
         // Deploy all apps successfully
-        Application app1 = tester.createApplication("app1", "tenant1", 1, 11L);
-        Application app2 = tester.createApplication("app2", "tenant1", 2, 22L);
-        Application app3 = tester.createApplication("app3", "tenant1", 3, 33L);
-        Application app4 = tester.createApplication("app4", "tenant1", 4, 44L);
-        tester.deployCompletely(app1, applicationPackage);
-        tester.deployCompletely(app2, applicationPackage);
-        tester.deployCompletely(app3, applicationPackage);
-        tester.deployCompletely(app4, applicationPackage);
+        Application app1 = tester.createApplication("app1", "tenant1", "default");
+        Application app2 = tester.createApplication("app2", "tenant1", "default");
+        Application app3 = tester.createApplication("app3", "tenant1", "default");
+        Application app4 = tester.createApplication("app4", "tenant1", "default");
+        var version1 = tester.newSubmission(app1.id(), applicationPackage);
+        tester.deployNewSubmission(app1.id(), version1);
+        tester.deployNewSubmission(app2.id(), tester.newSubmission(app2.id(), applicationPackage));
+        tester.deployNewSubmission(app3.id(), tester.newSubmission(app3.id(), applicationPackage));
+        tester.deployNewSubmission(app4.id(), tester.newSubmission(app4.id(), applicationPackage));
 
         metricsReporter.maintain();
         assertEquals(0.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC));
 
         // 1 app fails system-test
-        tester.jobCompletion(component).application(app4).nextBuildNumber().uploadArtifact(applicationPackage).submit();
-        tester.deployAndNotify(app4.id().defaultInstance(), Optional.of(applicationPackage), false, systemTest);
+        tester.newSubmission(app4.id(), applicationPackage);
+        tester.triggerJobs();
+        tester.failDeployment(app4.id().defaultInstance(), systemTest);
 
         metricsReporter.maintain();
         assertEquals(25.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC));
     }
 
     @Test
-    public void test_deployment_average_duration() {
+    public void deployment_average_duration() {
         DeploymentTester tester = new DeploymentTester();
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .environment(Environment.prod)
@@ -116,7 +118,7 @@ public class MetricsReporterTest {
     }
 
     @Test
-    public void test_deployments_failing_upgrade() {
+    public void deployments_failing_upgrade() {
         DeploymentTester tester = new DeploymentTester();
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .environment(Environment.prod)
@@ -167,25 +169,25 @@ public class MetricsReporterTest {
     }
 
     @Test
-    public void test_deployment_warnings_metric() {
-        DeploymentTester tester = new DeploymentTester();
+    public void deployment_warnings_metric() {
+        var tester = new InternalDeploymentTester();
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .environment(Environment.prod)
                 .region("us-west-1")
                 .region("us-east-3")
                 .build();
         MetricsReporter reporter = createReporter(tester.controller());
-        Application application = tester.createApplication("app1", "tenant1", 1, 11L);
+        Application application = tester.createApplication("app1", "tenant1", "default");
         tester.configServer().generateWarnings(new DeploymentId(application.id().defaultInstance(), ZoneId.from("prod", "us-west-1")), 3);
         tester.configServer().generateWarnings(new DeploymentId(application.id().defaultInstance(), ZoneId.from("prod", "us-east-3")), 4);
-        tester.deployCompletely(application, applicationPackage);
+        tester.deployNewSubmission(application.id(), tester.newSubmission(application.id(), applicationPackage));
         reporter.maintain();
         assertEquals(4, getDeploymentWarnings(application.id().defaultInstance()));
     }
 
     @Test
-    public void test_build_time_reporting() {
-        InternalDeploymentTester tester = new InternalDeploymentTester();
+    public void build_time_reporting() {
+        var tester = new InternalDeploymentTester();
         ApplicationVersion version = tester.newSubmission();
         tester.deployNewSubmission(version);
         assertEquals(1000, version.buildTime().get().toEpochMilli());
@@ -197,8 +199,8 @@ public class MetricsReporterTest {
     }
 
     @Test
-    public void test_name_service_queue_size_metric() {
-        DeploymentTester tester = new DeploymentTester(new ControllerTester(), false);
+    public void name_service_queue_size_metric() {
+        var tester = new InternalDeploymentTester();
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .environment(Environment.prod)
                 .globalServiceId("default")
@@ -206,11 +208,11 @@ public class MetricsReporterTest {
                 .region("us-east-3")
                 .build();
         MetricsReporter reporter = createReporter(tester.controller());
-        Application application = tester.createApplication("app1", "tenant1", 1, 11L);
+        Application application = tester.createApplication("app1", "tenant1", "default");
         reporter.maintain();
         assertEquals("Queue is empty initially", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue());
 
-        tester.deployCompletely(application, applicationPackage);
+        tester.deployNewSubmission(application.id(), tester.newSubmission(application.id(), applicationPackage));
         reporter.maintain();
         assertEquals("Deployment queues name services requests", 6, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue());
 
@@ -220,13 +222,13 @@ public class MetricsReporterTest {
     }
 
     @Test
-    public void test_nodes_failing_system_upgrade() {
-        var tester = new DeploymentTester();
+    public void nodes_failing_system_upgrade() {
+        var tester = new ControllerTester();
         var reporter = createReporter(tester.controller());
         var zone1 = ZoneApiMock.fromId("prod.eu-west-1");
-        tester.controllerTester().zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone1));
+        tester.zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone1));
         var systemUpgrader = new SystemUpgrader(tester.controller(), Duration.ofDays(1),
-                                                new JobControl(tester.controllerTester().curator()));
+                                                new JobControl(tester.curator()));
         tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer);
 
         // System on initial version
@@ -265,14 +267,14 @@ public class MetricsReporterTest {
     }
 
     @Test
-    public void test_nodes_failing_os_upgrade() {
-        var tester = new DeploymentTester();
+    public void nodes_failing_os_upgrade() {
+        var tester = new ControllerTester();
         var reporter = createReporter(tester.controller());
         var zone = ZoneApiMock.fromId("prod.eu-west-1");
         var cloud = CloudName.defaultName();
-        tester.controllerTester().zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(zone));
+        tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(zone));
         var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1),
-                                        new JobControl(tester.controllerTester().curator()), CloudName.defaultName());;
+                                        new JobControl(tester.curator()), CloudName.defaultName());;
         var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1),
                                                        new JobControl(tester.controller().curator()));
         tester.configServer().bootstrap(List.of(zone.getId()), SystemApplication.configServerHost, SystemApplication.tenantHost);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index b4b4045819e..65cd4a08841 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -56,7 +56,7 @@ public class JobControllerApiHandlerHelperTest {
         // Revision 1 gets deployed everywhere.
         ApplicationVersion revision1 = tester.newSubmission();
         tester.deployNewSubmission(revision1);
-        assertEquals(2, tester.application().projectId().getAsLong());
+        assertEquals(1, tester.application().projectId().getAsLong());
 
         tester.clock().advance(Duration.ofMillis(1000));
         // Revision 2 gets deployed everywhere except in us-east-3.