Require tests to be run when test job is declared

5 files changed, 60 insertions, 50 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index fb7a3fc4ba5..632eb556444 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -662,7 +662,10 @@ public class InternalStepRunner implements StepRunner {
                                      "or a Java test bundle under 'components/' with at least one test with the annotation " +
                                      "for this suite. See docs.vespa.ai/en/testing.html for details.");
                     controller.jobController().updateTestReport(id);
-                    return Optional.of(noTests);
+
+                    DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec();
+                    boolean requireTests = spec.steps().stream().anyMatch(step -> step.concerns(id.type().environment()));
+                    return Optional.of(requireTests ? testFailure : noTests);
                 }
             case SUCCESS:
                 logger.log("Tests completed successfully.");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index ae4b6259da1..27951d95689 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -445,7 +445,6 @@ public class DeploymentContext {
         tester.cloud().set(noTests ? Status.NO_TESTS : Status.FAILURE);
         runner.advance(currentRun(job));
         assertTrue(jobs.run(id).get().hasEnded());
-        assertEquals(noTests, jobs.run(id).get().hasSucceeded());
         assertTrue(configServer().nodeRepository().list(job.type().zone(), NodeFilter.all().applications(TesterId.of(instanceId).id())).isEmpty());
 
         return this;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index beda9bd551d..d602ee8cde3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -2267,4 +2267,16 @@ public class DeploymentTriggerTest {
         assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet());
     }
 
+    @Test
+    public void testNoTests() {
+        DeploymentContext app = tester.newDeploymentContext();
+        app.submit(new ApplicationPackageBuilder().systemTest().region("us-east-3").build());
+
+        // Declared tests must have run actual tests to succeed.
+        app.failTests(systemTest, true);
+        assertFalse(tester.jobs().last(app.instanceId(), systemTest).get().hasSucceeded());
+        app.failTests(stagingTest, true);
+        assertTrue(tester.jobs().last(app.instanceId(), stagingTest).get().hasSucceeded());
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
index 8db6bdf9a4a..279bd289c00 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
@@ -26,8 +26,7 @@ public class BadgeApiTest extends ControllerContainerTest {
     public void testBadgeApi() throws IOException {
         ContainerTester tester = new ContainerTester(container, responseFiles);
         var application = new DeploymentTester(new ControllerTester(tester)).newDeploymentContext("tenant", "application", "default");
-        ApplicationPackage applicationPackage = new ApplicationPackageBuilder().systemTest()
-                                                                               .parallel("us-west-1", "aws-us-east-1a")
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder().parallel("us-west-1", "aws-us-east-1a")
                                                                                .test("us-west-1")
                                                                                .region("ap-southeast-1")
                                                                                .test("ap-southeast-1")
@@ -59,6 +58,8 @@ public class BadgeApiTest extends ControllerContainerTest {
                               Files.readString(Paths.get(responseFiles + "overview.svg")), 200);
         tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"),
                               Files.readString(Paths.get(responseFiles + "single-running.svg")), 200);
+        tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/system-test"),
+                              Files.readString(Paths.get(responseFiles + "running-test.svg")), 200);
         tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"),
                               Files.readString(Paths.get(responseFiles + "history.svg")), 200);
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg
index a0005ed6d76..46e4acaace6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg
@@ -1,4 +1,4 @@
-<svg xmlns='http://www.w3.org/2000/svg' width='763.7809900000001' height='20' role='img' aria-label='Deployment Status'>
+<svg xmlns='http://www.w3.org/2000/svg' width='689.25265' height='20' role='img' aria-label='Deployment Status'>
     <title>Deployment Status</title>
     <linearGradient id='light' x2='0' y2='100%'>
         <stop offset='0'  stop-color='#fff' stop-opacity='.5'/>
@@ -46,32 +46,29 @@
         <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />
     </linearGradient>
     <clipPath id='rounded'>
-        <rect width='763.7809900000001' height='20' rx='3' fill='#fff'/>
+        <rect width='689.25265' height='20' rx='3' fill='#fff'/>
     </clipPath>
     <g clip-path='url(#rounded)'>
-        <rect x='757.7809900000001' rx='3' width='8' height='20' fill='url(#shadow)'/>
-        <rect x='725.59036' rx='3' width='38.19063' height='20' fill='url(#run-on-success)'/>
-        <polygon points='635.8470950000001 0 635.8470950000001 20 734.59036 20 742.59036 0' fill='#00f844'/>
-        <rect x='635.8470950000001' rx='3' width='131.74345499999998' height='20' fill='url(#shade)'/>
-        <rect x='635.8470950000001' rx='3' width='8' height='20' fill='url(#shadow)'/>
-        <rect x='603.656465' rx='3' width='38.19063' height='20' fill='#bf103c'/>
-        <polygon points='486.981225 0 486.981225 20 612.656465 20 620.656465 0' fill='#00f844'/>
-        <rect x='486.981225' rx='3' width='158.67543' height='20' fill='url(#shade)'/>
-        <rect x='486.981225' rx='3' width='8' height='20' fill='url(#shadow)'/>
-        <rect x='348.865175' rx='3' width='144.11604999999997' height='20' fill='url(#run-on-success)'/>
-        <rect x='358.865175' rx='3' width='134.11604999999997' height='20' fill='url(#shade)'/>
-        <rect x='358.865175' rx='3' width='8' height='20' fill='url(#shadow)'/>
-        <rect x='326.674545' rx='3' width='38.19063' height='20' fill='#00f844'/>
-        <polygon points='237.71563000000003 0 237.71563000000003 20 335.674545 20 343.674545 0' fill='url(#run-on-failure)'/>
-        <rect x='237.71563000000003' rx='3' width='130.959105' height='20' fill='url(#shade)'/>
-        <rect x='237.71563000000003' rx='3' width='8' height='20' fill='url(#shadow)'/>
-        <rect x='153.18729000000002' rx='3' width='90.52834000000001' height='20' fill='url(#run-on-warning)'/>
-        <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#shade)'/>
+        <rect x='683.25265' rx='3' width='8' height='20' fill='url(#shadow)'/>
+        <rect x='651.06202' rx='3' width='38.19063' height='20' fill='url(#run-on-success)'/>
+        <polygon points='561.318755 0 561.318755 20 660.06202 20 668.06202 0' fill='#00f844'/>
+        <rect x='561.318755' rx='3' width='131.74345499999998' height='20' fill='url(#shade)'/>
+        <rect x='561.318755' rx='3' width='8' height='20' fill='url(#shadow)'/>
+        <rect x='529.128125' rx='3' width='38.19063' height='20' fill='#bf103c'/>
+        <polygon points='412.452885 0 412.452885 20 538.128125 20 546.128125 0' fill='#00f844'/>
+        <rect x='412.452885' rx='3' width='158.67543' height='20' fill='url(#shade)'/>
+        <rect x='412.452885' rx='3' width='8' height='20' fill='url(#shadow)'/>
+        <rect x='274.336835' rx='3' width='144.11604999999997' height='20' fill='url(#run-on-success)'/>
+        <rect x='284.336835' rx='3' width='134.11604999999997' height='20' fill='url(#shade)'/>
+        <rect x='284.336835' rx='3' width='8' height='20' fill='url(#shadow)'/>
+        <rect x='252.146205' rx='3' width='38.19063' height='20' fill='#00f844'/>
+        <polygon points='163.18729000000002 0 163.18729000000002 20 261.146205 20 269.146205 0' fill='url(#run-on-failure)'/>
+        <rect x='163.18729000000002' rx='3' width='130.959105' height='20' fill='url(#shade)'/>
         <rect width='169.18729000000002' height='20' fill='#404040'/>
         <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/>
         <rect width='2' height='20' fill='url(#left-light)'/>
-        <rect x='761.7809900000001' width='2' height='20' fill='url(#right-shadow)'/>
-        <rect width='763.7809900000001' height='20' fill='url(#light)'/>
+        <rect x='687.25265' width='2' height='20' fill='url(#right-shadow)'/>
+        <rect width='689.25265' height='20' fill='url(#light)'/>
     </g>
     <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'>
         <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'>
@@ -96,29 +93,27 @@
         </svg>
         <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text>
         <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text>
-        <text font-size='11' x='206.95146000000003' y='15' fill='#000' fill-opacity='.4' textLength='62.52834000000001'>system-test</text>
-        <text font-size='11' x='206.45146000000003' y='14' fill='#fff' textLength='62.52834000000001'>system-test</text>
-        <text font-size='11' x='276.60659250000003' y='15' fill='#000' fill-opacity='.4' textLength='52.781925'>us-west-1</text>
-        <text font-size='11' x='276.10659250000003' y='14' fill='#fff' textLength='52.781925'>us-west-1</text>
-        <text font-size='9' x='323.08605000000006' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text>
-        <text font-size='9' x='322.58605000000006' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text>
-        <text font-size='9' x='351.26986000000005' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text>
-        <text font-size='9' x='350.76986000000005' y='14' fill='#fff' textLength='16.190630000000002'>test</text>
-        <text font-size='11' x='412.334705' y='15' fill='#000' fill-opacity='.4' textLength='81.93905999999998'>aws-us-east-1a</text>
-        <text font-size='11' x='411.834705' y='14' fill='#fff' textLength='81.93905999999998'>aws-us-east-1a</text>
-        <text font-size='9' x='473.39273000000003' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text>
-        <text font-size='9' x='472.89273000000003' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text>
-        <text font-size='11' x='539.73035' y='15' fill='#000' fill-opacity='.4' textLength='80.49825'>ap-southeast-1</text>
-        <text font-size='11' x='539.23035' y='14' fill='#fff' textLength='80.49825'>ap-southeast-1</text>
-        <text font-size='9' x='600.06797' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text>
-        <text font-size='9' x='599.56797' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text>
-        <text font-size='9' x='628.25178' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text>
-        <text font-size='9' x='627.75178' y='14' fill='#fff' textLength='16.190630000000002'>test</text>
-        <text font-size='11' x='675.1302325' y='15' fill='#000' fill-opacity='.4' textLength='53.566275'>eu-west-1</text>
-        <text font-size='11' x='674.6302325' y='14' fill='#fff' textLength='53.566275'>eu-west-1</text>
-        <text font-size='9' x='722.0018650000001' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text>
-        <text font-size='9' x='721.5018650000001' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text>
-        <text font-size='9' x='750.1856750000001' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text>
-        <text font-size='9' x='749.6856750000001' y='14' fill='#fff' textLength='16.190630000000002'>test</text>
+        <text font-size='11' x='202.07825250000002' y='15' fill='#000' fill-opacity='.4' textLength='52.781925'>us-west-1</text>
+        <text font-size='11' x='201.57825250000002' y='14' fill='#fff' textLength='52.781925'>us-west-1</text>
+        <text font-size='9' x='248.55771000000001' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text>
+        <text font-size='9' x='248.05771000000001' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text>
+        <text font-size='9' x='276.74152000000004' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text>
+        <text font-size='9' x='276.24152000000004' y='14' fill='#fff' textLength='16.190630000000002'>test</text>
+        <text font-size='11' x='337.806365' y='15' fill='#000' fill-opacity='.4' textLength='81.93905999999998'>aws-us-east-1a</text>
+        <text font-size='11' x='337.306365' y='14' fill='#fff' textLength='81.93905999999998'>aws-us-east-1a</text>
+        <text font-size='9' x='398.86439' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text>
+        <text font-size='9' x='398.36439' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text>
+        <text font-size='11' x='465.20201' y='15' fill='#000' fill-opacity='.4' textLength='80.49825'>ap-southeast-1</text>
+        <text font-size='11' x='464.70201' y='14' fill='#fff' textLength='80.49825'>ap-southeast-1</text>
+        <text font-size='9' x='525.53963' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text>
+        <text font-size='9' x='525.03963' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text>
+        <text font-size='9' x='553.72344' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text>
+        <text font-size='9' x='553.22344' y='14' fill='#fff' textLength='16.190630000000002'>test</text>
+        <text font-size='11' x='600.6018925' y='15' fill='#000' fill-opacity='.4' textLength='53.566275'>eu-west-1</text>
+        <text font-size='11' x='600.1018925' y='14' fill='#fff' textLength='53.566275'>eu-west-1</text>
+        <text font-size='9' x='647.473525' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text>
+        <text font-size='9' x='646.973525' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text>
+        <text font-size='9' x='675.657335' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text>
+        <text font-size='9' x='675.157335' y='14' fill='#fff' textLength='16.190630000000002'>test</text>
     </g>
 </svg>