Add new run status noTests, which is a success, but not quite as shiny

15 files changed, 77 insertions, 36 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 3be5345b377..0f7cbcee4ab 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -287,7 +287,7 @@ public class ApplicationController {
             if (oldest == null || version.isBefore(oldest))
                 oldest = version;
 
-            if (run.status() == RunStatus.success)
+            if (run.hasSucceeded())
                 return Optional.of(oldest);
         }
         // If no successful run was found, ask the node repository in the relevant zone.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index 0595245b3d7..cb2958745d0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -321,7 +321,7 @@ public class DeploymentStatus {
                                            .type(type).asList().stream()
                                            .flatMap(status -> RunList.from(status)
                                                                         .on(versions)
-                                                                        .status(RunStatus.success)
+                                                                        .matching(Run::hasSucceeded)
                                                                         .asList().stream()
                                                                         .map(Run::start))
                                            .min(naturalOrder());
@@ -860,7 +860,7 @@ public class DeploymentStatus {
                     Optional<Instant> end = Optional.empty();
                     for (Run run : job.runs().descendingMap().values()) {
                         if (run.versions().targetsMatch(change)) {
-                            if (run.status() == RunStatus.success) end = run.end();
+                            if (run.hasSucceeded()) end = run.end();
                         }
                         else if (dependent.equals(job())) // If strict completion, consider only last time this change was deployed.
                             break;
@@ -887,7 +887,7 @@ public class DeploymentStatus {
                     Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId));
                     return (dependent.equals(job()) ? job.lastTriggered().filter(run -> deployedAt.map(at -> ! run.start().isBefore(at)).orElse(false)).stream()
                                                     : job.runs().values().stream())
-                            .filter(run -> run.status() == RunStatus.success)
+                            .filter(Run::hasSucceeded)
                             .filter(run -> run.versions().targetsMatch(change))
                             .flatMap(run -> run.end().stream()).findFirst();
                 }
@@ -907,7 +907,7 @@ public class DeploymentStatus {
                                                                                                                          status.systemVersion)))
                                                             .orElseGet(() ->    (change.platform().isEmpty()    || change.platform().get().equals(run.versions().targetPlatform()))
                                                                              && (change.revision().isEmpty() || change.revision().get().equals(run.versions().targetRevision()))))
-                                  .status(RunStatus.success)
+                                  .matching(Run::hasSucceeded)
                                   .asList().stream()
                                   .map(run -> run.end().get())
                                   .max(naturalOrder());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index ce4bb70c174..28d9439b457 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -23,8 +23,6 @@ import com.yahoo.security.SignatureAlgorithm;
 import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.text.Text;
-import com.yahoo.vespa.flags.FetchVector;
-import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
@@ -36,7 +34,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
@@ -90,6 +87,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Nod
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed;
+import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
@@ -638,6 +636,7 @@ public class InternalStepRunner implements StepRunner {
         return Optional.of(running);
     }
 
+    @SuppressWarnings("fallthrough")
     private Optional<RunStatus> endTests(RunId id, boolean isSetup, DualLogger logger) {
         Optional<Deployment> deployment = deployment(id.application(), id.type());
         if (deployment.isEmpty()) {
@@ -679,12 +678,14 @@ public class InternalStepRunner implements StepRunner {
                 controller.jobController().updateTestReport(id);
                 return Optional.of(error);
             case NO_TESTS:
-                TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup);
-                logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'");
-                logger.log(INFO, "The test package must either contain basic HTTP tests under 'tests/<suite-name>/', " +
-                                 "or a Java test bundle under 'components/' with at least one test with the annotation " +
-                                 "for this suite. See docs.vespa.ai/en/testing.html for details.");
-                return Optional.of(allowNoTests(id.application()) ? running : testFailure);
+                if ( ! isSetup) { // TODO: consider changing this Later™
+                    TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup);
+                    logger.log(INFO, "No tests were found in the test package, for test suite '" + suite + "'");
+                    logger.log(INFO, "The test package should either contain basic HTTP tests under 'tests/<suite-name>/', " +
+                                     "or a Java test bundle under 'components/' with at least one test with the annotation " +
+                                     "for this suite. See docs.vespa.ai/en/testing.html for details.");
+                    return Optional.of(noTests);
+                }
             case SUCCESS:
                 logger.log("Tests completed successfully.");
                 controller.jobController().updateTestReport(id);
@@ -694,12 +695,6 @@ public class InternalStepRunner implements StepRunner {
         }
     }
 
-    private boolean allowNoTests(ApplicationId appId) {
-        return Flags.ALLOW_NO_TESTS.bindTo(controller.flagSource())
-                                   .with(FetchVector.Dimension.TENANT_ID, appId.tenant().value())
-                                   .value();
-    }
-
     private Optional<RunStatus> copyVespaLogs(RunId id, DualLogger logger) {
         if (deployment(id.application(), id.type()).isPresent())
             try {
@@ -835,6 +830,10 @@ public class InternalStepRunner implements StepRunner {
             case testFailure:
                 updater.accept("one or more verification tests against the deployment failed. Please review test output in the deployment job log.");
                 return;
+            case noTests:
+                controller.notificationsDb().setNotification(source, Notification.Type.deployment, Notification.Level.warning,
+                                                             "no tests were found for this job type. Please review test output in the deployment job log.");
+                return;
             case error:
             case endpointCertificateTimeout:
                 break;
@@ -849,6 +848,7 @@ public class InternalStepRunner implements StepRunner {
         switch (run.status()) {
             case running:
             case aborted:
+            case noTests:
             case success:
                 return Optional.empty();
             case nodeAllocationFailure:
@@ -861,11 +861,11 @@ public class InternalStepRunner implements StepRunner {
                 return Optional.of(mails.testFailure(run.id(), recipients));
             case error:
             case endpointCertificateTimeout:
-                return Optional.of(mails.systemError(run.id(), recipients));
+                break;
             default:
                 logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'");
-                return Optional.of(mails.systemError(run.id(), recipients));
         }
+        return Optional.of(mails.systemError(run.id(), recipients));
     }
 
     /** Returns the deployment of the real application in the zone of the given job, if it exists. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 74730a78e31..b9ce094c1c0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -400,7 +400,7 @@ public class JobController {
                 locked(id.application(), id.type(), runs -> {
                     runs.put(run.id(), finishedRun);
                     long last = id.number();
-                    long successes = runs.values().stream().filter(old -> old.status() == RunStatus.success).count();
+                    long successes = runs.values().stream().filter(Run::hasSucceeded).count();
                     var oldEntries = runs.entrySet().iterator();
                     for (var old = oldEntries.next();
                             old.getKey().number() <= last - historyLength
@@ -409,7 +409,7 @@ public class JobController {
 
                         // Make sure we keep the last success and the first failing
                         if (     successes == 1
-                            &&   old.getValue().status() == RunStatus.success
+                            &&   old.getValue().hasSucceeded()
                             && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) {
                             oldEntries.next();
                             continue;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
index 639702128d3..387ea755414 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
@@ -120,7 +120,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> {
 
     /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */
     public JobList successOn(Versions versions) {
-        return matching(job -> ! RunList.from(job).status(RunStatus.success).on(versions).isEmpty());
+        return matching(job -> ! RunList.from(job).matching(Run::hasSucceeded).on(versions).isEmpty());
     }
 
     // ----------------------------------- JobRun filtering
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java
index 874b1828f5f..e95e515685f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java
@@ -21,6 +21,7 @@ public class JobMetrics {
     public static final String deploymentFailure = "deployment.deploymentFailure";
     public static final String convergenceFailure = "deployment.convergenceFailure";
     public static final String testFailure = "deployment.testFailure";
+    public static final String noTests = "deployment.noTests";
     public static final String error = "deployment.error";
     public static final String abort = "deployment.abort";
     public static final String success = "deployment.success";
@@ -56,6 +57,7 @@ public class JobMetrics {
             case deploymentFailed: return deploymentFailure;
             case installationFailed: return convergenceFailure;
             case testFailure: return testFailure;
+            case noTests: return noTests;
             case error: return error;
             case aborted: return abort;
             case success: return success;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java
index aad5d510261..1a53b26f9c2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java
@@ -60,7 +60,7 @@ public class JobStatus {
     }
 
     public boolean isSuccess() {
-        return lastStatus().isPresent() && lastStatus().get() == RunStatus.success;
+        return lastCompleted.map(last -> ! last.hasFailed()).orElse(false);
     }
 
     public boolean isRunning() {
@@ -90,18 +90,17 @@ public class JobStatus {
 
     static Optional<Run> lastSuccess(NavigableMap<RunId, Run> runs) {
         return runs.descendingMap().values().stream()
-                   .filter(run -> run.status() == RunStatus.success)
+                   .filter(run -> run.hasEnded() && ! run.hasFailed())
                    .findFirst();
     }
 
     static Optional<Run> firstFailing(NavigableMap<RunId, Run> runs) {
         Run failed = null;
-        loop: for (Run run : runs.descendingMap().values())
-            switch (run.status()) {
-                case running: continue loop;
-                case success: break loop;
-                default: failed = run;
-            }
+        for (Run run : runs.descendingMap().values()) {
+            if ( ! run.hasEnded()) continue;
+            if ( ! run.hasFailed()) break;
+            failed = run;
+        }
         return Optional.ofNullable(failed);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
index dcde21c8cf5..03cc6c6ba8d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
@@ -13,6 +13,7 @@ import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted;
+import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
@@ -80,8 +81,9 @@ public class Run {
 
         EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps);
         steps.put(step.get(), stepInfo.with(Step.Status.of(status)));
-        return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, this.status == running ? status : this.status,
-                       lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason);
+        RunStatus newStatus = hasFailed() || status == running ? this.status : status;
+        return new Run(id, steps, versions, isRedeployment, start, end, sleepUntil, newStatus, lastTestRecord,
+                       lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate, dryRun, reason);
     }
 
     /** Returns a new Run with a new start time*/
@@ -210,7 +212,7 @@ public class Run {
 
     /** Returns whether the run has failed, and should switch to its run-always steps. */
     public boolean hasFailed() {
-        return status != running && status != success;
+        return status != running && status != success && status != noTests;
     }
 
     /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */
@@ -218,6 +220,8 @@ public class Run {
         return end.isPresent();
     }
 
+    public boolean hasSucceeded() { return hasEnded() && ! hasFailed(); }
+
     /** Returns the target, and possibly source, versions for this run. */
     public Versions versions() {
         return versions;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
index 0bb4a30425e..9ca634b19fd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
@@ -26,6 +26,9 @@ public enum RunStatus {
     /** The verification tests failed. */
     testFailure,
 
+    /** No tests, for a test job. */
+    noTests,
+
     /** An unexpected error occurred. */
     error,
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index 14d6b35f6ad..82d154dcf03 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -113,6 +113,7 @@ public enum Step {
                 case success : throw new AssertionError("Unexpected run status '" + status + "'!");
                 case reset   :
                 case aborted : return unfinished;
+                case noTests :
                 case running : return succeeded;
                 default      : return failed;
             }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 731e5790a64..c8afb727edd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -35,6 +35,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentF
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed;
+import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
@@ -342,6 +343,7 @@ class RunSerializer {
             case deploymentFailed           : return "deploymentFailed";
             case installationFailed         : return "installationFailed";
             case testFailure                : return "testFailure";
+            case noTests                    : return "noTests";
             case error                      : return "error";
             case success                    : return "success";
             case aborted                    : return "aborted";
@@ -359,6 +361,7 @@ class RunSerializer {
             case "endpointCertificateTimeout" : return endpointCertificateTimeout;
             case "deploymentFailed"           : return deploymentFailed;
             case "installationFailed"         : return installationFailed;
+            case "noTests"                    : return noTests;
             case "testFailure"                : return testFailure;
             case "error"                      : return error;
             case "success"                    : return success;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java
index 1fe5ebfa9a9..6ebe010326c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java
@@ -15,6 +15,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 
 import static java.util.stream.Collectors.toList;
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index f18733fbcde..9685decda84 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 98cbf33fb2b..6552ca4a2fb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
@@ -47,6 +48,8 @@ import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error;
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info;
@@ -55,6 +58,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.app
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed;
+import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.noTests;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
@@ -62,6 +66,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
 import static java.time.temporal.ChronoUnit.SECONDS;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -269,6 +274,27 @@ public class InternalStepRunnerTest {
     }
 
     @Test
+    public void noTestsThenErrorIsError() {
+        RunId id = app.startSystemTestTests();
+        Run run = tester.jobs().run(id).get();
+        run = run.with(noTests, new LockedStep(() -> { }, Step.endTests));
+        assertFalse(run.hasFailed());
+        run = run.with(RunStatus.error, new LockedStep(() -> { }, Step.deactivateReal));
+        assertTrue(run.hasFailed());
+        assertEquals(RunStatus.error, run.status());
+    }
+
+    @Test
+    public void noTestsThenSuccessIsNoTests() {
+        RunId id = app.startSystemTestTests();
+        tester.cloud().set(Status.NO_TESTS);
+        tester.runner().run();
+        assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.endTests));
+        Run run = tester.jobs().run(id).get();
+        assertEquals(noTests, run.status());
+    }
+
+    @Test
     public void testsFailIfTesterRestarts() {
         RunId id = app.startSystemTestTests();
         tester.cloud().set(TesterCloud.Status.NOT_STARTED);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index 9ccd7244392..610d8d4ca9a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -413,6 +413,7 @@ public class JobRunnerTest {
         assertEquals(1, metric.getMetric(context::equals, JobMetrics.nodeAllocationFailure).get().intValue());
         assertEquals(1, metric.getMetric(context::equals, JobMetrics.endpointCertificateTimeout).get().intValue());
         assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue());
+        assertEquals(1, metric.getMetric(context::equals, JobMetrics.noTests).get().intValue());
     }
 
     private void start(JobController jobs, ApplicationId id, JobType type) {