Require certification of scheduled OS upgrades

4 files changed, 91 insertions, 10 deletions

diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
index 1b1c4e3bf1a..84227b4fd9f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
@@ -23,6 +23,7 @@ import java.util.Map;
 import java.util.Optional;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
@@ -61,7 +62,15 @@ public class OsUpgradeSchedulerTest {
         Version version1 = Version.fromString("7.0.0.20220301");
         tester.clock().advance(Duration.ofDays(14));
         assertEquals("2022-03-01T09:05:00", formatInstant(tester.clock().instant()));
-        assertEquals(version1, scheduler.changeIn(cloud, tester.clock().instant()).get().version());
+
+        // Change does not become available until certification
+        assertFalse(scheduler.changeIn(cloud, tester.clock().instant()).isPresent());
+        Version systemVersion = tester.controller().readSystemVersion();
+        Version olderThanSystemVersion = new Version(systemVersion.getMajor(), systemVersion.getMinor() - 1, systemVersion.getMicro());
+        tester.controller().os().certify(version1, cloud, olderThanSystemVersion);
+
+        // Change is now certified
+        assertEquals(version1, scheduler.changeIn(cloud, tester.clock().instant()).get().osVersion().version());
         scheduler.maintain();
         assertEquals(version0,
                      tester.controller().os().target(cloud).get().osVersion().version(),
@@ -97,9 +106,11 @@ public class OsUpgradeSchedulerTest {
         assertEquals(version1, tester.controller().os().target(cloud).get().osVersion().version());
 
         // Estimate next change
+        Version expected = Version.fromString("7.0.0.20220426");
+        tester.controller().os().certify(expected, cloud, systemVersion);
         Optional<OsUpgradeScheduler.Change> nextChange = scheduler.changeIn(cloud, tester.clock().instant());
         assertTrue(nextChange.isPresent());
-        assertEquals("7.0.0.20220426", nextChange.get().version().toFullString());
+        assertEquals(expected, nextChange.get().osVersion().version());
         assertEquals("2022-04-27T07:00:00", formatInstant(nextChange.get().scheduleAt()));
     }
 
@@ -125,14 +136,14 @@ public class OsUpgradeSchedulerTest {
         assertEquals(version0, tester.controller().os().target(cloud).get().osVersion().version());
         // Cool-down passes
         tester.clock().advance(Duration.ofDays(1));
-        assertEquals(version1, scheduler.changeIn(cloud, tester.clock().instant()).get().version());
+        assertEquals(version1, scheduler.changeIn(cloud, tester.clock().instant()).get().osVersion().version());
         scheduler.maintain();
         assertEquals(version1, tester.controller().os().target(cloud).get().osVersion().version());
 
         // Estimate next change
         Optional<OsUpgradeScheduler.Change> nextChange = scheduler.changeIn(cloud, tester.clock().instant());
         assertTrue(nextChange.isPresent());
-        assertEquals("7.0.0.20220426", nextChange.get().version().toFullString());
+        assertEquals("7.0.0.20220426", nextChange.get().osVersion().version().toFullString());
         assertEquals("2022-04-27T02:00:00", formatInstant(nextChange.get().scheduleAt()));
     }
 
@@ -153,9 +164,18 @@ public class OsUpgradeSchedulerTest {
         tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.stable,
                                                                                Instant.parse("2021-06-21T23:59:00.00Z")));
         scheduleUpgradeAfter(Duration.ZERO, version0, scheduler, tester);
-        OsUpgradeScheduler.Change nextChange = scheduler.changeIn(cloud, tester.clock().instant()).get();
-        assertEquals(version1, nextChange.version());
-        assertEquals("2021-06-22T07:00:00", formatInstant(nextChange.scheduleAt()));
+
+        // No change yet because it hasn't been certified
+        Optional<OsUpgradeScheduler.Change> nextChange = scheduler.changeIn(cloud, tester.clock().instant());
+        assertFalse(nextChange.isPresent(), "No change");
+
+        // Change is certified and upgrade is scheduled
+        Version systemVersion = tester.controller().readSystemVersion();
+        tester.controller().os().certify(version1, cloud, systemVersion);
+        nextChange = scheduler.changeIn(cloud, tester.clock().instant());
+        assertTrue(nextChange.isPresent());
+        assertEquals(version1, nextChange.get().osVersion().version());
+        assertEquals("2021-06-22T07:00:00", formatInstant(nextChange.get().scheduleAt()));
         scheduleUpgradeAfter(Duration.ofHours(7), version1, scheduler, tester); // Inside trigger period
 
         // A newer version is triggered manually
@@ -183,7 +203,7 @@ public class OsUpgradeSchedulerTest {
         Version version1 = Version.fromString("8.1");
         tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.latest,
                 tester.clock().instant()));
-        assertEquals(version1, scheduler.changeIn(cloud, tester.clock().instant()).get().version());
+        assertEquals(version1, scheduler.changeIn(cloud, tester.clock().instant()).get().osVersion().version());
         assertEquals("2021-06-22T07:05:00", formatInstant(scheduler.changeIn(cloud, tester.clock().instant()).get().scheduleAt()),
                      "Not valid until cool-down period passes");
         scheduleUpgradeAfter(Duration.ZERO, version0, scheduler, tester);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
index f45c7bfcdfb..6644c3013ff 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
@@ -6,11 +6,16 @@ import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.zone.UpgradePolicy;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.versions.CertifiedOsVersion;
 import com.yahoo.vespa.hosted.controller.versions.OsVersion;
 import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
 import org.junit.jupiter.api.Test;
 
 import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -25,8 +30,7 @@ public class OsVersionStatusUpdaterTest {
     @Test
     void test_update() {
         ControllerTester tester = new ControllerTester();
-        OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1)
-        );
+        OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1));
         // Add all zones to upgrade policy
         UpgradePolicy.Builder upgradePolicy = UpgradePolicy.builder();
         for (ZoneApi zone : tester.zoneRegistry().zones().controllerUpgraded().zones()) {
@@ -58,6 +62,24 @@ public class OsVersionStatusUpdaterTest {
         assertTrue(osVersions.get(new OsVersion(version1, cloud)).isEmpty(), "No nodes on current target");
         assertFalse(osVersions.get(new OsVersion(Version.emptyVersion, otherCloud)).isEmpty(), "All nodes on unknown version");
         assertTrue(osVersions.get(new OsVersion(version1, otherCloud)).isEmpty(), "No nodes on current target");
+
+        // Updating status cleans up stale certifications
+        Set<OsVersion> knownVersions = osVersions.keySet();
+        List<OsVersion> versionsToCertify = new ArrayList<>(knownVersions);
+        versionsToCertify.addAll(List.of(new OsVersion(Version.fromString("95.0.1"), cloud),
+                                         new OsVersion(Version.fromString("98.0.2"), cloud)));
+        for (OsVersion version : versionsToCertify) {
+            tester.controller().os().certify(version.version(), version.cloud(), Version.fromString("1.2.3"));
+        }
+        assertEquals(knownVersions.size() + 2, certifiedOsVersions(tester).size());
+        statusUpdater.maintain();
+        assertEquals(knownVersions, certifiedOsVersions(tester));
+    }
+
+    private static Set<OsVersion> certifiedOsVersions(ControllerTester tester) {
+        return tester.controller().curator().readCertifiedOsVersions().stream()
+                     .map(CertifiedOsVersion::osVersion)
+                     .collect(Collectors.toSet());
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/CertifiedOsVersionSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/CertifiedOsVersionSerializerTest.java
new file mode 100644
index 00000000000..fcf5be08abf
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/CertifiedOsVersionSerializerTest.java
@@ -0,0 +1,30 @@
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.vespa.hosted.controller.versions.CertifiedOsVersion;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+import org.junit.jupiter.api.Test;
+
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author mpolden
+ */
+class CertifiedOsVersionSerializerTest {
+
+    @Test
+    public void serialization() {
+        Set<CertifiedOsVersion> certifiedVersion = Set.of(new CertifiedOsVersion(new OsVersion(Version.fromString("1.2.3"),
+                                                                                               CloudName.from("cloud1")),
+                                                                                 Version.fromString("4.5.6")),
+                                                          new CertifiedOsVersion(new OsVersion(Version.fromString("3.2.1"),
+                                                                                               CloudName.from("cloud2")),
+                                                                                 Version.fromString("6.5.4")));
+        CertifiedOsVersionSerializer serializer = new CertifiedOsVersionSerializer();
+        assertEquals(certifiedVersion, serializer.fromSlime(serializer.toSlime(certifiedVersion)));
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
index e569e0aca5b..15505dc3e95 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
@@ -79,6 +79,7 @@ public class OsApiTest extends ControllerContainerTest {
 
         // All nodes are initially on empty version
         upgradeAndUpdateStatus();
+
         // Upgrade OS to a different version in each cloud
         assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.2\", \"cloud\": \"cloud1\"}", Request.Method.PATCH),
                 "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.2\"}", 200);
@@ -111,6 +112,14 @@ public class OsApiTest extends ControllerContainerTest {
         assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.2\", \"cloud\": \"cloud1\", \"pin\": true}", Request.Method.PATCH),
                        "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.2 (pinned)\"}", 200);
 
+        // Certify an OS and Vespa version pair
+        assertResponse(new Request("http://localhost:8080/os/v1/certify/cloud1/7.5.2", "8.200.37", Request.Method.POST),
+                       "{\"message\":\"Certified 7.5.2 in cloud cloud1 as compatible with Vespa version 8.200.37\"}", 200);
+        assertResponse(new Request("http://localhost:8080/os/v1/certify/cloud1/7.5.2", "8.200.42", Request.Method.POST),
+                       "{\"message\":\"7.5.2 is already certified in cloud cloud1 as compatible with Vespa version 8.200.37. Leaving certification unchanged\"}", 200);
+        assertResponse(new Request("http://localhost:8080/os/v1/certify/cloud1/7.5.2", "", Request.Method.DELETE),
+                       "{\"message\":\"Removed certification of 7.5.2 in cloud cloud1\"}", 200);
+
         // Error: Missing fields
         assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.6\"}", Request.Method.PATCH),
                 "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Field 'cloud' is required\"}", 400);