Merge pull request #28940 from vespa-engine/mpolden/read-endpoint-config-flag

Read endpoint-config flag

6 files changed, 62 insertions, 74 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 51e20d0017c..f9798fb2559 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -14,9 +14,9 @@ import com.yahoo.config.provision.zone.AuthMethod;
 import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.flags.BooleanFlag;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
@@ -82,8 +82,7 @@ public class RoutingController {
     private final Controller controller;
     private final RoutingPolicies routingPolicies;
     private final RotationRepository rotationRepository;
-    private final BooleanFlag generatedEndpoints;
-    private final BooleanFlag legacyEndpoints;
+    private final StringFlag endpointConfig;
 
     public RoutingController(Controller controller, RotationsConfig rotationsConfig) {
         this.controller = Objects.requireNonNull(controller, "controller must be non-null");
@@ -91,8 +90,7 @@ public class RoutingController {
         this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"),
                                                          controller.applications(),
                                                          controller.curator());
-        this.generatedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource());
-        this.legacyEndpoints = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource());
+        this.endpointConfig = Flags.ENDPOINT_CONFIG.bindTo(controller.flagSource());
     }
 
     /** Create a routing context for given deployment */
@@ -124,15 +122,17 @@ public class RoutingController {
 
     /** Returns the endpoint config to use for given instance */
     public EndpointConfig endpointConfig(ApplicationId instance) {
-        // TODO(mpolden): Switch to reading endpoint-config flag
-        if (legacyEndpointsEnabled(instance)) {
-            if (generatedEndpointsEnabled(instance)) {
-                return EndpointConfig.combined;
-            } else {
-                return EndpointConfig.legacy;
-            }
-        }
-        return EndpointConfig.generated;
+        String flagValue = endpointConfig.with(FetchVector.Dimension.TENANT_ID, instance.tenant().value())
+                                         .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized())
+                                         .with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm())
+                                         .value();
+        return switch (flagValue) {
+            case "legacy" -> EndpointConfig.legacy;
+            case "combined" -> EndpointConfig.combined;
+            case "generated" -> EndpointConfig.generated;
+            default -> throw new IllegalArgumentException("Invalid endpoint-config flag value: '" + flagValue + "', must be " +
+                                                          "'legacy', 'combined' or 'generated'");
+        };
     }
 
     /** Prepares and returns the endpoints relevant for given deployment */
@@ -600,20 +600,6 @@ public class RoutingController {
         return Collections.unmodifiableList(routingMethods);
     }
 
-    private boolean generatedEndpointsEnabled(ApplicationId instance) {
-        return generatedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm())
-                                 .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value())
-                                 .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized())
-                                 .value();
-    }
-
-    private boolean legacyEndpointsEnabled(ApplicationId instance) {
-        return legacyEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm())
-                              .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value())
-                              .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized())
-                              .value();
-    }
-
     private static void requireGeneratedEndpoints(GeneratedEndpointList generatedEndpoints, boolean declared) {
         if (generatedEndpoints.asList().stream().anyMatch(ge -> ge.declared() != declared)) {
             throw new IllegalStateException("All generated endpoints require declared=" + declared +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java
index 7faaee95abb..378b92d37ce 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java
@@ -303,8 +303,7 @@ public class EndpointCertificatesTest {
 
     @Test
     public void assign_certificate_from_pool() {
-        tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true);
-        tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false);
+        setEndpointConfig(tester, EndpointConfig.generated);
         try {
             addCertificateToPool("bad0f00d", UnassignedCertificate.State.requested, tester);
             endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock);
@@ -340,7 +339,7 @@ public class EndpointCertificatesTest {
 
     @Test
     public void certificate_migration() {
-        // An application is initially deployed with legacy config
+        // An application is initially deployed with legacy config (the default)
         ZoneId zone1 = ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1c"));
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region(zone1.region())
                                                                                .build();
@@ -408,8 +407,7 @@ public class EndpointCertificatesTest {
                      devCert0.requestedDnsSans());
 
         // Application switches to combined config
-        tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true);
-        tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), true);
+        setEndpointConfig(tester, EndpointConfig.combined);
         tester.clock().advance(Duration.ofHours(1));
         assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()),
                      endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock),
@@ -420,7 +418,7 @@ public class EndpointCertificatesTest {
                    "Certificate is not assigned at application level");
 
         // Application switches to generated config
-        tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false);
+        setEndpointConfig(tester, EndpointConfig.generated);
         tester.clock().advance(Duration.ofHours(1));
         assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()),
                      endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock),
@@ -451,8 +449,7 @@ public class EndpointCertificatesTest {
         assertEquals(poolCertId1, prodCertificate.generatedId().get());
 
         // Application switches back to legacy config
-        tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), false);
-        tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), true);
+        setEndpointConfig(tester, EndpointConfig.legacy);
         EndpointCertificate reissuedCertificate = endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock);
         assertEquals(certificate.requestedDnsSans(), reissuedCertificate.requestedDnsSans());
         assertTrue(tester.curator().readAssignedCertificate(deployment0.applicationId()).isPresent(), "Certificate is assigned at instance level again");
@@ -460,6 +457,10 @@ public class EndpointCertificatesTest {
                    "Certificate is still assigned at application level"); // Not removed because the assumption is that the application will eventually migrate back
     }
 
+    private void setEndpointConfig(ControllerTester tester, EndpointConfig config) {
+        tester.flagSource().withStringFlag(Flags.ENDPOINT_CONFIG.id(), config.name());
+    }
+
     private void addCertificateToPool(String id, UnassignedCertificate.State state, ControllerTester tester) {
         EndpointCertificate cert = new EndpointCertificate(testKeyName,
                                                            testCertName,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
index f551a99829e..fe9e9b28655 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
@@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock;
+import com.yahoo.vespa.hosted.controller.routing.EndpointConfig;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -262,9 +263,8 @@ public class EndpointCertificateMaintainerTest {
     }
 
     private void prepareCertificatePool(int numCertificates) {
-        ((InMemoryFlagSource)tester.controller().flagSource()).withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), numCertificates);
-        ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true);
-        ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false);
+        ((InMemoryFlagSource) tester.controller().flagSource()).withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), numCertificates);
+        ((InMemoryFlagSource) tester.controller().flagSource()).withStringFlag(Flags.ENDPOINT_CONFIG.id(), EndpointConfig.generated.name());
 
         // Provision certificates
         for (int i = 0; i < numCertificates; i++) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index a10bfd46b0c..a671f567895 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -46,7 +46,6 @@ import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue;
 import com.yahoo.vespa.hosted.controller.dns.RemoveRecords;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
 
@@ -113,6 +112,11 @@ public class RoutingPoliciesTest {
     private static final ZoneId zone5 = zoneApi5.getId();
     private static final ZoneId zone6 = zoneApi6.getId();
 
+    private static final ZoneId testZonePublic = ZoneId.from("test", "aws-us-east-2c");
+    private static final ZoneId stagingZonePublic = ZoneId.from("staging", "aws-us-east-3c");
+    private static final ZoneId testZoneMain = ZoneId.from("test", "us-east-1");
+    private static final ZoneId stagingZoneMain = ZoneId.from("staging", "us-east-3");
+
     private static final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
                                                                                             .region(zone2.region())
                                                                                             .build();
@@ -400,20 +404,24 @@ public class RoutingPoliciesTest {
     }
 
     @Test
-    @Disabled // TODO(mpolden): Enable this test when we start creating generated endpoints for shared routing
-    void zone_routing_policies_with_shared_routing_and_generated_endpoint() {
+    void zone_routing_policies_with_shared_routing_and_generated_endpoint_config_and_token() {
         var tester = new RoutingPoliciesTester(new DeploymentTester(), false);
         var context = tester.newDeploymentContext("tenant1", "app1", "default");
-        tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2);
-        tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true);
+        tester.setEndpointConfig(EndpointConfig.generated);
         addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester);
         ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
                                                                            .region(zone2.region())
                                                                            .container("c0", AuthMethod.mtls, AuthMethod.token)
                                                                            .build();
+        tester.provisionLoadBalancers(1, context.instanceId(), true,
+                                      testZoneMain, stagingZoneMain, zone1, zone2);
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
-        assertEquals(List.of("c0a25b7c.cafed00d.z.vespa.oath.cloud",
-                             "dc5e383c.cafed00d.z.vespa.oath.cloud"),
+        // This only creates wildcard endpoint names in DNS because legacy names in shared routing-mode use a static
+        // wildcard DNS record pointing to the routing layer
+        assertEquals(List.of("a9c8c045.cafed00d.z.vespa.oath.cloud",
+                             "dc5e383c.cafed00d.z.vespa.oath.cloud",
+                             "ebd395b6.cafed00d.z.vespa.oath.cloud",
+                             "ee82b867.cafed00d.z.vespa.oath.cloud"),
                      tester.recordNames());
     }
 
@@ -1215,9 +1223,7 @@ public class RoutingPoliciesTest {
                                                                            .container("c0", AuthMethod.mtls)
                                                                            .endpoint("foo", "c0")
                                                                            .build();
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c"));
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c"));
-        tester.provisionLoadBalancers(1, context.instanceId(), zone1);
+        tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic, zone1);
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy();
         assertEquals(List.of("a9c8c045.cafed00d.g.vespa-app.cloud",
                              "ebd395b6.cafed00d.z.vespa-app.cloud",
@@ -1229,8 +1235,7 @@ public class RoutingPoliciesTest {
                                                         .container("c0", AuthMethod.mtls, AuthMethod.token)
                                                         .endpoint("foo", "c0")
                                                         .build();
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c"));
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c"));
+        tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic);
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy();
         // Additional zone- and global-scoped endpoints are added (token)
         assertEquals(List.of("a9c8c045.cafed00d.g.vespa-app.cloud",
@@ -1246,8 +1251,7 @@ public class RoutingPoliciesTest {
                                                         .endpoint("foo", "c0")
                                                         .endpoint("bar", "c0")
                                                         .build();
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c"));
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c"));
+        tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic);
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy();
         List<String> expectedRecords = List.of("a9c8c045.cafed00d.g.vespa-app.cloud",
                                                "aa7591aa.cafed00d.g.vespa-app.cloud",
@@ -1259,8 +1263,7 @@ public class RoutingPoliciesTest {
         assertEquals(expectedRecords, tester.recordNames());
 
         // No change on redeployment
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c"));
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c"));
+        tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic);
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy();
         assertEquals(expectedRecords, tester.recordNames());
     }
@@ -1281,8 +1284,7 @@ public class RoutingPoliciesTest {
         tester.provisionLoadBalancers(1, context.instanceId(), zone1);
         // ConfigServerMock provisions a load balancer for the "default" cluster, but in this scenario we need full
         // control over the load balancer name because "default" has no special treatment when using generated endpoints
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c"));
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c"));
+        tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic);
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy();
         tester.assertTargets(context.instance().id(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"),
                              0, Map.of(zone1, 1L), true);
@@ -1297,8 +1299,7 @@ public class RoutingPoliciesTest {
                                                         .container("c0", AuthMethod.mtls)
                                                         .endpoint("foo", "c0")
                                                         .build();
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c"));
-        tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c"));
+        tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic);
         tester.provisionLoadBalancers(1, context.instanceId(), zone2);
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy();
         assertEquals(List.of("a6414896.cafed00d.aws-eu-west-1.w.vespa-app.cloud",
@@ -1516,8 +1517,7 @@ public class RoutingPoliciesTest {
         }
 
         public RoutingPoliciesTester setEndpointConfig(EndpointConfig config) {
-            tester.controllerTester().flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), config.supportsLegacy());
-            tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), config.supportsGenerated());
+            tester.controllerTester().flagSource().withStringFlag(Flags.ENDPOINT_CONFIG.id(), config.name());
             return this;
         }
 
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 56cd06d3b35..d7073ebc0eb 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -434,7 +434,7 @@ public class Flags {
             List.of("mpolden", "tokle"), "2023-10-06", "2024-02-01",
             "Set the endpoint config to use for an application. Must be 'legacy', 'combined' or 'generated'. See EndpointConfig for further details",
             "Takes effect on next deployment through controller",
-            APPLICATION_ID);
+            TENANT_ID, APPLICATION_ID, INSTANCE_ID);
 
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index 23e7fe16797..6c4be09c489 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -111,7 +111,7 @@ public class NodeFailer extends NodeRepositoryMaintainer {
 
         for (Node node : activeNodes) {
             Instant graceTimeStart = clock().instant().minus(nodeRepository().nodes().suspended(node) ? suspendedDownTimeLimit : downTimeLimit);
-            if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node) && !undergoingCmr(node)) {
+            if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node) && !affectedByMaintenance(node)) {
                 // Allow a grace period after node re-activation
                 if (!node.history().hasEventAfter(History.Event.Type.activated, graceTimeStart))
                     failingNodes.add(new FailingNode(node, "Node has been down longer than " + downTimeLimit));
@@ -146,7 +146,7 @@ public class NodeFailer extends NodeRepositoryMaintainer {
     /** Returns whether node has any kind of hardware issue */
     static boolean hasHardwareIssue(Node node, NodeList allNodes) {
         Node host = node.parentHostname().flatMap(allNodes::node).orElse(node);
-        return reasonsToFailHost(host).size() > 0;
+        return !reasonsToFailHost(host).isEmpty();
     }
 
     private boolean applicationSuspended(Node node) {
@@ -159,17 +159,18 @@ public class NodeFailer extends NodeRepositoryMaintainer {
         }
     }
 
-    private boolean undergoingCmr(Node node) {
+    /** Is a maintenance event affecting this node? */
+    private boolean affectedByMaintenance(Node node) {
         return node.reports().getReport("vcmr")
-                .map(report ->
-                        SlimeUtils.entriesStream(report.getInspector().field("upcoming"))
-                                .anyMatch(cmr -> {
-                                    var startTime = cmr.field("plannedStartTime").asLong();
-                                    var endTime = cmr.field("plannedEndTime").asLong();
-                                    var now = clock().instant().getEpochSecond();
-                                    return now > startTime && now < endTime;
-                                })
-                ).orElse(false);
+                   .map(report ->
+                                SlimeUtils.entriesStream(report.getInspector().field("upcoming"))
+                                          .anyMatch(cmr -> {
+                                              var startTime = cmr.field("plannedStartTime").asLong();
+                                              var endTime = cmr.field("plannedEndTime").asLong();
+                                              var now = clock().instant().getEpochSecond();
+                                              return now > startTime && now < endTime;
+                                          })
+                   ).orElse(false);
     }
 
     /** Is the node and all active children suspended? */