Merge pull request #8551 from vespa-engine/freva/fixes

Minor HostProvisioner fixes 

7 files changed, 63 insertions, 15 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisionMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisionMaintainer.java
index d38738ff019..becad405415 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisionMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisionMaintainer.java
@@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -24,15 +27,19 @@ public class HostDeprovisionMaintainer extends Maintainer {
     private static final Logger log = Logger.getLogger(HostDeprovisionMaintainer.class.getName());
 
     private final HostProvisioner hostProvisioner;
+    private final BooleanFlag dynamicProvisioningEnabled;
 
-    public HostDeprovisionMaintainer(
-            NodeRepository nodeRepository, Duration interval, JobControl jobControl, HostProvisioner hostProvisioner) {
+    public HostDeprovisionMaintainer(NodeRepository nodeRepository, Duration interval, JobControl jobControl,
+                                     HostProvisioner hostProvisioner, FlagSource flagSource) {
         super(nodeRepository, interval, jobControl);
         this.hostProvisioner = hostProvisioner;
+        this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource);
     }
 
     @Override
     protected void maintain() {
+        if (! dynamicProvisioningEnabled.value()) return;
+
         try (Mutex lock = nodeRepository().lockAllocation()) {
             NodeList nodes = nodeRepository().list();
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainer.java
index 5acdc56ffda..5490216fdf3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainer.java
@@ -3,12 +3,16 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
-import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
 import java.util.List;
@@ -27,15 +31,19 @@ public class HostProvisionMaintainer extends Maintainer {
     private static final Logger log = Logger.getLogger(HostProvisionMaintainer.class.getName());
 
     private final HostProvisioner hostProvisioner;
+    private final BooleanFlag dynamicProvisioningEnabled;
 
-    public HostProvisionMaintainer(
-            NodeRepository nodeRepository, Duration interval, JobControl jobControl, HostProvisioner hostProvisioner) {
+    public HostProvisionMaintainer(NodeRepository nodeRepository, Duration interval, JobControl jobControl,
+                                   HostProvisioner hostProvisioner, FlagSource flagSource) {
         super(nodeRepository, interval, jobControl);
         this.hostProvisioner = hostProvisioner;
+        this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource);
     }
 
     @Override
     protected void maintain() {
+        if (! dynamicProvisioningEnabled.value()) return;
+
         try (Mutex lock = nodeRepository().lockAllocation()) {
             NodeList nodes = nodeRepository().list();
 
@@ -43,6 +51,8 @@ public class HostProvisionMaintainer extends Maintainer {
                 try {
                     List<Node> updatedNodes = hostProvisioner.provision(host, children);
                     nodeRepository().write(updatedNodes);
+                } catch (IllegalArgumentException | IllegalStateException e) {
+                    log.log(Level.INFO, "Failed to provision " + host.hostname() + ": " + Exceptions.toMessageString(e));
                 } catch (FatalProvisioningException e) {
                     log.log(Level.SEVERE, "Failed to provision " + host.hostname() + ", failing out the host recursively", e);
                     // Fail out as operator to force a quick redeployment
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 2c784de06ca..2496d9ba8c9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -11,6 +11,7 @@ import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.maintenance.retire.RetireIPv4OnlyNodes;
 import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy;
@@ -64,16 +65,17 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
                                      Zone zone, Orchestrator orchestrator, Metric metric,
                                      ConfigserverConfig configserverConfig,
                                      DuperModelInfraApi duperModelInfraApi,
-                                     ProvisionServiceProvider provisionServiceProvider) {
+                                     ProvisionServiceProvider provisionServiceProvider,
+                                     FlagSource flagSource) {
         this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
-                orchestrator, metric, configserverConfig, duperModelInfraApi, provisionServiceProvider);
+                orchestrator, metric, configserverConfig, duperModelInfraApi, provisionServiceProvider, flagSource);
     }
 
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
                                      ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi,
-                                     ProvisionServiceProvider provisionServiceProvider) {
+                                     ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) {
         DefaultTimes defaults = new DefaultTimes(zone);
         jobControl = new JobControl(nodeRepository.database());
         infrastructureVersions = new InfrastructureVersions(nodeRepository.database());
@@ -93,9 +95,9 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService ->
                 new LoadBalancerExpirer(nodeRepository, durationFromEnv("load_balancer_expiry").orElse(defaults.loadBalancerExpiry), jobControl, lbService));
         hostProvisionMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner ->
-                new HostProvisionMaintainer(nodeRepository, durationFromEnv("host_provisioner_interval").orElse(defaults.hostProvisionerInterval), jobControl, hostProvisioner));
+                new HostProvisionMaintainer(nodeRepository, durationFromEnv("host_provisioner_interval").orElse(defaults.hostProvisionerInterval), jobControl, hostProvisioner, flagSource));
         hostDeprovisionMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner ->
-                new HostDeprovisionMaintainer(nodeRepository, durationFromEnv("host_deprovisioner_interval").orElse(defaults.hostDeprovisionerInterval), jobControl, hostProvisioner));
+                new HostDeprovisionMaintainer(nodeRepository, durationFromEnv("host_deprovisioner_interval").orElse(defaults.hostDeprovisionerInterval), jobControl, hostProvisioner, flagSource));
 
         // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
         infrastructureProvisioner.maintain();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
index d13643de5b1..355ba92acdf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
@@ -30,15 +30,35 @@ public class ProvisionedHost {
     }
 
     /** Generate {@link Node} instance representing the provisioned physical host */
-    Node generateHost() {
+    public Node generateHost() {
         return Node.create(id, Set.of(), Set.of(), hostHostname, Optional.empty(), hostFlavor, NodeType.host);
     }
 
     /** Generate {@link Node} instance representing the node running on this physical host */
-    Node generateNode() {
+    public Node generateNode() {
         return Node.createDockerNode(Set.of(), Set.of(), nodeHostname, Optional.of(hostHostname), nodeFlavor, NodeType.tenant);
     }
 
+    public String getId() {
+        return id;
+    }
+
+    public String getHostHostname() {
+        return hostHostname;
+    }
+
+    public Flavor getHostFlavor() {
+        return hostFlavor;
+    }
+
+    public String getNodeHostname() {
+        return nodeHostname;
+    }
+
+    public Flavor getNodeFlavor() {
+        return nodeFlavor;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index ecbe268cae0..e17e1871555 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -26,6 +26,7 @@ public class ContainerConfig {
                "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" +
+               "  <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" +
                "  <component id='com.yahoo.config.provision.Zone'/>\n" +
                "  <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler'>\n" +
                "    <binding>http://*/nodes/v2/*</binding>\n" +
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisionMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisionMaintainerTest.java
index 95c857527a1..aa057d794d1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisionMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisionMaintainerTest.java
@@ -2,6 +2,9 @@
 package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
@@ -32,8 +35,9 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
 public class HostDeprovisionMaintainerTest {
     private final HostProvisionerTester tester = new HostProvisionerTester();
     private final HostProvisioner hostProvisioner = mock(HostProvisioner.class);
+    private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true);
     private final HostDeprovisionMaintainer maintainer = new HostDeprovisionMaintainer(
-            tester.nodeRepository(), Duration.ofDays(1), tester.jobControl(), hostProvisioner);
+            tester.nodeRepository(), Duration.ofDays(1), tester.jobControl(), hostProvisioner, flagSource);
 
     @Test
     public void removes_nodes_if_successful() {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainerTest.java
index 1857a4ad74d..7089d436c6d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostProvisionMaintainerTest.java
@@ -10,6 +10,9 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -18,9 +21,9 @@ import com.yahoo.vespa.hosted.provision.node.Generation;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.hosted.provision.node.Reports;
 import com.yahoo.vespa.hosted.provision.node.Status;
+import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
-import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import org.junit.Test;
 
@@ -52,8 +55,9 @@ public class HostProvisionMaintainerTest {
 
     private final HostProvisionerTester tester = new HostProvisionerTester();
     private final HostProvisioner hostProvisioner = mock(HostProvisioner.class);
+    private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true);
     private final HostProvisionMaintainer maintainer = new HostProvisionMaintainer(
-            tester.nodeRepository(), Duration.ofDays(1), tester.jobControl(), hostProvisioner);
+            tester.nodeRepository(), Duration.ofDays(1), tester.jobControl(), hostProvisioner, flagSource);
 
     @Test
     public void delegates_to_host_provisioner_and_writes_back_result() {