Add debug logging for LB provisioning

2 files changed, 18 insertions, 11 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
index 9665d8872de..10069fd1a18 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
+import com.google.common.collect.Sets;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -22,6 +23,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.logging.Level;
+import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
 /**
@@ -37,6 +39,8 @@ import java.util.stream.Collectors;
  */
 public class LoadBalancerExpirer extends NodeRepositoryMaintainer {
 
+    private static final Logger LOG = Logger.getLogger(LoadBalancerExpirer.class.getName());
+
     private static final Duration reservedExpiry = Duration.ofHours(1);
     private static final Duration inactiveExpiry = Duration.ofHours(1);
 
@@ -72,6 +76,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer {
                                  lb.changedAt().isBefore(expiry) &&
                                  allocatedNodes(lb.id()).isEmpty(), lb -> {
             try {
+                log.log(Level.INFO, () -> "Removing expired inactive load balancer " + lb.id());
                 service.remove(lb.id().application(), lb.id().cluster());
                 db.removeLoadBalancer(lb.id());
             } catch (Exception e){
@@ -80,13 +85,12 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer {
             }
         });
         if (!failed.isEmpty()) {
-            log.log(Level.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s",
-                                                    failed.size(),
-                                                    failed.stream()
-                                                          .map(LoadBalancerId::serializedForm)
-                                                          .collect(Collectors.joining(", ")),
-                                                    interval()),
-                    lastException.get());
+            log.log(Level.WARNING, lastException.get(), () -> String.format("Failed to remove %d load balancers: %s, retrying in %s",
+                                                                            failed.size(),
+                                                                            failed.stream()
+                                                                                  .map(LoadBalancerId::serializedForm)
+                                                                                  .collect(Collectors.joining(", ")),
+                                                                            interval()));
         }
         return lastException.get() == null;
     }
@@ -101,7 +105,9 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer {
             var reals = new LinkedHashSet<>(lb.instance().get().reals());
             // Remove any real no longer allocated to this application
             reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value()));
+            if (reals.equals(lb.instance().get().reals())) return; // Nothing to remove
             try {
+                LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals));
                 service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals), true);
                 db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals))));
             } catch (Exception e) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
index e665ace6334..c114aa58a05 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -182,7 +182,8 @@ public class LoadBalancerProvisioner {
                     : loadBalancer.get().state();
             newLoadBalancer = loadBalancer.get().with(instance).with(state, now);
             if (loadBalancer.get().state() != newLoadBalancer.state()) {
-                log.log(Level.FINE, () -> "Moving " + newLoadBalancer.id() + " to state " + newLoadBalancer.state());
+                log.log(Level.INFO, () -> "Moving " + newLoadBalancer.id() + " from " + loadBalancer.get().state() +
+                                          " to " + newLoadBalancer.state());
             }
         }
 
@@ -209,13 +210,13 @@ public class LoadBalancerProvisioner {
     private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, Set<Real> reals,
                                                              Optional<LoadBalancer> currentLoadBalancer) {
         if (hasReals(currentLoadBalancer, reals)) return currentLoadBalancer.get().instance();
-        log.log(Level.FINE, () -> "Creating " + id + ", targeting: " + reals);
+        log.log(Level.INFO, () -> "Creating " + id + ", targeting: " + reals);
         try {
             return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals),
                                               allowEmptyReals(currentLoadBalancer)));
         } catch (Exception e) {
-            log.log(Level.WARNING, "Could not (re)configure " + id + ", targeting: " +
-                                   reals + ". The operation will be retried on next deployment", e);
+            log.log(Level.WARNING, e, () -> "Could not (re)configure " + id + ", targeting: " +
+                                            reals + ". The operation will be retried on next deployment");
         }
         return Optional.empty();
     }