Merge branch 'master' into hmusum/refactor-reconfigurer

26 files changed, 110 insertions, 249 deletions

diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java
index ef5e676e36b..4ad34074199 100644
--- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java
+++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java
@@ -167,7 +167,7 @@ public class Reindexer {
 
         // Wait until done; or until termination is forced, in which we shut down the visitor session immediately.
         phaser.arriveAndAwaitAdvance(); // Synchronize with visitor completion.
-        sessionShutdown.run(); // Shutdown aborts the session, then waits for it to terminate normally.
+        sessionShutdown.run(); // Shutdown aborts the session unless already complete, then waits for it to terminate normally.
 
         switch (control.getResult().getCode()) {
             default:
diff --git a/component/abi-spec.json b/component/abi-spec.json
index 5665075f1f2..2dfa3e0d71d 100644
--- a/component/abi-spec.json
+++ b/component/abi-spec.json
@@ -2,7 +2,8 @@
   "com.yahoo.component.AbstractComponent": {
     "superClass": "java.lang.Object",
     "interfaces": [
-      "com.yahoo.component.Component"
+      "com.yahoo.component.Component",
+      "com.yahoo.component.Deconstructable"
     ],
     "attributes": [
       "public"
@@ -105,6 +106,19 @@
     ],
     "fields": []
   },
+  "com.yahoo.component.Deconstructable": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "interface",
+      "abstract"
+    ],
+    "methods": [
+      "public abstract void deconstruct()"
+    ],
+    "fields": []
+  },
   "com.yahoo.component.Version": {
     "superClass": "java.lang.Object",
     "interfaces": [
diff --git a/component/src/main/java/com/yahoo/component/AbstractComponent.java b/component/src/main/java/com/yahoo/component/AbstractComponent.java
index fcaa1e62d56..18a19f3e238 100644
--- a/component/src/main/java/com/yahoo/component/AbstractComponent.java
+++ b/component/src/main/java/com/yahoo/component/AbstractComponent.java
@@ -11,7 +11,7 @@ import java.lang.reflect.Method;
  *
  * @author bratseth
  */
-public class AbstractComponent implements Component {
+public class AbstractComponent implements Component, Deconstructable {
 
     private static final MethodCache deconstructMethods = new MethodCache("deconstruct");
 
@@ -129,6 +129,7 @@ public class AbstractComponent implements Component {
      * <p>
      * This default implementation does nothing.
      */
+    @Override
     public void deconstruct() { }
 
     /**
diff --git a/component/src/main/java/com/yahoo/component/Deconstructable.java b/component/src/main/java/com/yahoo/component/Deconstructable.java
new file mode 100644
index 00000000000..4da39cd3420
--- /dev/null
+++ b/component/src/main/java/com/yahoo/component/Deconstructable.java
@@ -0,0 +1,13 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.component;
+
+/**
+ * A component in the component graph that should be deconstructed, to release resources.
+ *
+ * @author jonmv
+ */
+public interface Deconstructable {
+
+    void deconstruct();
+
+}
diff --git a/container-di/abi-spec.json b/container-di/abi-spec.json
index a9dda2ae224..02cc29cd07f 100644
--- a/container-di/abi-spec.json
+++ b/container-di/abi-spec.json
@@ -1,15 +1,16 @@
 {
   "com.yahoo.container.di.componentgraph.Provider": {
     "superClass": "java.lang.Object",
-    "interfaces": [],
+    "interfaces": [
+      "com.yahoo.component.Deconstructable"
+    ],
     "attributes": [
       "public",
       "interface",
       "abstract"
     ],
     "methods": [
-      "public abstract java.lang.Object get()",
-      "public abstract void deconstruct()"
+      "public abstract java.lang.Object get()"
     ],
     "fields": []
   }
diff --git a/container-di/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java b/container-di/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java
index 61497cf71bc..4e3881a6fe6 100644
--- a/container-di/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java
+++ b/container-di/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java
@@ -4,6 +4,7 @@ package com.yahoo.container.di;
 import org.osgi.framework.Bundle;
 
 import java.util.Collection;
+import java.util.List;
 
 /**
  * @author gjoranv
@@ -11,6 +12,7 @@ import java.util.Collection;
  */
 public interface ComponentDeconstructor {
 
-    void deconstruct(Collection<Object> components, Collection<Bundle> bundles);
+    /** Deconstructs the given components in order, then the given bundles. */
+    void deconstruct(List<Object> components, Collection<Bundle> bundles);
 
 }
diff --git a/container-di/src/main/java/com/yahoo/container/di/Container.java b/container-di/src/main/java/com/yahoo/container/di/Container.java
index af580767a17..7fc2f2e55bc 100644
--- a/container-di/src/main/java/com/yahoo/container/di/Container.java
+++ b/container-di/src/main/java/com/yahoo/container/di/Container.java
@@ -21,6 +21,7 @@ import com.yahoo.container.di.config.SubscriberFactory;
 import com.yahoo.vespa.config.ConfigKey;
 import org.osgi.framework.Bundle;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -159,10 +160,16 @@ public class Container {
     private void deconstructObsoleteComponents(ComponentGraph oldGraph,
                                                ComponentGraph newGraph,
                                                Collection<Bundle> obsoleteBundles) {
-        IdentityHashMap<Object, Object> oldComponents = new IdentityHashMap<>();
-        oldGraph.allConstructedComponentsAndProviders().forEach(c -> oldComponents.put(c, null));
-        newGraph.allConstructedComponentsAndProviders().forEach(oldComponents::remove);
-        componentDeconstructor.deconstruct(oldComponents.keySet(), obsoleteBundles);
+        Map<Object, ?> newComponents = new IdentityHashMap<>(newGraph.size());
+        for (Object component : newGraph.allConstructedComponentsAndProviders())
+            newComponents.put(component, null);
+
+        List<Object> obsoleteComponents = new ArrayList<>();
+        for (Object component : oldGraph.allConstructedComponentsAndProviders())
+            if ( ! newComponents.containsKey(component))
+                obsoleteComponents.add(component);
+
+        componentDeconstructor.deconstruct(obsoleteComponents, obsoleteBundles);
     }
 
     private Set<Bundle> installApplicationBundles(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs) {
diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/Provider.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/Provider.java
index a2d193654ea..3fd3195e5dd 100644
--- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/Provider.java
+++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/Provider.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.container.di.componentgraph;
 
+import com.yahoo.component.Deconstructable;
+
 /**
  * <p>Provides a component of the parameter type T.
  * If (and only if) dependency injection does not have a component of type T,
@@ -17,9 +19,8 @@ package com.yahoo.container.di.componentgraph;
  * @author Tony Vaagenes
  * @author gjoranv
  */
-public interface Provider<T> {
+public interface Provider<T> extends Deconstructable {
 
     T get();
-    void deconstruct();
 
 }
diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java
index 256c965052a..fef2809f236 100644
--- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java
+++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java
@@ -166,6 +166,7 @@ public class ComponentGraph {
         }
     }
 
+    /** All constructed components and providers of this, in reverse creation order, i.e., suited for ordered deconstruction. */
     public List<Object> allConstructedComponentsAndProviders() {
         List<Node> orderedNodes = topologicalSort(nodes());
         Collections.reverse(orderedNodes);
diff --git a/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java b/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java
index 19f277ff8fb..d39e8a36aed 100644
--- a/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java
+++ b/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java
@@ -19,6 +19,7 @@ import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleException;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -380,7 +381,7 @@ public class ContainerTest extends ContainerTestBase {
 
     public static class TestDeconstructor implements ComponentDeconstructor {
         @Override
-        public void deconstruct(Collection<Object> components, Collection<Bundle> bundles) {
+        public void deconstruct(List<Object> components, Collection<Bundle> bundles) {
             components.forEach(component -> {
                 if (component instanceof DestructableComponent) {
                     DestructableComponent vespaComponent = (DestructableComponent) component;
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java
index e60e8d407cd..0bb60c8aca8 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java
@@ -2,10 +2,13 @@
 package com.yahoo.container.jdisc.component;
 
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.Deconstructable;
 import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.container.di.ComponentDeconstructor;
 import com.yahoo.container.di.componentgraph.Provider;
 import com.yahoo.jdisc.SharedResource;
+
+import java.util.List;
 import java.util.logging.Level;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleException;
@@ -32,7 +35,7 @@ public class Deconstructor implements ComponentDeconstructor {
 
     private static final Logger log = Logger.getLogger(Deconstructor.class.getName());
 
-    private final ScheduledExecutorService executor =
+    final ScheduledExecutorService executor =
             Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getThreadFactory("component-deconstructor"));
 
     private final Duration delay;
@@ -42,8 +45,8 @@ public class Deconstructor implements ComponentDeconstructor {
     }
 
     @Override
-    public void deconstruct(Collection<Object> components, Collection<Bundle> bundles) {
-        Collection<AbstractComponent> destructibleComponents = new ArrayList<>();
+    public void deconstruct(List<Object> components, Collection<Bundle> bundles) {
+        Collection<Deconstructable> destructibleComponents = new ArrayList<>();
         for (var component : components) {
             if (component instanceof AbstractComponent) {
                 AbstractComponent abstractComponent = (AbstractComponent) component;
@@ -51,10 +54,7 @@ public class Deconstructor implements ComponentDeconstructor {
                     destructibleComponents.add(abstractComponent);
                 }
             } else if (component instanceof Provider) {
-                // TODO Providers should most likely be deconstructed similarly to AbstractComponent
-                log.log(FINE, () -> "Starting deconstruction of provider " + component);
-                ((Provider<?>) component).deconstruct();
-                log.log(FINE, () -> "Finished deconstruction of provider " + component);
+                destructibleComponents.add((Deconstructable) component);
             } else if (component instanceof SharedResource) {
                 log.log(FINE, () -> "Releasing container reference to resource " + component);
                 // No need to delay release, as jdisc does ref-counting
@@ -69,10 +69,10 @@ public class Deconstructor implements ComponentDeconstructor {
     private static class DestructComponentTask implements Runnable {
 
         private final Random random = new Random(System.nanoTime());
-        private final Collection<AbstractComponent> components;
+        private final Collection<Deconstructable> components;
         private final Collection<Bundle> bundles;
 
-        DestructComponentTask(Collection<AbstractComponent> components, Collection<Bundle> bundles) {
+        DestructComponentTask(Collection<Deconstructable> components, Collection<Bundle> bundles) {
             this.components = components;
             this.bundles = bundles;
         }
@@ -89,10 +89,10 @@ public class Deconstructor implements ComponentDeconstructor {
         @Override
         public void run() {
             for (var component : components) {
-                log.log(FINE, () -> "Starting deconstruction of component " + component);
+                log.log(FINE, () -> "Starting deconstruction of " + component);
                 try {
                     component.deconstruct();
-                    log.log(FINE, () -> "Finished deconstructing of component " + component);
+                    log.log(FINE, () -> "Finished deconstructing of " + component);
                 } catch (Exception | NoClassDefFoundError e) { // May get class not found due to it being already unloaded
                     log.log(WARNING, "Exception thrown when deconstructing component " + component, e);
                 } catch (Error e) {
diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java
index efdc8f44c17..9ae05f5c193 100644
--- a/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java
+++ b/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java
@@ -9,6 +9,9 @@ import com.yahoo.jdisc.SharedResource;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
 import static org.junit.Assert.assertTrue;
@@ -28,25 +31,25 @@ public class DeconstructorTest {
     public void require_abstract_component_destructed() throws InterruptedException {
         TestAbstractComponent abstractComponent = new TestAbstractComponent();
         // Done by executor, so it takes some time even with a 0 delay.
-        deconstructor.deconstruct(singleton(abstractComponent), emptyList());
-        int cnt = 0;
-        while (! abstractComponent.destructed && (cnt++ < 12000)) {
-            Thread.sleep(10);
-        }
+        deconstructor.deconstruct(List.of(abstractComponent), emptyList());
+        deconstructor.executor.shutdown();
+        deconstructor.executor.awaitTermination(1, TimeUnit.MINUTES);
         assertTrue(abstractComponent.destructed);
     }
 
     @Test
-    public void require_provider_destructed() {
+    public void require_provider_destructed() throws InterruptedException {
         TestProvider provider = new TestProvider();
-        deconstructor.deconstruct(singleton(provider), emptyList());
+        deconstructor.deconstruct(List.of(provider), emptyList());
+        deconstructor.executor.shutdown();
+        deconstructor.executor.awaitTermination(1, TimeUnit.MINUTES);
         assertTrue(provider.destructed);
     }
 
     @Test
     public void require_shared_resource_released() {
         TestSharedResource sharedResource = new TestSharedResource();
-        deconstructor.deconstruct(singleton(sharedResource), emptyList());
+        deconstructor.deconstruct(List.of(sharedResource), emptyList());
         assertTrue(sharedResource.released);
     }
 
@@ -55,10 +58,8 @@ public class DeconstructorTest {
         var bundle = new UninstallableMockBundle();
         // Done by executor, so it takes some time even with a 0 delay.
         deconstructor.deconstruct(emptyList(), singleton(bundle));
-        int cnt = 0;
-        while (! bundle.uninstalled && (cnt++ < 12000)) {
-            Thread.sleep(10);
-        }
+        deconstructor.executor.shutdown();
+        deconstructor.executor.awaitTermination(1, TimeUnit.MINUTES);
         assertTrue(bundle.uninstalled);
     }
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
index 8f2d2161f92..4006d68ba1f 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
@@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandl
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor;
-import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
@@ -81,8 +80,6 @@ public interface ServiceRegistry {
 
     BillingController billingController();
 
-    HostRepairClient hostRepairClient();
-
     ContainerRegistry containerRegistry();
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java
deleted file mode 100644
index c3fa0890cbb..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/HostRepairClient.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.repair;
-
-import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.zone.ZoneApi;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author olaa
- */
-public interface HostRepairClient {
-
-    /* Checks current ticket status and takes appropriate action */
-    void updateRepairStatus(ZoneApi zone, List<Node> nodes);
-
-    /* Creates reparation ticket for given host. Returns ticket number */
-    String createTicket(HostName hostname, String colo, ZoneId zoneId, String description, String category);
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java
deleted file mode 100644
index 7a4398d69bb..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/MockRepairClient.java
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.repair;
-
-import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.zone.ZoneApi;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author olaa
- */
-public class MockRepairClient implements HostRepairClient {
-
-    List<Node> updatedNodes = new ArrayList<>();
-
-    @Override
-    public void updateRepairStatus(ZoneApi zone, List<Node> nodes) {
-        updatedNodes.addAll(nodes);
-    }
-
-    @Override
-    public String createTicket(HostName hostname, String colo, ZoneId zoneId, String description, String category) {
-        throw new UnsupportedOperationException("Not implemented");
-    }
-
-    public List<Node> getUpdatedNodes() {
-        return updatedNodes;
-    }
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index be4c889cd75..1dd84cfc7b4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -50,7 +50,6 @@ public class ControllerMaintenance extends AbstractComponent {
     private final ResourceTagMaintainer resourceTagMaintainer;
     private final SystemRoutingPolicyMaintainer systemRoutingPolicyMaintainer;
     private final ApplicationMetaDataGarbageCollector applicationMetaDataGarbageCollector;
-    private final HostRepairMaintainer hostRepairMaintainer;
     private final ContainerImageExpirer containerImageExpirer;
     private final HostSwitchUpdater hostSwitchUpdater;
 
@@ -80,7 +79,6 @@ public class ControllerMaintenance extends AbstractComponent {
         resourceTagMaintainer = new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger());
         systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, intervals.systemRoutingPolicyMaintainer);
         applicationMetaDataGarbageCollector = new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector);
-        hostRepairMaintainer = new HostRepairMaintainer(controller, intervals.hostRepairMaintainer);
         containerImageExpirer = new ContainerImageExpirer(controller, intervals.containerImageExpirer);
         hostSwitchUpdater = new HostSwitchUpdater(controller, intervals.hostSwitchUpdater);
     }
@@ -111,7 +109,6 @@ public class ControllerMaintenance extends AbstractComponent {
         resourceTagMaintainer.close();
         systemRoutingPolicyMaintainer.close();
         applicationMetaDataGarbageCollector.close();
-        hostRepairMaintainer.close();
         containerImageExpirer.close();
         hostSwitchUpdater.close();
     }
@@ -149,7 +146,6 @@ public class ControllerMaintenance extends AbstractComponent {
         private final Duration resourceTagMaintainer;
         private final Duration systemRoutingPolicyMaintainer;
         private final Duration applicationMetaDataGarbageCollector;
-        private final Duration hostRepairMaintainer;
         private final Duration containerImageExpirer;
         private final Duration hostSwitchUpdater;
 
@@ -172,7 +168,6 @@ public class ControllerMaintenance extends AbstractComponent {
             this.resourceTagMaintainer = duration(30, MINUTES);
             this.systemRoutingPolicyMaintainer = duration(10, MINUTES);
             this.applicationMetaDataGarbageCollector = duration(12, HOURS);
-            this.hostRepairMaintainer = duration(12, HOURS);
             this.containerImageExpirer = duration(2, HOURS);
             this.hostSwitchUpdater = duration(12, HOURS);
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java
deleted file mode 100644
index 57727e64e30..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainer.java
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.maintenance;
-
-import com.yahoo.config.provision.CloudName;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.zone.ZoneApi;
-import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.repair.RepairTicketReport;
-import com.yahoo.vespa.hosted.controller.api.integration.repair.HostRepairClient;
-import com.yahoo.yolean.Exceptions;
-
-import java.time.Duration;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-import static com.yahoo.yolean.Exceptions.uncheck;
-
-/**
- *
- * Responsible for keeping track of hosts under repair.
- *
- * @author olaa
- */
-public class HostRepairMaintainer extends ControllerMaintainer {
-
-    private final NodeRepository nodeRepository;
-    private final HostRepairClient repairClient;
-
-    private static final Logger log = Logger.getLogger(HostRepairMaintainer.class.getName());
-
-
-    public HostRepairMaintainer(Controller controller, Duration interval) {
-        super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)));
-        this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
-        this.repairClient = controller.serviceRegistry().hostRepairClient();
-    }
-
-
-    @Override
-    protected boolean maintain() {
-        AtomicInteger exceptions = new AtomicInteger(0);
-
-        controller().zoneRegistry().zones()
-                .reachable().zones().stream()
-                .forEach(zoneApi -> {
-                            var breakfixedNodes = nodeRepository.list((zoneApi).getId())
-                                    .stream()
-                                    .filter(node -> node.state() == Node.State.breakfixed)
-                                    .collect(Collectors.toList());
-                            try {
-                                repairClient.updateRepairStatus(zoneApi, breakfixedNodes);
-                            } catch (Exception e) {
-                                log.warning("Failed to update repair status; " + Exceptions.toMessageString(e));
-                                exceptions.incrementAndGet();
-                            }
-                        }
-                );
-
-        return exceptions.get() == 0;
-    }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index 96eb4b39510..ae1e2c38e6a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
 import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler;
-import com.yahoo.vespa.hosted.controller.api.integration.repair.MockRepairClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService;
@@ -61,7 +60,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final MockResourceTagger mockResourceTagger = new MockResourceTagger();
     private final ApplicationRoleService applicationRoleService = new NoopApplicationRoleService();
     private final BillingController billingController = new MockBillingController();
-    private final MockRepairClient repairClient = new MockRepairClient();
     private final ContainerRegistryMock containerRegistry = new ContainerRegistryMock();
 
     public ServiceRegistryMock(SystemName system) {
@@ -195,11 +193,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     }
 
     @Override
-    public MockRepairClient hostRepairClient() {
-        return repairClient;
-    }
-
-    @Override
     public ContainerRegistryMock containerRegistry() {
         return containerRegistry;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java
deleted file mode 100644
index ab6e13bb5a2..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostRepairMaintainerTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package com.yahoo.vespa.hosted.controller.maintenance;
-
-import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.ControllerTester;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
-import org.junit.Test;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-/**
- * @author olaa
- */
-public class HostRepairMaintainerTest {
-
-    private final ControllerTester tester = new ControllerTester();
-    private final HostRepairMaintainer maintainer = new HostRepairMaintainer(tester.controller(), Duration.ofHours(12));
-
-    @Test
-    public void maintain() {
-        var zoneId = ZoneId.from("dev.us-east-1");
-        var hostname1 = HostName.from("node-1-tenant-host-dev.us-east-1");
-        var hostname2 = HostName.from("node-2-tenant-host-dev.us-east-1");
-        var timestamp = Instant.now().toEpochMilli();
-
-        var node1 = new Node.Builder()
-                .state(Node.State.active)
-                .hostname(hostname1)
-                .build();
-        var node2 = new Node.Builder()
-                .state(Node.State.breakfixed)
-                .hostname(hostname2)
-                .build();
-        tester.configServer().nodeRepository().putNodes(zoneId, List.of(node1, node2));
-        maintainer.maintain();
-        var updatedNodes = tester.serviceRegistry().hostRepairClient().getUpdatedNodes();
-        assertEquals(1, updatedNodes.size());
-        assertEquals(hostname2, updatedNodes.get(0).hostname());
-    }
-}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 5ab087aab17..b6b45bd76e0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -31,9 +31,6 @@
       "name": "DeploymentMetricsMaintainer"
     },
     {
-      "name": "HostRepairMaintainer"
-    },
-    {
       "name": "HostSwitchUpdater"
     },
     {
diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def
index e075f57514b..8f24646367f 100644
--- a/storage/src/vespa/storage/config/stor-communicationmanager.def
+++ b/storage/src/vespa/storage/config/stor-communicationmanager.def
@@ -18,39 +18,39 @@ mbus_distributor_node_max_pending_size int default=0
 mbus_content_node_max_pending_size int default=0
 
 # Minimum size of packets to compress (0 means no compression)
-mbus.compress.limit int default=1024
+mbus.compress.limit int default=1024 restart
 
 ## Compression level for packets
-mbus.compress.level int default=3
+mbus.compress.level int default=3 restart
 
 ## Compression type for packets.
-mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4
+mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 restart
 
 ## TTL for rpc target cache
-mbus.rpctargetcache.ttl double default = 600
+mbus.rpctargetcache.ttl double default = 600 restart
 
 ## Number of threads for mbus threadpool
 ## Any value below 1 will be 1.
-mbus.num_threads int default=4
+mbus.num_threads int default=4 restart
 
 mbus.optimize_for enum {LATENCY, THROUGHPUT, ADAPTIVE} default = LATENCY
 
 ## Enable to use above thread pool for encoding replies
 ## False will use network(fnet) thread
-mbus.dispatch_on_encode bool default=true
+mbus.dispatch_on_encode bool default=true restart
 
 ## Enable to use above thread pool for decoding replies
 ## False will use network(fnet) thread
 ## Todo: Change default once verified in large scale deployment.
-mbus.dispatch_on_decode bool default=false
+mbus.dispatch_on_decode bool default=false restart
 
 ## Skip messenger thread on reply
 ## Experimental
-mbus.skip_reply_thread bool default=false
+mbus.skip_reply_thread bool default=false restart
 
 ## Skip messenger thread on reply
 ## Experimental
-mbus.skip_request_thread bool default=false
+mbus.skip_request_thread bool default=false restart
 
 ## Skip communication manager thread on mbus requests
 ## Experimental
@@ -61,7 +61,7 @@ skip_thread bool default=false
 use_direct_storageapi_rpc bool default=false
 
 ## The number of network (FNET) threads used by the shared rpc resource.
-rpc.num_network_threads int default=2
+rpc.num_network_threads int default=2 restart
 
 ## The number of (FNET) RPC targets to use per node in the cluster.
 ##
@@ -70,13 +70,13 @@ rpc.num_network_threads int default=2
 ## and the RPC target itself handles sequencing of these messages.
 ## NB !! It is vital that this number is kept in sync with stor-filestor:num_network_threads.
 ## Only skilled vespa core developers should touch this.
-rpc.num_targets_per_node int default=1
+rpc.num_targets_per_node int default=1 restart
 
 # Minimum size of packets to compress (0 means no compression)
-rpc.compress.limit int default=1024
+rpc.compress.limit int default=1024 restart
 
 ## Compression level for packets
-rpc.compress.level int default=3
+rpc.compress.level int default=3 restart
 
 ## Compression type for packets.
-rpc.compress.type enum {NONE, LZ4, ZSTD} default=LZ4
+rpc.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 restart
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
index e05b502f856..9d4b7b58a6f 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
@@ -375,6 +375,7 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig>
 {
     // Only allow dynamic (live) reconfiguration of message bus limits.
     _skip_thread = config->skipThread;
+    _use_direct_storageapi_rpc = config->useDirectStorageapiRpc;
     if (_mbus) {
         configureMessageBusLimits(*config);
         if (_mbus->getRPCNetwork().getPort() != config->mbusport) {
@@ -427,7 +428,6 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig>
         configureMessageBusLimits(*config);
     }
 
-    _use_direct_storageapi_rpc = config->useDirectStorageapiRpc;
     _message_codec_provider = std::make_unique<rpc::MessageCodecProvider>(_component.getTypeRepo()->documentTypeRepo);
     _shared_rpc_resources = std::make_unique<rpc::SharedRpcResources>(_configUri, config->rpcport, config->rpc.numNetworkThreads);
     _cc_rpc_service = std::make_unique<rpc::ClusterControllerApiRpcService>(*this, *_shared_rpc_resources);
@@ -476,7 +476,7 @@ void
 CommunicationManager::enqueue_or_process(std::shared_ptr<api::StorageMessage> msg)
 {
     assert(msg);
-    if (_skip_thread) {
+    if (_skip_thread.load(std::memory_order_relaxed)) {
         LOG(spam, "Process storage message %s, priority %d", msg->toString().c_str(), msg->getPriority());
         process(msg);
     } else {
@@ -573,7 +573,9 @@ CommunicationManager::sendCommand(
         case api::StorageMessageAddress::Protocol::STORAGE:
     {
         LOG(debug, "Send to %s: %s", address.toString().c_str(), msg->toString().c_str());
-        if (_use_direct_storageapi_rpc && _storage_api_rpc_service->target_supports_direct_rpc(address)) {
+        if (_use_direct_storageapi_rpc.load(std::memory_order_relaxed) &&
+            _storage_api_rpc_service->target_supports_direct_rpc(address))
+        {
             _storage_api_rpc_service->send_rpc_v1_request(msg);
         } else {
             auto cmd = std::make_unique<mbusprot::StorageCommand>(msg);
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h
index 7227f1d7e5b..db88f95af6d 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.h
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.h
@@ -118,8 +118,8 @@ private:
     std::atomic<bool>     _closed;
     DocumentApiConverter  _docApiConverter;
     framework::Thread::UP _thread;
-    bool                  _skip_thread;
-    bool                  _use_direct_storageapi_rpc;
+    std::atomic<bool>     _skip_thread;
+    std::atomic<bool>     _use_direct_storageapi_rpc;
 
     void updateMetrics(const MetricLockGuard &) override;
     void enqueue_or_process(std::shared_ptr<api::StorageMessage> msg);
diff --git a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java
index 95fccd3fce8..9779aea0e4a 100644
--- a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java
+++ b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java
@@ -18,7 +18,7 @@ import java.util.Set;
  * in IdentityHashMap, where the key set is often used as an identity set.
  * </p>
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  * @since 5.1.4
  * @see java.util.IdentityHashMap
  *
diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
index 90de4a9472e..6a87d8547c1 100644
--- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
+++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.zookeeper;
 import com.google.inject.Inject;
 import com.yahoo.cloud.config.ZookeeperServerConfig;
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.net.HostName;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
@@ -88,7 +89,7 @@ public class Reconfigurer extends AbstractComponent {
         log.log(Level.INFO, "Will reconfigure ZooKeeper cluster in " + reconfigWaitPeriod() +
                             ". Joining servers: " + joiningServers + ", leaving servers: " + leavingServers);
         sleeper.accept(reconfigWaitPeriod());
-        String connectionSpec = connectionSpec(activeConfig);
+        String connectionSpec = localConnectionSpec(activeConfig);
         Instant end = Instant.now().plus(reconfigTimeout);
         // Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed
         for (int attempts = 1; Instant.now().isBefore(end); attempts++) {
@@ -116,10 +117,8 @@ public class Reconfigurer extends AbstractComponent {
         return reconfigInterval.multipliedBy(activeConfig.myid());
     }
 
-    private static String connectionSpec(ZookeeperServerConfig config) {
-        return config.server().stream()
-                     .map(server -> server.hostname() + ":" + config.clientPort())
-                     .collect(Collectors.joining(","));
+    private static String localConnectionSpec(ZookeeperServerConfig config) {
+        return HostName.getLocalhost() + ":" + config.clientPort();
     }
 
     private static List<String> serverIds(ZookeeperServerConfig config) {
diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java b/zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
index e8ee17192ee..662c88289d8 100644
--- a/zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
+++ b/zookeeper-server/zookeeper-server-3.5.6/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.zookeeper;
 
 import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.net.HostName;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -46,7 +47,7 @@ public class ReconfigurerTest {
         // Cluster grows
         ZookeeperServerConfig nextConfig = createConfig(5, true);
         reconfigurer.startOrReconfigure(nextConfig);
-        assertEquals("node0:2181,node1:2181,node2:2181", reconfigurer.connectionSpec());
+        assertEquals("node1:2181", reconfigurer.connectionSpec());
         assertEquals("3=node3:2182:2183;2181,4=node4:2182:2183;2181", reconfigurer.joiningServers());
         assertNull("No servers are leaving", reconfigurer.leavingServers());
         assertEquals(1, reconfigurer.reconfigurations());
@@ -61,7 +62,7 @@ public class ReconfigurerTest {
         nextConfig = createConfig(3, true);
         reconfigurer.startOrReconfigure(nextConfig);
         assertEquals(2, reconfigurer.reconfigurations());
-        assertEquals("node0:2181,node1:2181,node2:2181,node3:2181,node4:2181", reconfigurer.connectionSpec());
+        assertEquals("node1:2181", reconfigurer.connectionSpec());
         assertNull("No servers are joining", reconfigurer.joiningServers());
         assertEquals("3,4", reconfigurer.leavingServers());
         assertSame(nextConfig, reconfigurer.activeConfig());
@@ -76,7 +77,7 @@ public class ReconfigurerTest {
 
         ZookeeperServerConfig nextConfig = createConfig(5, true);
         reconfigurer.startOrReconfigure(nextConfig);
-        assertEquals("node0:2181,node1:2181,node2:2181", reconfigurer.connectionSpec());
+        assertEquals("node1:2181", reconfigurer.connectionSpec());
         assertEquals("3=node3:2182:2183;2181,4=node4:2182:2183;2181", reconfigurer.joiningServers());
         assertNull("No servers are leaving", reconfigurer.leavingServers());
         assertEquals(1, reconfigurer.reconfigurations());
@@ -124,6 +125,7 @@ public class ReconfigurerTest {
         TestableReconfigurer(TestableZkAdmin zkReconfigurer) {
             super(zkReconfigurer);
             this.zkReconfigurer = zkReconfigurer;
+            HostName.setHostNameForTestingOnly("node1");
         }
 
         @Override