Merge remote-tracking branch 'origin/master' into ogronnesby/trial-tenant-limit

Conflicts:
	controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
	controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
	flags/src/main/java/com/yahoo/vespa/flags/Flags.java

14 files changed, 239 insertions, 30 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index ed1e442f266..0d47d738bed 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -184,9 +184,7 @@ public class InternalStepRunner implements StepRunner {
     }
 
     private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) {
-        return deploy(id.application(),
-                      id.type(),
-                      () -> controller.applications().deploy2(id.job(), setTheStage),
+        return deploy(() -> controller.applications().deploy2(id.job(), setTheStage),
                       controller.jobController().run(id).get()
                                 .stepInfo(setTheStage ? deployInitialReal : deployReal).get()
                                 .startTime().get(),
@@ -196,9 +194,7 @@ public class InternalStepRunner implements StepRunner {
     private Optional<RunStatus> deployTester(RunId id, DualLogger logger) {
         Version platform = testerPlatformVersion(id);
         logger.log("Deploying the tester container on platform " + platform + " ...");
-        return deploy(id.tester().id(),
-                      id.type(),
-                      () -> controller.applications().deployTester(id.tester(),
+        return deploy(() -> controller.applications().deployTester(id.tester(),
                                                                    testerPackage(id),
                                                                    id.type().zone(controller.system()),
                                                                    platform),
@@ -208,8 +204,7 @@ public class InternalStepRunner implements StepRunner {
                       logger);
     }
 
-    private Optional<RunStatus> deploy(ApplicationId id, JobType type, Supplier<ActivateResult> deployment,
-                                       Instant startTime, DualLogger logger) {
+    private Optional<RunStatus> deploy(Supplier<ActivateResult> deployment, Instant startTime, DualLogger logger) {
         try {
             PrepareResponse prepareResponse = deployment.get().prepareResponse();
             if (prepareResponse.log != null)
@@ -232,9 +227,9 @@ public class InternalStepRunner implements StepRunner {
                                          ? Optional.of(deploymentFailed) : Optional.empty();
             switch (e.getErrorCode()) {
                 case CERTIFICATE_NOT_READY:
-                    logger.log("Waiting for provisioned web certificate — new application, or old one has expired");
+                    logger.log("Waiting for certificate to become ready on config server: New application, or old one has expired");
                     if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) {
-                        logger.log("Deployment failed to find provisioned endpoint certificate after " + timeouts.endpointCertificate());
+                        logger.log("Certificate did not become available on config server within (" + timeouts.endpointCertificate() + ")");
                         return Optional.of(RunStatus.endpointCertificateTimeout);
                     }
                     return result;
@@ -264,9 +259,10 @@ public class InternalStepRunner implements StepRunner {
             switch (e.type()) {
                 case CERT_NOT_AVAILABLE:
                     // Same as CERTIFICATE_NOT_READY above, only from the controller
-                    logger.log("Waiting for provisioned web certificate — new application, or old one has expired");
+                    logger.log("Waiting for certificate to become valid: New application, or old one has expired");
                     if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) {
-                        logger.log("Deployment failed to find provisioned endpoint certificate after " + timeouts.endpointCertificate());
+                        logger.log("Controller could not validate certificate within " +
+                                   timeouts.endpointCertificate() + ": " + Exceptions.toMessageString(e));
                         return Optional.of(RunStatus.endpointCertificateTimeout);
                     }
                     return Optional.empty();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 82f37c9bc93..be4c889cd75 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -52,6 +52,7 @@ public class ControllerMaintenance extends AbstractComponent {
     private final ApplicationMetaDataGarbageCollector applicationMetaDataGarbageCollector;
     private final HostRepairMaintainer hostRepairMaintainer;
     private final ContainerImageExpirer containerImageExpirer;
+    private final HostSwitchUpdater hostSwitchUpdater;
 
     @Inject
     @SuppressWarnings("unused") // instantiated by Dependency Injection
@@ -81,6 +82,7 @@ public class ControllerMaintenance extends AbstractComponent {
         applicationMetaDataGarbageCollector = new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector);
         hostRepairMaintainer = new HostRepairMaintainer(controller, intervals.hostRepairMaintainer);
         containerImageExpirer = new ContainerImageExpirer(controller, intervals.containerImageExpirer);
+        hostSwitchUpdater = new HostSwitchUpdater(controller, intervals.hostSwitchUpdater);
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -111,6 +113,7 @@ public class ControllerMaintenance extends AbstractComponent {
         applicationMetaDataGarbageCollector.close();
         hostRepairMaintainer.close();
         containerImageExpirer.close();
+        hostSwitchUpdater.close();
     }
 
     /** Create one OS upgrader per cloud found in the zone registry of controller */
@@ -148,6 +151,7 @@ public class ControllerMaintenance extends AbstractComponent {
         private final Duration applicationMetaDataGarbageCollector;
         private final Duration hostRepairMaintainer;
         private final Duration containerImageExpirer;
+        private final Duration hostSwitchUpdater;
 
         public Intervals(SystemName system) {
             this.system = Objects.requireNonNull(system);
@@ -170,6 +174,7 @@ public class ControllerMaintenance extends AbstractComponent {
             this.applicationMetaDataGarbageCollector = duration(12, HOURS);
             this.hostRepairMaintainer = duration(12, HOURS);
             this.containerImageExpirer = duration(2, HOURS);
+            this.hostSwitchUpdater = duration(12, HOURS);
         }
 
         private Duration duration(long amount, TemporalUnit unit) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostSwitchUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostSwitchUpdater.java
new file mode 100644
index 00000000000..8e7a364b5f3
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostSwitchUpdater.java
@@ -0,0 +1,80 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService;
+import com.yahoo.vespa.hosted.controller.api.integration.entity.NodeEntity;
+import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
+
+import java.time.Duration;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Ensures that the switch information for all hosts is up to date.
+ *
+ * @author mpolden
+ */
+public class HostSwitchUpdater extends ControllerMaintainer {
+
+    private static final Logger LOG = Logger.getLogger(HostSwitchUpdater.class.getName());
+    private static final Pattern HOST_PATTERN = Pattern.compile("^(proxy|cfg|controller)host(.+)$");
+
+    private final NodeRepository nodeRepository;
+
+    public HostSwitchUpdater(Controller controller, Duration interval) {
+        super(controller, interval, null, EnumSet.of(SystemName.cd, SystemName.main));
+        this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
+    }
+
+    @Override
+    protected boolean maintain() {
+        Map<String, NodeEntity> nodeEntities = controller().serviceRegistry().entityService().listNodes().stream()
+                                                           .collect(Collectors.toMap(NodeEntity::hostname,
+                                                                                     Function.identity()));
+        int nodesUpdated = 0;
+        try {
+            for (var zone : controller().zoneRegistry().zones().controllerUpgraded().all().ids()) {
+                for (var node : nodeRepository.list(zone)) {
+                    if (!node.type().isHost()) continue;
+                    NodeEntity nodeEntity = nodeEntities.get(registeredHostnameOf(node));
+                    if (!shouldUpdate(node, nodeEntity)) continue;
+
+                    NodeRepositoryNode updatedNode = new NodeRepositoryNode();
+                    updatedNode.setSwitchHostname(nodeEntity.switchHostname().get());
+                    nodeRepository.patchNode(zone, node.hostname().value(), updatedNode);
+                    nodesUpdated++;
+                }
+            }
+        } finally {
+            if (nodesUpdated > 0) {
+                LOG.info("Updated switch hostname for " + nodesUpdated + " node(s)");
+            }
+        }
+        return true;
+    }
+
+    /** Returns the hostname that given host is registered under in the {@link EntityService} */
+    private static String registeredHostnameOf(Node host) {
+        String hostname = host.hostname().value();
+        if (!host.type().isHost()) return hostname;
+        Matcher matcher = HOST_PATTERN.matcher(hostname);
+        if (!matcher.matches()) return hostname;
+        return matcher.replaceFirst("$1$2");
+    }
+
+    private static boolean shouldUpdate(Node node, NodeEntity nodeEntity) {
+        if (nodeEntity == null) return false;
+        if (nodeEntity.switchHostname().isEmpty()) return false;
+        return !node.switchHostname().equals(nodeEntity.switchHostname());
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java
index 1cb23d0c515..109e761f925 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobControlFlags.java
@@ -4,8 +4,8 @@ package com.yahoo.vespa.hosted.controller.persistence;
 import com.yahoo.concurrent.maintenance.JobControlState;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.flags.FlagSource;
-import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.ListFlag;
+import com.yahoo.vespa.flags.PermanentFlags;
 
 import java.util.Set;
 
@@ -21,7 +21,7 @@ public class JobControlFlags implements JobControlState {
 
     public JobControlFlags(CuratorDb curator, FlagSource flagSource) {
         this.curator = curator;
-        this.inactiveJobsFlag = Flags.INACTIVE_MAINTENANCE_JOBS.bindTo(flagSource);
+        this.inactiveJobsFlag = PermanentFlags.INACTIVE_MAINTENANCE_JOBS.bindTo(flagSource);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 4a6d6e2e72d..cbf6307f5e6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1576,7 +1576,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                   .forEach(cluster -> {
                       Cursor clusterObject = clustersArray.addObject();
                       clusterObject.setString("name", cluster.getKey());
-                      setStatus(clusterObject.setObject("status"), cluster.getValue().common());
+                      cluster.getValue().common().ifPresent(common -> setStatus(clusterObject.setObject("status"), common));
 
                       Cursor pendingArray = clusterObject.setArray("pending");
                       cluster.getValue().pending().entrySet().stream().sorted(comparingByKey())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
index 954582a8ec7..e097b82b7d0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
@@ -23,6 +23,7 @@ import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.IntFlag;
+import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.LockedTenant;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId;
@@ -75,7 +76,7 @@ public class UserApiHandler extends LoggingRequestHandler {
         super(parentCtx);
         this.users = users;
         this.controller = controller;
-        this.enable_public_signup_flow = Flags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource);
+        this.enable_public_signup_flow = PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource);
         this.maxTrialTenants = Flags.MAX_TRIAL_TENANTS.bindTo(flagSource);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
index 1f8c44ea82d..d37e1e05030 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.IntFlag;
+import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController;
@@ -26,7 +27,9 @@ import javax.ws.rs.ForbiddenException;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.*;
+import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.administrator;
+import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.hostedOperator;
+import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.hostedSupporter;
 
 /**
  * @author jonmv
@@ -42,7 +45,7 @@ public class CloudAccessControl implements AccessControl {
     @Inject
     public CloudAccessControl(UserManagement userManagement, FlagSource flagSource, ServiceRegistry serviceRegistry) {
         this.userManagement = userManagement;
-        this.enablePublicSignup = Flags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource);
+        this.enablePublicSignup = PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource);
         this.maxTrialTenants = Flags.MAX_TRIAL_TENANTS.bindTo(flagSource);
         billingController = serviceRegistry.billingController();
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
index 61bd7858da2..ca478905893 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
@@ -41,6 +41,8 @@ public class NodeRepositoryMock implements NodeRepository {
     private final Map<ZoneId, TargetVersions> targetVersions = new HashMap<>();
     private final Map<Integer, Duration> osUpgradeBudgets = new HashMap<>();
 
+    private boolean allowPatching = false;
+
     /** Add or update given nodes in zone */
     public void putNodes(ZoneId zone, List<Node> nodes) {
         nodeRepository.putIfAbsent(zone, new HashMap<>());
@@ -230,7 +232,14 @@ public class NodeRepositoryMock implements NodeRepository {
 
     @Override
     public void patchNode(ZoneId zoneId, String hostName, NodeRepositoryNode node) {
-        throw new UnsupportedOperationException();
+        if (!allowPatching) throw new UnsupportedOperationException();
+        List<Node> existing = list(zoneId, List.of(HostName.from(hostName)));
+        if (existing.size() != 1) throw new IllegalArgumentException("Node " + hostName + " not found in " + zoneId);
+
+        // Note: Only supports switchHostname
+        Node newNode = new Node.Builder(existing.get(0)).switchHostname(node.getSwitchHostname())
+                                                        .build();
+        putNodes(zoneId, newNode);
     }
 
     @Override
@@ -280,4 +289,9 @@ public class NodeRepositoryMock implements NodeRepository {
         nodeRepository.get(zoneId).get(hostName).reports().put(reportId, report);
     }
 
+    public NodeRepositoryMock allowPatching(boolean allowPatching) {
+        this.allowPatching = allowPatching;
+        return this;
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index ea31667d249..96eb4b39510 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
 import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever;
@@ -81,7 +80,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     }
 
     @Override
-    public ConfigServer configServer() {
+    public ConfigServerMock configServer() {
         return configServerMock;
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostSwitchUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostSwitchUpdaterTest.java
new file mode 100644
index 00000000000..4dcacb3934b
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostSwitchUpdaterTest.java
@@ -0,0 +1,110 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.entity.NodeEntity;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class HostSwitchUpdaterTest {
+
+    @Test
+    public void maintain() {
+        ControllerTester tester = new ControllerTester();
+        tester.serviceRegistry().configServer().nodeRepository().allowPatching(true);
+        addNodeEntities(tester);
+
+        // First iteration patches all hosts
+        HostSwitchUpdater maintainer = new HostSwitchUpdater(tester.controller(), Duration.ofDays(1));
+        maintainer.maintain();
+        List<Node> nodes = allNodes(tester);
+        assertFalse(nodes.isEmpty());
+        for (var node : nodes) {
+            assertEquals("Node " + node.hostname().value() + (node.type().isHost() ? " has" : " does not have")
+                         + " switch hostname", node.type().isHost(), node.switchHostname().isPresent());
+            if (node.type().isHost()) {
+                assertEquals("tor-" + node.hostname().value(), node.switchHostname().get());
+            }
+        }
+
+        // Second iteration does not patch anything as all switch information is current
+        tester.serviceRegistry().configServer().nodeRepository().allowPatching(false);
+        maintainer.maintain();
+
+        // One host is moved to a different switch
+        Node host = allNodes(tester).stream().filter(node -> node.type().isHost()).findFirst().get();
+        String newSwitch = "tor2-" + host.hostname().value();
+        NodeEntity nodeEntity = new NodeEntity(host.hostname().value(), "", "", newSwitch);
+        tester.serviceRegistry().entityService().addNodeEntity(nodeEntity);
+
+        // Host is updated
+        tester.serviceRegistry().configServer().nodeRepository().allowPatching(true);
+        maintainer.maintain();
+        assertEquals(newSwitch, getNode(host.hostname(), tester).switchHostname().get());
+
+        // Host keeps old switch hostname if removed from the node entity
+        nodeEntity = new NodeEntity(host.hostname().value(), "", "", "");
+        tester.serviceRegistry().entityService().addNodeEntity(nodeEntity);
+        maintainer.maintain();
+        assertEquals(newSwitch, getNode(host.hostname(), tester).switchHostname().get());
+
+        // Updates node registered under a different hostname
+        ZoneId zone = tester.zoneRegistry().zones().controllerUpgraded().all().ids().get(0);
+        String hostnameSuffix = ".prod." + zone.value();
+        Node configNode = new Node.Builder().hostname(HostName.from("cfg3" + hostnameSuffix))
+                                            .type(NodeType.config)
+                                            .build();
+        Node configHost = new Node.Builder().hostname(HostName.from("cfghost3" + hostnameSuffix))
+                                            .type(NodeType.confighost)
+                                            .build();
+        tester.serviceRegistry().configServer().nodeRepository().putNodes(zone, List.of(configNode, configHost));
+        String switchHostname = switchHostname(configHost);
+        NodeEntity configNodeEntity = new NodeEntity("cfg3"  + hostnameSuffix, "", "", switchHostname);
+        tester.serviceRegistry().entityService().addNodeEntity(configNodeEntity);
+        maintainer.maintain();
+        assertEquals(switchHostname, getNode(configHost.hostname(), tester).switchHostname().get());
+        assertTrue("Switch hostname is not set for non-host", getNode(configNode.hostname(), tester).switchHostname().isEmpty());
+    }
+
+    private static Node getNode(HostName hostname, ControllerTester tester) {
+        return allNodes(tester).stream()
+                               .filter(node -> node.hostname().equals(hostname))
+                               .findFirst()
+                               .orElseThrow(() -> new IllegalArgumentException("No such node: " + hostname));
+    }
+
+    private static List<Node> allNodes(ControllerTester tester) {
+        List<Node> nodes = new ArrayList<>();
+        for (var zone : tester.zoneRegistry().zones().controllerUpgraded().all().ids()) {
+            nodes.addAll(tester.serviceRegistry().configServer().nodeRepository().list(zone));
+        }
+        return nodes;
+    }
+
+    private static String switchHostname(Node node) {
+        return "tor-" + node.hostname().value();
+    }
+
+    private static void addNodeEntities(ControllerTester tester) {
+        for (var node : allNodes(tester)) {
+            if (!node.type().isHost()) continue;
+            NodeEntity nodeEntity = new NodeEntity(node.hostname().value(), "", "", switchHostname(node));
+            tester.serviceRegistry().entityService().addNodeEntity(nodeEntity);
+        }
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java
index 1da7bdeca19..f327851f902 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java
@@ -2,10 +2,9 @@
 package com.yahoo.vespa.hosted.controller.restapi.application;
 
 import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
@@ -25,8 +24,8 @@ import java.util.Collections;
 import java.util.Set;
 
 import static com.yahoo.application.container.handler.Request.Method.GET;
-import static com.yahoo.application.container.handler.Request.Method.PUT;
 import static com.yahoo.application.container.handler.Request.Method.POST;
+import static com.yahoo.application.container.handler.Request.Method.PUT;
 import static com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiTest.createApplicationSubmissionData;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
@@ -45,7 +44,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest {
     public void before() {
         tester = new ContainerTester(container, responseFiles);
         ((InMemoryFlagSource) tester.controller().flagSource())
-                .withBooleanFlag(Flags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true);
+                .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true);
         deploymentTester = new DeploymentTester(new ControllerTester(tester));
         deploymentTester.controllerTester().computeVersionStatus();
         setupTenantAndApplication();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index c061bfed21a..5523ce8dd9f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -6,8 +6,8 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.test.ManualClock;
-import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
@@ -42,7 +42,7 @@ public class ControllerApiTest extends ControllerContainerTest {
     public void testControllerApi() {
         tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/", "", Request.Method.GET), new File("root.json"));
 
-        ((InMemoryFlagSource) tester.controller().flagSource()).withListFlag(Flags.INACTIVE_MAINTENANCE_JOBS.id(), List.of("DeploymentExpirer"), String.class);
+        ((InMemoryFlagSource) tester.controller().flagSource()).withListFlag(PermanentFlags.INACTIVE_MAINTENANCE_JOBS.id(), List.of("DeploymentExpirer"), String.class);
 
         // GET a list of all maintenance jobs
         tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/", "", Request.Method.GET),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index c1ee1489cd4..5ab087aab17 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -34,6 +34,9 @@
       "name": "HostRepairMaintainer"
     },
     {
+      "name": "HostSwitchUpdater"
+    },
+    {
       "name": "JobRunner"
     },
     {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
index 0ed30871ded..547ce769e87 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
@@ -4,8 +4,8 @@ package com.yahoo.vespa.hosted.controller.restapi.user;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.user.User;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
@@ -19,7 +19,6 @@ import java.util.Set;
 
 import static com.yahoo.application.container.handler.Request.Method.DELETE;
 import static com.yahoo.application.container.handler.Request.Method.POST;
-import static com.yahoo.application.container.handler.Request.Method.PUT;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -203,7 +202,7 @@ public class UserApiTest extends ControllerContainerCloudTest {
     public void userMetadataTest() {
         ContainerTester tester = new ContainerTester(container, responseFiles);
         ((InMemoryFlagSource) tester.controller().flagSource())
-                .withBooleanFlag(Flags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true);
+                .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true);
         ControllerTester controller = new ControllerTester(tester);
         Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant());
         User user = new User("dev@domail", "Joe Developer", "dev", null);