diff options
10 files changed, 201 insertions, 84 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index de5d8de1f9e..a511b33313b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -395,7 +395,7 @@ public class ApplicationController { deploySystemApplicationPackage(application, zone, version); } else { // Deploy by calling node repository directly - application.nodeTypes().forEach(nodeType -> configServer().nodeRepository().upgrade(zone, nodeType, version)); + configServer().nodeRepository().upgrade(zone, application.nodeType(), version); } } @@ -406,7 +406,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + return deploy(application.id(), applicationPackage, zone, options, Set.of(), Set.of()); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 3eeaf09c10b..d31ae185b76 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -11,8 +11,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceCon import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; /** * This represents a system-level application in hosted Vespa. All infrastructure nodes in a hosted Vespa zones are @@ -25,23 +23,17 @@ public enum SystemApplication { configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), - zone(ApplicationId.from("hosted-vespa", "routing", "default"), Set.of(NodeType.proxy, NodeType.host), + tenantHost(ApplicationId.from("hosted-vespa", "tenant-host", "default"), NodeType.host), + zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, configServerHost, proxyHost, configServer); private final ApplicationId id; - private final Set<NodeType> nodeTypes; + private final NodeType nodeType; private final List<SystemApplication> dependencies; SystemApplication(ApplicationId id, NodeType nodeType, SystemApplication... dependencies) { - this(id, Set.of(nodeType), dependencies); - } - - SystemApplication(ApplicationId id, Set<NodeType> nodeTypes, SystemApplication... dependencies) { - if (nodeTypes.isEmpty()) { - throw new IllegalArgumentException("Node types must be non-empty"); - } this.id = id; - this.nodeTypes = Set.copyOf(nodeTypes); + this.nodeType = nodeType; this.dependencies = List.of(dependencies); } @@ -49,9 +41,9 @@ public enum SystemApplication { return id; } - /** The node type(s) that are implicitly allocated to this */ - public Set<NodeType> nodeTypes() { - return nodeTypes; + /** The node type that is implicitly allocated to this */ + public NodeType nodeType() { + return nodeType; } /** Returns the system applications that should upgrade before this */ @@ -73,8 +65,8 @@ public enum SystemApplication { } /** Returns the node types of this that should receive OS upgrades */ - public Set<NodeType> nodeTypesWithUpgradableOs() { - return nodeTypes().stream().filter(NodeType::isDockerHost).collect(Collectors.toSet()); + public boolean isEligibleForOsUpgrades() { + return nodeType.isDockerHost(); } /** All known system applications */ @@ -84,7 +76,7 @@ public enum SystemApplication { @Override public String toString() { - return String.format("system application %s of type %s", id, nodeTypes); + return String.format("system application %s of type %s", id, nodeType); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 3b521657f15..ed3dd552085 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -39,12 +39,11 @@ public class OsUpgrader extends InfrastructureUpgrader { @Override protected void upgrade(Version target, SystemApplication application, ZoneId zone) { - if (wantedVersion(zone, application, target).equals(target)) { + if (!application.isEligibleForOsUpgrades() || wantedVersion(zone, application, target).equals(target)) { return; } log.info(String.format("Upgrading OS of %s to version %s in %s", application.id(), target, zone)); - application.nodeTypesWithUpgradableOs().forEach(nodeType -> controller().configServer().nodeRepository() - .upgradeOs(zone, nodeType, target)); + controller().configServer().nodeRepository().upgradeOs(zone, application.nodeType(), target); } @Override @@ -77,7 +76,7 @@ public class OsUpgrader extends InfrastructureUpgrader { /** Returns whether node in application should be upgraded by this */ public static boolean eligibleForUpgrade(Node node, SystemApplication application) { return upgradableNodeStates.contains(node.state()) && - application.nodeTypesWithUpgradableOs().contains(node.type()); + application.isEligibleForOsUpgrades(); } private static String name(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index d55855a2f36..f9ce75d2297 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -67,8 +67,8 @@ public class OsVersionStatus { controller.osVersions().forEach(osVersion -> versions.put(osVersion, new ArrayList<>())); for (SystemApplication application : SystemApplication.all()) { - if (application.nodeTypesWithUpgradableOs().isEmpty()) { - continue; // Avoid querying applications that do not contain nodes with upgradable OS + if (!application.isEligibleForOsUpgrades()) { + continue; // Avoid querying applications that are not eligible for OS upgrades } for (ZoneId zone : zonesToUpgrade(controller)) { controller.configServer().nodeRepository().list(zone, application.id()).stream() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 31b48bea883..d0cf6a9bdf7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -67,7 +67,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Inject public ConfigServerMock(ZoneRegistryMock zoneRegistry) { - bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.all(), Optional.empty()); + bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.all()); } /** Sets the ConfigChangeActions that will be returned on next deployment. */ @@ -90,28 +90,22 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } public void bootstrap(List<ZoneId> zones, SystemApplication... applications) { - bootstrap(zones, List.of(applications), Optional.empty()); + bootstrap(zones, List.of(applications)); } - public void bootstrap(List<ZoneId> zones, List<SystemApplication> applications, Optional<NodeType> type) { + public void bootstrap(List<ZoneId> zones, List<SystemApplication> applications) { nodeRepository().clear(); - addNodes(zones, applications, type); + addNodes(zones, applications); } - public void addNodes(List<ZoneId> zones, List<SystemApplication> applications, Optional<NodeType> type) { + public void addNodes(List<ZoneId> zones, List<SystemApplication> applications) { for (ZoneId zone : zones) { for (SystemApplication application : applications) { - NodeType nodeType = type.orElseGet(() -> { - // Zone application has two node types. Use proxy - if (application == SystemApplication.zone) return NodeType.proxy; - if (application.nodeTypes().size() != 1) throw new IllegalArgumentException(application + " has several node types. Unable to detect type automatically"); - return application.nodeTypes().iterator().next(); - }); List<Node> nodes = IntStream.rangeClosed(1, 3) .mapToObj(i -> new Node( HostName.from("node-" + i + "-" + application.id().application() .value()), - Node.State.active, nodeType, + Node.State.active, application.nodeType(), Optional.of(application.id()), initialVersion, initialVersion diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 7a008d1f478..7cd82d83500 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -22,7 +22,6 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; -import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -60,17 +59,15 @@ public class OsUpgraderTest { // Bootstrap system tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4, zone5), - singletonList(SystemApplication.zone), - Optional.of(NodeType.host)); + List.of(SystemApplication.tenantHost)); // Add system applications that exist in a real system, but are currently not upgraded tester.configServer().addNodes(List.of(zone1, zone2, zone3, zone4, zone5), - Collections.singletonList(SystemApplication.configServer), - Optional.empty()); + List.of(SystemApplication.configServer)); // Fail a few nodes. Failed nodes should not affect versions - failNodeIn(zone1, SystemApplication.zone); - failNodeIn(zone3, SystemApplication.zone); + failNodeIn(zone1, SystemApplication.tenantHost); + failNodeIn(zone3, SystemApplication.tenantHost); // New OS version released Version version1 = Version.fromString("7.1"); @@ -82,13 +79,13 @@ public class OsUpgraderTest { // zone 1: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.zone, zone1); + assertWanted(version1, SystemApplication.tenantHost, zone1); // Other zones remain on previous version (none) assertWanted(Version.emptyVersion, SystemApplication.zone, zone2, zone3, zone4); // zone 1: completes upgrade - completeUpgrade(version1, SystemApplication.zone, zone1); + completeUpgrade(version1, SystemApplication.tenantHost, zone1); statusUpdater.maintain(); assertEquals(2, nodesOn(version1).size()); assertEquals(11, nodesOn(Version.emptyVersion).size()); @@ -98,21 +95,21 @@ public class OsUpgraderTest { assertWanted(version1, SystemApplication.zone, zone2, zone3); // zone 4: still on previous version - assertWanted(Version.emptyVersion, SystemApplication.zone, zone4); + assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4); // zone 2 and 3: completes upgrade - completeUpgrade(version1, SystemApplication.zone, zone2, zone3); + completeUpgrade(version1, SystemApplication.tenantHost, zone2, zone3); // zone 4: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.zone, zone4); + assertWanted(version1, SystemApplication.tenantHost, zone4); // zone 4: completes upgrade - completeUpgrade(version1, SystemApplication.zone, zone4); + completeUpgrade(version1, SystemApplication.tenantHost, zone4); // Next run does nothing as all zones are upgraded osUpgrader.maintain(); - assertWanted(version1, SystemApplication.zone, zone1, zone2, zone3, zone4); + assertWanted(version1, SystemApplication.tenantHost, zone1, zone2, zone3, zone4); statusUpdater.maintain(); assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() .allMatch(node -> node.version().equals(version1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index d28b36265e3..180dff9feba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -15,7 +15,6 @@ import org.junit.Test; import java.time.Duration; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -184,7 +183,7 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.all(), Optional.empty()); + tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.all()); tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); @@ -198,7 +197,8 @@ public class SystemUpgraderTest { systemUpgrader.maintain(); List<SystemApplication> allExceptZone = List.of(SystemApplication.configServerHost, SystemApplication.configServer, - SystemApplication.proxyHost); + SystemApplication.proxyHost, + SystemApplication.tenantHost); completeUpgrade(allExceptZone, version2, zone1); systemUpgrader.maintain(); completeUpgrade(SystemApplication.zone, version2, zone1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json index e1d81a874dc..17f90259fa8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json @@ -6,6 +6,11 @@ "cloud": "cloud1", "nodes": [ { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { "hostname": "node-3-configserver-host", "environment": "prod", "region": "us-east-3" @@ -18,7 +23,7 @@ { "hostname": "node-1-configserver-host", "environment": "prod", - "region": "us-east-3" + "region": "us-west-1" }, { "hostname": "node-3-configserver-host", @@ -31,12 +36,12 @@ "region": "us-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", - "region": "us-west-1" + "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "us-east-3" }, @@ -46,22 +51,47 @@ "region": "us-east-3" }, { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { "hostname": "node-3-proxy-host", "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-tenant-host", + "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-tenant-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-tenant-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-tenant-host", "environment": "prod", "region": "us-west-1" } @@ -73,6 +103,11 @@ "cloud": "cloud2", "nodes": [ { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { "hostname": "node-3-configserver-host", "environment": "prod", "region": "eu-west-1" @@ -83,12 +118,12 @@ "region": "eu-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "eu-west-1" }, @@ -98,7 +133,17 @@ "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json index 9c1625fdcd5..86bc272fcd1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json @@ -6,6 +6,11 @@ "cloud": "cloud1", "nodes": [ { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { "hostname": "node-3-configserver-host", "environment": "prod", "region": "us-east-3" @@ -18,7 +23,7 @@ { "hostname": "node-1-configserver-host", "environment": "prod", - "region": "us-east-3" + "region": "us-west-1" }, { "hostname": "node-3-configserver-host", @@ -31,12 +36,12 @@ "region": "us-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", - "region": "us-west-1" + "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "us-east-3" }, @@ -46,22 +51,47 @@ "region": "us-east-3" }, { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { "hostname": "node-3-proxy-host", "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-tenant-host", + "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-tenant-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-tenant-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-tenant-host", "environment": "prod", "region": "us-west-1" } @@ -73,6 +103,11 @@ "cloud": "cloud2", "nodes": [ { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { "hostname": "node-3-configserver-host", "environment": "prod", "region": "eu-west-1" @@ -83,12 +118,12 @@ "region": "eu-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "eu-west-1" }, @@ -98,7 +133,17 @@ "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json index 48a96b70df7..e8007fbf6c5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json @@ -6,6 +6,11 @@ "cloud": "cloud1", "nodes": [ { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { "hostname": "node-3-configserver-host", "environment": "prod", "region": "us-west-1" @@ -16,12 +21,12 @@ "region": "us-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "us-west-1" }, @@ -31,7 +36,17 @@ "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "us-west-1" } @@ -43,6 +58,11 @@ "cloud": "cloud1", "nodes": [ { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { "hostname": "node-3-configserver-host", "environment": "prod", "region": "us-east-3" @@ -53,12 +73,12 @@ "region": "us-east-3" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "us-east-3" }, @@ -68,7 +88,17 @@ "region": "us-east-3" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "us-east-3" } @@ -80,6 +110,11 @@ "cloud": "cloud2", "nodes": [ { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { "hostname": "node-3-configserver-host", "environment": "prod", "region": "eu-west-1" @@ -90,12 +125,12 @@ "region": "eu-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "eu-west-1" }, @@ -105,7 +140,17 @@ "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "eu-west-1" } |