diff options
17 files changed, 96 insertions, 15 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java index ad73905c395..704217843fb 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; import java.util.List; +import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -19,14 +20,17 @@ public enum InfrastructureApplication { CONFIG_SERVER("zone-config-servers", NodeType.config), PROXY_HOST("proxy-host", NodeType.proxyhost), PROXY("routing", NodeType.proxy), - TENANT_HOST("tenant-host", NodeType.host); + TENANT_HOST("tenant-host", NodeType.host), + DEV_HOST("dev-host", NodeType.devhost); private final ApplicationId id; private final NodeType nodeType; /** Returns all applications that MAY be encountered in hosted Vespa, e.g. not DEV_HOST. */ - public static List<InfrastructureApplication> toList() { - return List.of(values()); + public static List<InfrastructureApplication> inHosted() { + return Stream.of(values()) + .filter(application -> application != DEV_HOST) + .collect(Collectors.toList()); } public static InfrastructureApplication withNodeType(NodeType nodeType) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index 25ec7485c24..a617c7bcea3 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -32,7 +32,10 @@ public enum NodeType { controller("Controller node"), /** Host of a controller node */ - controllerhost("Controller host", controller); + controllerhost("Controller host", controller), + + /** Host capable of running multiple node types, only used in {@link SystemName#dev} */ + devhost("Dev host", config, controller, tenant); private final String description; private final List<NodeType> childNodeTypes; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index fef5babc806..e2349f6f63f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -24,7 +24,10 @@ public enum SystemName { Public(true, false), /** Continuous deployment system for testing the Public system */ - PublicCd(true, true); + PublicCd(true, true), + + /** Local development system */ + dev(false, false); private final boolean isPublic; private final boolean isCd; @@ -40,6 +43,7 @@ public enum SystemName { public static SystemName from(String value) { switch (value.toLowerCase()) { + case "dev": return dev; case "cd": return cd; case "main": return main; case "public": return Public; @@ -50,6 +54,7 @@ public enum SystemName { public String value() { switch (this) { + case dev: return "dev"; case cd: return "cd"; case main: return "main"; case Public: return "public"; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index d40027d91a7..5768a7a3b3b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -49,7 +49,7 @@ enum Policy { /** Access to create a user tenant in select systems. */ user(Privilege.grant(Action.create, Action.update) .on(PathGroup.user) - .in(SystemName.main, SystemName.cd)), + .in(SystemName.main, SystemName.cd, SystemName.dev)), /** Access to create a tenant. */ tenantCreate(Privilege.grant(Action.create) @@ -130,7 +130,7 @@ enum Policy { /** Read access to all information in select systems. */ classifiedRead(Privilege.grant(Action.read) .on(PathGroup.allExcept(PathGroup.classifiedOperator)) - .in(SystemName.main, SystemName.cd)), + .in(SystemName.main, SystemName.cd, SystemName.dev)), /** Read access to public info. */ publicRead(Privilege.grant(Action.read) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index f7dfadbffbc..913d6dfeab8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -135,7 +135,7 @@ public class ControllerMaintenance extends AbstractComponent { public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); - this.defaultInterval = duration(system.isCd() ? 1 : 5, MINUTES); + this.defaultInterval = duration(system.isCd() || system == SystemName.dev ? 1 : 5, MINUTES); this.outstandingChangeDeployer = duration(3, MINUTES); this.versionStatusUpdater = duration(3, MINUTES); this.readyJobsTrigger = duration(1, MINUTES); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java index 6e88a41b8c6..56e690864ac 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerNetworkMode.java @@ -14,7 +14,10 @@ public enum ContainerNetworkMode { NPT("vespa-bridge"), /** A host running a single container in the host network namespace. */ - HOST_NETWORK("host"); + HOST_NETWORK("host"), + + /** A host running multiple containers in a shared local network. */ + LOCAL("vespa-bridge"); private final String networkName; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java index a1f36a4f1a5..4f913bb55dd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureVersions.java @@ -88,6 +88,7 @@ public class InfrastructureVersions { case controllerhost: case proxyhost: case host: + case devhost: break; default: throw new IllegalArgumentException("Target version for type " + nodeType + " is not allowed"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index cd1b786afd1..543972a9cb3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -554,6 +554,7 @@ public class NodeSerializer { case "confighost": return NodeType.confighost; case "controller": return NodeType.controller; case "controllerhost": return NodeType.controllerhost; + case "devhost": return NodeType.devhost; default : throw new IllegalArgumentException("Unknown node type '" + typeString + "'"); } } @@ -568,6 +569,7 @@ public class NodeSerializer { case confighost: return "confighost"; case controller: return "controller"; case controllerhost: return "controllerhost"; + case devhost: return "devhost"; } throw new IllegalArgumentException("Serialized form of '" + type + "' not defined"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 290a3f8f947..4088d717a67 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -71,6 +72,10 @@ public class CapacityPolicies { public NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { if (clusterType == ClusterSpec.Type.admin) { + if (zone.system() == SystemName.dev) { + // Use small logserver in dev system + return new NodeResources(0.1, 1, 10, 0.3); + } return zone.getCloud().dynamicProvisioning() && ! sharedHosts.apply(clusterType) ? new NodeResources(0.5, 4, 50, 0.3) : new NodeResources(0.5, 2, 50, 0.3); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 310f921367e..d5dbe08dca9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -71,6 +71,7 @@ public class NodeResourceLimits { } private double minAdvertisedMemoryGb(ClusterSpec.Type clusterType) { + if (zone().system() == SystemName.dev) return 1; // Allow small containers in dev system if (clusterType == ClusterSpec.Type.admin) return 1; return 4; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java index 6282c072001..be011c886a5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java @@ -53,6 +53,7 @@ public class NodeSerializer { case "confighost": return NodeType.confighost; case "controller": return NodeType.controller; case "controllerhost": return NodeType.controllerhost; + case "devhost": return NodeType.devhost; default: throw new IllegalArgumentException("Unknown node type '" + nodeType + "'"); } } @@ -67,6 +68,7 @@ public class NodeSerializer { case confighost: return "confighost"; case controller: return "controller"; case controllerhost: return "controllerhost"; + case devhost: return "devhost"; default: throw new IllegalArgumentException("Unknown node type '" + type.name() + "'"); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index 31643b2ac79..390df571b04 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -128,6 +128,26 @@ public class HostCapacityTest { } @Test + public void devhostCapacityTest() { + // Dev host can assign both configserver and tenant containers. + + var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container"); + var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), createIps(2, 10)), "devhost", nodeFlavors.getFlavorOrThrow("devhost"), NodeType.devhost).build(); + + var cfg = Node.reserve(Set.of("::2"), "cfg", "devhost", resources0, NodeType.config).build(); + + var nodes = new ArrayList<>(List.of(cfg)); + var capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); + assertTrue(capacity.hasCapacity(devHost, resources0)); + + var container1 = Node.reserve(Set.of("::3"), "container1", "devhost", resources0, NodeType.tenant).build(); + nodes = new ArrayList<>(List.of(cfg, container1)); + capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); + assertFalse(capacity.hasCapacity(devHost, resources0)); + + } + + @Test public void verifyCapacityFromAddresses() { Node nodeA = Node.reserve(Set.of("::2"), "nodeA", "host1", resources0, NodeType.tenant).build(); Node nodeB = Node.reserve(Set.of("::3"), "nodeB", "host1", resources0, NodeType.tenant).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index c9b384b95a6..95f25612dd7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -871,6 +871,19 @@ public class ProvisioningTest { } @Test + public void devsystem_application_deployment_on_devhost() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(SystemName.dev, Environment.dev, RegionName.from("no-central"))).build(); + + tester.makeReadyNodes(4, defaultResources, NodeType.devhost, 1); + tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.devhost); + + ApplicationId application = ProvisioningTester.applicationId(); + SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); + assertEquals(4, state.allHosts.size()); + tester.activate(application, state.allHosts); + } + + @Test public void allocates_reserved_nodes_for_type_spec_deployment() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); Function<InfraApplication, Collection<HostSpec>> prepareAndActivate = app -> tester.activate(app.getApplicationId(), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java index a88e6766050..698c7b544b3 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java @@ -61,7 +61,7 @@ public class OrchestratorTest { var zone = new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-east-1")); this.superModelManager = new MySuperModelProvider(); var duperModel = new DuperModel(); - this.duperModelManager = new DuperModelManager(true, false, superModelManager, duperModel); + this.duperModelManager = new DuperModelManager(true, false, superModelManager, duperModel, flagSource, zone.system()); this.monitorManager = mock(UnionMonitorManager.class); var metric = mock(Metric.class); var serviceMonitor = new ServiceMonitorImpl(duperModelManager, monitorManager, metric, timer, zone); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DevHostApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DevHostApplication.java new file mode 100644 index 00000000000..e522b736290 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DevHostApplication.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.duper; + +import com.yahoo.vespa.applicationmodel.InfrastructureApplication; + +/** + * @author mortent + */ +public class DevHostApplication extends HostAdminApplication { + public DevHostApplication() { + super(InfrastructureApplication.DEV_HOST); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 9215581a2b9..67d54091adc 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -9,6 +9,8 @@ import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.service.monitor.CriticalRegion; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.DuperModelListener; @@ -43,6 +45,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi static final ConfigServerApplication configServerApplication = new ConfigServerApplication(); static final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); static final TenantHostApplication tenantHostApplication = new TenantHostApplication(); + static final DevHostApplication devHostApplication = new DevHostApplication(); private final Map<ApplicationId, InfraApplication> supportedInfraApplications; @@ -58,18 +61,23 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi private boolean infraApplicationsIsComplete = false; @Inject - public DuperModelManager(ConfigserverConfig configServerConfig, SuperModelProvider superModelProvider) { + public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) { this(configServerConfig.multitenant(), configServerConfig.serverNodeType() == ConfigserverConfig.ServerNodeType.Enum.controller, - superModelProvider, new DuperModel()); + superModelProvider, new DuperModel(), flagSource, SystemName.from(configServerConfig.system())); } /** Non-private for testing */ public DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, - DuperModel duperModel) { + DuperModel duperModel, FlagSource flagSource, SystemName system) { this.duperModel = duperModel; - if (multitenant) { + if (system == SystemName.dev) { + // TODO (mortent): Support controllerApplication in dev system + supportedInfraApplications = + Stream.of(devHostApplication, configServerApplication) + .collect(Collectors.toUnmodifiableMap(InfraApplication::getApplicationId, Function.identity())); + } else if (multitenant) { supportedInfraApplications = (isController ? Stream.of(controllerHostApplication, controllerApplication) : diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java index a3af44ec911..8b0cc621358 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java @@ -33,12 +33,13 @@ public class DuperModelManagerTest { private final SuperModelProvider superModelProvider = mock(SuperModelProvider.class); private final SuperModel superModel = mock(SuperModel.class); private final DuperModel duperModel = mock(DuperModel.class); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private DuperModelManager manager; private SuperModelListener superModelListener; private void makeManager(boolean isController) { - manager = new DuperModelManager(true, isController, superModelProvider, duperModel); + manager = new DuperModelManager(true, isController, superModelProvider, duperModel, flagSource, SystemName.cd); when(superModelProvider.getSuperModel()).thenReturn(superModel); verify(duperModel, times(0)).add(any()); |