From 8eb4c2dd3aa750c2ebcee4e1736f99e98d89c3bf Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 29 Jul 2021 13:19:52 +0200 Subject: Revert "Revert "Pass around orchestration parameters"" --- .../orchestrator/model/ApplicationApiFactory.java | 11 ++- .../orchestrator/model/ApplicationApiImpl.java | 9 +- .../vespa/orchestrator/model/ClusterApiImpl.java | 8 +- .../orchestrator/policy/ApplicationParams.java | 68 +++++++++++++ .../vespa/orchestrator/policy/ClusterParams.java | 61 ++++++++++++ .../policy/HostedVespaOrchestration.java | 106 +++++++++++++++++++++ .../orchestrator/policy/OrchestrationParams.java | 53 +++++++++++ .../orchestrator/model/ClusterApiImplTest.java | 17 +++- .../vespa/orchestrator/model/ModelTestUtils.java | 17 +++- 9 files changed, 333 insertions(+), 17 deletions(-) create mode 100644 orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ApplicationParams.java create mode 100644 orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java create mode 100644 orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java create mode 100644 orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/OrchestrationParams.java (limited to 'orchestrator') diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java index d913a1791ba..a3f2fd6e235 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java @@ -1,7 +1,11 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.model; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; +import com.yahoo.vespa.orchestrator.policy.HostedVespaOrchestration; +import com.yahoo.vespa.orchestrator.policy.OrchestrationParams; import com.yahoo.vespa.orchestrator.status.ApplicationLock; import java.time.Clock; @@ -11,19 +15,20 @@ import java.time.Clock; */ public class ApplicationApiFactory { - private final int numberOfConfigServers; + private final OrchestrationParams orchestrationParams; private final Clock clock; public ApplicationApiFactory(int numberOfConfigServers, Clock clock) { - this.numberOfConfigServers = numberOfConfigServers; + this.orchestrationParams = HostedVespaOrchestration.create(numberOfConfigServers, 0); this.clock = clock; } public ApplicationApi create(NodeGroup nodeGroup, ApplicationLock lock, ClusterControllerClientFactory clusterControllerClientFactory) { + ApplicationId applicationId = OrchestratorUtil.toApplicationId(lock.getApplicationInstanceReference()); return new ApplicationApiImpl(nodeGroup, lock, clusterControllerClientFactory, - numberOfConfigServers, clock); + orchestrationParams.getApplicationParams(applicationId), clock); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java index efbf9ff7981..804aff70a7f 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; +import com.yahoo.vespa.orchestrator.policy.ApplicationParams; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.ApplicationLock; import com.yahoo.vespa.orchestrator.status.HostInfos; @@ -38,13 +39,13 @@ public class ApplicationApiImpl implements ApplicationApi { public ApplicationApiImpl(NodeGroup nodeGroup, ApplicationLock lock, ClusterControllerClientFactory clusterControllerClientFactory, - int numberOfConfigServers, Clock clock) { + ApplicationParams params, Clock clock) { this.applicationInstance = nodeGroup.getApplication(); this.nodeGroup = nodeGroup; this.lock = lock; this.clock = clock; this.hostInfos = lock.getHostInfos(); - this.clusterInOrder = makeClustersInOrder(nodeGroup, hostInfos, clusterControllerClientFactory, numberOfConfigServers); + this.clusterInOrder = makeClustersInOrder(nodeGroup, hostInfos, clusterControllerClientFactory, params); } @Override @@ -113,7 +114,7 @@ public class ApplicationApiImpl implements ApplicationApi { private List makeClustersInOrder(NodeGroup nodeGroup, HostInfos hostInfos, ClusterControllerClientFactory clusterControllerClientFactory, - int numberOfConfigServers) { + ApplicationParams params) { Set clustersInGroup = getServiceClustersInGroup(nodeGroup); return clustersInGroup.stream() .map(serviceCluster -> new ClusterApiImpl( @@ -122,7 +123,7 @@ public class ApplicationApiImpl implements ApplicationApi { nodeGroup, hostInfos, clusterControllerClientFactory, - numberOfConfigServers, + params.clusterParamsFor(serviceCluster.clusterId(), serviceCluster.serviceType()), clock)) .sorted(ApplicationApiImpl::compareClusters) .collect(Collectors.toList()); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index 57dcb5f3069..6e88b227e61 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; +import com.yahoo.vespa.orchestrator.policy.ClusterParams; import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; @@ -59,7 +60,7 @@ class ClusterApiImpl implements ClusterApi { NodeGroup nodeGroup, HostInfos hostInfos, ClusterControllerClientFactory clusterControllerClientFactory, - int numberOfConfigServers, + ClusterParams clusterParams, Clock clock) { this.applicationApi = applicationApi; this.serviceCluster = serviceCluster; @@ -81,9 +82,8 @@ class ClusterApiImpl implements ClusterApi { servicesDownAndNotInGroup = servicesNotInGroup.stream().filter(this::serviceEffectivelyDown).collect(Collectors.toSet()); int serviceInstances = serviceCluster.serviceInstances().size(); - if ((serviceCluster.isConfigServerLike() || serviceCluster.isConfigServerHostLike()) && - serviceInstances < numberOfConfigServers) { - missingServices = numberOfConfigServers - serviceInstances; + if (clusterParams.size().isPresent() && serviceInstances < clusterParams.size().getAsInt()) { + missingServices = clusterParams.size().getAsInt() - serviceInstances; descriptionOfMissingServices = missingServices + " missing " + serviceCluster.nodeDescription(missingServices > 1); } else { missingServices = 0; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ApplicationParams.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ApplicationParams.java new file mode 100644 index 00000000000..97574fc90ce --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ApplicationParams.java @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.policy; + +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceClusterKey; +import com.yahoo.vespa.applicationmodel.ServiceType; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Per-application parameters controlling the orchestration. + * + * @author hakonhall + */ +public class ApplicationParams { + + private static final ApplicationParams DEFAULT = new ApplicationParams.Builder().build(); + + private final Map clusterParams; + + public static class Builder { + private final Map clusterParams = new HashMap<>(); + + public Builder() {} + + public Builder add(ClusterId clusterId, ServiceType serviceType, ClusterParams clusterParams) { + this.clusterParams.put(new ServiceClusterKey(clusterId, serviceType), clusterParams); + return this; + } + + public ApplicationParams build() { + return new ApplicationParams(clusterParams); + } + } + + public static ApplicationParams getDefault() { + return DEFAULT; + } + + private ApplicationParams(Map clusterParams) { + this.clusterParams = Map.copyOf(clusterParams); + } + + public ClusterParams clusterParamsFor(ServiceCluster serviceCluster) { + return clusterParamsFor(serviceCluster.clusterId(), serviceCluster.serviceType()); + } + + public ClusterParams clusterParamsFor(ClusterId clusterId, ServiceType serviceType) { + var key = new ServiceClusterKey(clusterId, serviceType); + return clusterParams.getOrDefault(key, ClusterParams.getDefault()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ApplicationParams that = (ApplicationParams) o; + return clusterParams.equals(that.clusterParams); + } + + @Override + public int hashCode() { + return Objects.hash(clusterParams); + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java new file mode 100644 index 00000000000..b6f90ab7302 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java @@ -0,0 +1,61 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.policy; + +import java.util.Objects; +import java.util.OptionalInt; + +/** + * Parameters controlling orchestration of a particular service cluster of an implied application. + * + * @author hakonhall + */ +public class ClusterParams { + + private static final ClusterParams DEFAULT = new ClusterParams.Builder().build(); + + private final int size; + + public static class Builder { + private int size = 0; + + public Builder() {} + + public Builder setSize(int size) { + this.size = size; + return this; + } + + public ClusterParams build() { + return new ClusterParams(size); + } + } + + public static ClusterParams getDefault() { + return DEFAULT; + } + + private ClusterParams(int size) { + this.size = size; + } + + /** + * The expected and ideal number of members of the cluster: Count missing services as down, + * and expected to be added to the application soon. + */ + public OptionalInt size() { + return size > 0 ? OptionalInt.of(size) : OptionalInt.empty(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterParams that = (ClusterParams) o; + return size == that.size; + } + + @Override + public int hashCode() { + return Objects.hash(size); + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java new file mode 100644 index 00000000000..d444f922f82 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java @@ -0,0 +1,106 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.policy; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.duper.ConfigServerHostApplication; +import com.yahoo.vespa.service.duper.ControllerApplication; +import com.yahoo.vespa.service.duper.ControllerHostApplication; +import com.yahoo.vespa.service.duper.ProxyApplication; +import com.yahoo.vespa.service.duper.ProxyHostApplication; + +/** + * Creates orchestration parameters for hosted Vespa. + * + * @author hakonhall + */ +public class HostedVespaOrchestration { + public static OrchestrationParams create(ConfigserverConfig configserverConfig) { + // todo: get the number of proxy nodes + return create(configserverConfig.zookeeperserver().size(), 0); + } + + public static OrchestrationParams create(int numConfigServers, int numProxies) { + // We'll create parameters for both the controller and config server applications, even though + // only one of them is present, as (a) no harm is done by having the extra parameters, and + // (b) it leads to simpler code below. + + return new OrchestrationParams.Builder() + + // Controller host + .addApplicationParams(new ControllerHostApplication().getApplicationId(), + new ApplicationParams + .Builder() + .add(ClusterId.CONTROLLER, + ServiceType.HOST_ADMIN, + new ClusterParams + .Builder() + .setSize(numConfigServers) + .build()) + .build()) + + // Controller + .addApplicationParams(new ControllerApplication().getApplicationId(), + new ApplicationParams + .Builder() + .add(ClusterId.CONTROLLER, + ServiceType.CONTROLLER, + new ClusterParams + .Builder() + .setSize(numConfigServers) + .build()) + .build()) + + // Config server host + .addApplicationParams(new ConfigServerHostApplication().getApplicationId(), + new ApplicationParams + .Builder() + .add(ClusterId.CONFIG_SERVER_HOST, + ServiceType.HOST_ADMIN, + new ClusterParams + .Builder() + .setSize(numConfigServers) + .build()) + .build()) + + // Config server + .addApplicationParams(new ConfigServerApplication().getApplicationId(), + new ApplicationParams + .Builder() + .add(ClusterId.CONFIG_SERVER, + ServiceType.CONFIG_SERVER, + new ClusterParams + .Builder() + .setSize(numConfigServers) + .build()) + .build()) + + // Proxy host + .addApplicationParams(new ProxyHostApplication().getApplicationId(), + new ApplicationParams + .Builder() + .add(ClusterId.PROXY_HOST, + ServiceType.HOST_ADMIN, + new ClusterParams + .Builder() + .setSize(numProxies) + .build()) + .build()) + + // Proxy + .addApplicationParams(new ProxyApplication().getApplicationId(), + new ApplicationParams + .Builder() + .add(ClusterId.ROUTING, + ServiceType.CONTAINER, + new ClusterParams + .Builder() + .setSize(numProxies) + .build()) + .build()) + + .build(); + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/OrchestrationParams.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/OrchestrationParams.java new file mode 100644 index 00000000000..7519564b080 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/OrchestrationParams.java @@ -0,0 +1,53 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.policy; + +import com.yahoo.config.provision.ApplicationId; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Parameters controlling orchestration. + * + * @author hakonhall + */ +public class OrchestrationParams { + private final Map applicationParams; + + public static class Builder { + private final Map applicationParams = new HashMap<>(); + + public Builder() {} + + public Builder addApplicationParams(ApplicationId applicationId, ApplicationParams params) { + this.applicationParams.put(applicationId, params); + return this; + } + + public OrchestrationParams build() { + return new OrchestrationParams(applicationParams); + } + } + + private OrchestrationParams(Map applicationParams) { + this.applicationParams = Map.copyOf(applicationParams); + } + + public ApplicationParams getApplicationParams(ApplicationId applicationId) { + return applicationParams.getOrDefault(applicationId, ApplicationParams.getDefault()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OrchestrationParams that = (OrchestrationParams) o; + return applicationParams.equals(that.applicationParams); + } + + @Override + public int hashCode() { + return Objects.hash(applicationParams); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 6bf46052933..237d418aae5 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.OrchestratorUtil; +import com.yahoo.vespa.orchestrator.policy.ClusterParams; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; @@ -88,7 +89,9 @@ public class ClusterApiImplTest { serviceCluster, new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), hostName5), modelUtils.getHostInfos(), - modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, clock); + modelUtils.getClusterControllerClientFactory(), + ModelTestUtils.APPLICATION_PARAMS.clusterParamsFor(serviceCluster), + clock); assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo()); assertFalse(clusterApi.isStorageCluster()); @@ -274,7 +277,9 @@ public class ClusterApiImplTest { serviceCluster, new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), groupNodes), modelUtils.getHostInfos(), - modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, clock); + modelUtils.getClusterControllerClientFactory(), + ModelTestUtils.APPLICATION_PARAMS.clusterParamsFor(serviceCluster), + clock); assertEquals(expectedNoServicesInGroupIsUp.map(SuspensionReasons::getMessagesInOrder), clusterApi.reasonsForNoServicesInGroupIsUp().map(SuspensionReasons::getMessagesInOrder)); @@ -305,7 +310,9 @@ public class ClusterApiImplTest { serviceCluster, new NodeGroup(applicationInstance, hostName1, hostName3), new HostInfos(), - modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, clock); + modelUtils.getClusterControllerClientFactory(), + ModelTestUtils.APPLICATION_PARAMS.clusterParamsFor(serviceCluster), + clock); assertTrue(clusterApi.isStorageCluster()); assertEquals(Optional.of(hostName1), clusterApi.storageNodeInGroup().map(storageNode -> storageNode.hostName())); @@ -353,7 +360,9 @@ public class ClusterApiImplTest { serviceCluster, new NodeGroup(application, hostnames.get(0)), modelUtils.getHostInfos(), - modelUtils.getClusterControllerClientFactory(), clusterSize, clock); + modelUtils.getClusterControllerClientFactory(), + new ClusterParams.Builder().setSize(clusterSize).build(), + clock); assertEquals(clusterSize - serviceStatusList.size(), clusterApi.missingServices()); assertEquals(clusterSize == serviceStatusList.size(), clusterApi.noServicesOutsideGroupIsDown()); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java index 7cb20350845..6ec03ba3c44 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java @@ -24,10 +24,14 @@ import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; +import com.yahoo.vespa.orchestrator.policy.ApplicationParams; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; +import com.yahoo.vespa.orchestrator.policy.HostedVespaOrchestration; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import com.yahoo.vespa.orchestrator.policy.OrchestrationParams; import com.yahoo.vespa.orchestrator.status.ApplicationLock; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostInfos; @@ -54,7 +58,16 @@ import static org.mockito.Mockito.mock; */ class ModelTestUtils { + private static final TenantId TENANT_ID = new TenantId("tenant"); + private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = + new ApplicationInstanceId("application-name:foo:bar:default"); + public static final int NUMBER_OF_CONFIG_SERVERS = 3; + public static final OrchestrationParams ORCHESTRATION_PARAMS = + HostedVespaOrchestration.create(NUMBER_OF_CONFIG_SERVERS, 0); + public static final ApplicationParams APPLICATION_PARAMS = ORCHESTRATION_PARAMS + .getApplicationParams(OrchestratorUtil.toApplicationId( + new ApplicationInstanceReference(TENANT_ID, APPLICATION_INSTANCE_ID))); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final Map applications = new HashMap<>(); @@ -134,8 +147,8 @@ class ModelTestUtils { Set serviceClusterSet = new HashSet<>(serviceClusters); ApplicationInstance application = new ApplicationInstance( - new TenantId("tenant"), - new ApplicationInstanceId("application-name:foo:bar:default"), + TENANT_ID, + APPLICATION_INSTANCE_ID, serviceClusterSet); applications.put(application.reference(), application); -- cgit v1.2.3