diff options
Diffstat (limited to 'config-model/src/main')
9 files changed, 30 insertions, 78 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index c8f088509c5..befe57a97e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.ProvisionLogger; import java.util.ArrayList; @@ -310,9 +309,7 @@ public class InMemoryProvisioner implements HostProvisioner { if (a.memoryGb() < b.memoryGb()) return -1; if (a.diskGb() > b.diskGb()) return 1; if (a.diskGb() < b.diskGb()) return -1; - if (a.vcpu() > b.vcpu()) return 1; - if (a.vcpu() < b.vcpu()) return -1; - return 0; + return Double.compare(a.vcpu(), b.vcpu()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 482c4477cdc..9cf5fe84c21 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -18,7 +18,6 @@ public class JvmHeapSizeValidator implements Validator { @Override public void validate(Context context) { - if (!context.deployState().featureFlags().dynamicHeapSize()) return; if (!context.deployState().isHostedTenantApplication(context.model().getAdmin().getApplicationType())) return; context.model().getContainerClusters().forEach((clusterId, appCluster) -> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 9896ca95e97..ed7646b3066 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -101,7 +101,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private int zookeeperSessionTimeoutSeconds = 30; private final int transport_events_before_wakeup; private final int transport_connections_per_target; - private final boolean dynamicHeapSize; /** The heap size % of total memory available to the JVM process. */ private final int heapSizePercentageOfAvailableMemory; @@ -115,7 +114,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public ApplicationContainerCluster(TreeConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true, 10); this.tlsClientAuthority = deployState.tlsClientAuthority(); - dynamicHeapSize = deployState.featureFlags().dynamicHeapSize(); previousHosts = Collections.unmodifiableSet(deployState.getPreviousModel().stream() .map(Model::allocatedHosts) .map(AllocatedHosts::getHosts) @@ -215,10 +213,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known // Node memory is known so convert available memory percentage to node memory percentage - double totalMemory = dynamicHeapSize - ? getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow() - : getContainers().get(0).getHostResource().realResources().memoryGb(); - double jvmHeapDeductionGb = dynamicHeapSize ? onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024) : 0; + double totalMemory = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); + double jvmHeapDeductionGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 4e56d1d1d5a..1254f8e110a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.IndexingDocproc; import com.yahoo.vespa.model.search.IndexingDocprocChain; import com.yahoo.vespa.model.search.SearchCluster; @@ -215,25 +214,21 @@ public class Content extends ConfigModel { private void buildIndexingClusters(Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { var search = content.getCluster().getSearch(); - if (!search.getIndexingDocproc().isPresent()) { - return; - } - var indexingDocproc = search.getIndexingDocproc().get(); + var indexingDocproc = search.getIndexingDocproc(); if (indexingDocproc.hasExplicitCluster()) { setExistingIndexingCluster(content, indexingDocproc, content.containers); } else { - if (search.hasIndexedCluster()) { - setContainerAsIndexingCluster(search.getIndexed(), content, modelContext, root); - } + setContainerAsIndexingCluster(search.getSearchNodes(), indexingDocproc, content, modelContext, root); } } - private void setContainerAsIndexingCluster(IndexedSearchCluster indexedSearchCluster, + private void setContainerAsIndexingCluster(List<SearchNode> cluster, + IndexingDocproc indexingDocproc, Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { if (content.containers.isEmpty()) { - createImplicitIndexingCluster(indexedSearchCluster, content, modelContext, root); + createImplicitIndexingCluster(cluster, indexingDocproc, content, modelContext, root); } else { ContainerCluster<?> targetCluster = getContainerWithDocproc(content.containers); if (targetCluster == null) @@ -242,7 +237,6 @@ public class Content extends ConfigModel { targetCluster = content.containers.iterator().next().getCluster(); addDocproc(targetCluster); - var indexingDocproc = indexedSearchCluster.getIndexingDocproc(); indexingDocproc.setClusterName(targetCluster.getName()); addIndexingChainsTo(targetCluster, content, indexingDocproc); } @@ -303,11 +297,12 @@ public class Content extends ConfigModel { } /** Create a new container cluster for indexing and add it to the Vespa model */ - private void createImplicitIndexingCluster(IndexedSearchCluster cluster, + private void createImplicitIndexingCluster(List<SearchNode> cluster, + IndexingDocproc indexingDocproc, Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { - String indexerName = cluster.getIndexingDocproc().getClusterName(content.getCluster().getName()); + String indexerName = indexingDocproc.getClusterName(content.getCluster().getName()); TreeConfigProducer<AnyConfigProducer> parent = getDocProc(root); ApplicationContainerCluster indexingCluster = new ApplicationContainerCluster(parent, "cluster." + indexerName, indexerName, modelContext.getDeployState()); ContainerModel indexingClusterModel = new ContainerModel(modelContext.withParent(parent).withId(indexingCluster.getSubId())); @@ -323,7 +318,7 @@ public class Content extends ConfigModel { List<ApplicationContainer> nodes = new ArrayList<>(); int index = 0; Set<HostResource> processedHosts = new LinkedHashSet<>(); - for (SearchNode searchNode : cluster.getSearchNodes()) { + for (SearchNode searchNode : cluster) { HostResource host = searchNode.getHostResource(); if (!processedHosts.contains(host)) { String containerName = String.valueOf(searchNode.getDistributionKey()); @@ -340,7 +335,7 @@ public class Content extends ConfigModel { indexingCluster.addContainers(nodes); addIndexingChain(indexingCluster); - cluster.getIndexingDocproc().setChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); + indexingDocproc.setChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); } private ContainerCluster<?> getContainerWithDocproc(Collection<ContainerModel> containers) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 1420cd82247..0027a9ca45c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -58,7 +58,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ private IndexedSearchCluster indexedCluster; - private Optional<IndexingDocproc> indexingDocproc; + private final IndexingDocproc indexingDocproc; private Redundancy redundancy; private final String clusterName; @@ -208,7 +208,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> double fractionOfMemoryReserved) { super(parent, "search"); - this.indexingDocproc = Optional.empty(); + this.indexingDocproc = new IndexingDocproc(); this.clusterName = clusterName; this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; @@ -262,10 +262,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); } indexedCluster = (IndexedSearchCluster)sc; - if (indexingDocproc.isPresent()) { - throw new IllegalArgumentException("Indexing docproc has previously been setup for streaming search"); - } - indexingDocproc = Optional.of(indexedCluster.getIndexingDocproc()); } clusters.put(sc.getClusterName(), sc); } @@ -483,12 +479,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> public Map<String, SearchCluster> getClusters() { return clusters; } public IndexedSearchCluster getIndexed() { return indexedCluster; } public boolean hasIndexedCluster() { return indexedCluster != null; } - public Optional<IndexingDocproc> getIndexingDocproc() { return indexingDocproc; } - public void setupStreamingSearchIndexingDocProc() { - if (indexingDocproc.isEmpty()) { - indexingDocproc = Optional.of(new IndexingDocproc()); - } - } + public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } public String getClusterName() { return clusterName; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 7c48ec11729..791faa401ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -209,20 +209,10 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem docprocChain = docprocChain.trim(); } if (docprocCluster != null && !docprocCluster.isEmpty()) { - if (!c.getSearch().hasIndexedCluster() && c.getSearch().getIndexingDocproc().isEmpty() && - docprocChain != null && !docprocChain.isEmpty()) { - c.getSearch().setupStreamingSearchIndexingDocProc(); - } - var indexingDocproc = c.getSearch().getIndexingDocproc(); - if (indexingDocproc.isPresent()) { - indexingDocproc.get().setClusterName(docprocCluster); - } + c.getSearch().getIndexingDocproc().setClusterName(docprocCluster); } if (docprocChain != null && !docprocChain.isEmpty()) { - var indexingDocproc = c.getSearch().getIndexingDocproc(); - if (indexingDocproc.isPresent()) { - indexingDocproc.get().setChainName(docprocChain); - } + c.getSearch().getIndexingDocproc().setChainName(docprocChain); } } @@ -301,10 +291,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem Objects.requireNonNull(admin.getLogserver(), "logserver cannot be null"); List<HostResource> host = List.of(admin.getLogserver().getHostResource()); admin.setClusterControllers(createClusterControllers(new ClusterControllerCluster(admin, "standalone", deployState), - host, - clusterName, - true, - deployState), + host, clusterName, true, deployState), deployState); } clusterControllers = admin.getClusterControllers(); @@ -457,7 +444,6 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem @Override public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) { - if (getSearch().getIndexingDocproc().isEmpty()) return; DocumentProtocol.getConfig(builder, getConfigId()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index d555d13c09c..1c8567b4079 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -110,11 +110,7 @@ public final class DocumentProtocol implements Protocol, for (ContentCluster cluster : Content.getContentClusters(repo)) { DocumentProtocolPoliciesConfig.Cluster.Builder clusterBuilder = new DocumentProtocolPoliciesConfig.Cluster.Builder(); addSelector(cluster.getConfigId(), cluster.getRoutingSelector(), clusterBuilder); - if (cluster.getSearch().getIndexingDocproc().isPresent()) - addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder); - else - clusterBuilder.defaultRoute(cluster.getConfigId()); - + addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder); builder.cluster(cluster.getConfigId(), clusterBuilder); } } @@ -226,18 +222,13 @@ public final class DocumentProtocol implements Protocol, private static void addContentRouting(List<ContentCluster> content, RoutingTableSpec table) { for (ContentCluster cluster : content) { RouteSpec spec = new RouteSpec(cluster.getConfigId()); - - if (cluster.getSearch().getIndexingDocproc().isPresent()) { - var indexingDocproc = cluster.getSearch().getIndexingDocproc().get(); - table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); - table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) - .addHop(indexingDocproc.getServiceName()) - .addHop("[Content:cluster=" + cluster.getName() + "]")); - table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) - .addHop("[Content:cluster=" + cluster.getName() + "]")); - } else { - table.addRoute(spec.addHop("[Content:cluster=" + cluster.getName() + "]")); - } + var indexingDocproc = cluster.getSearch().getIndexingDocproc(); + table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); + table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) + .addHop(indexingDocproc.getServiceName()) + .addHop("[Content:cluster=" + cluster.getName() + "]")); + table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) + .addHop("[Content:cluster=" + cluster.getName() + "]")); table.addRoute(new RouteSpec("storage/cluster." + cluster.getName()) .addHop("route:" + cluster.getConfigId())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index a79bc14db52..77c6f8f99c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -41,7 +41,6 @@ public class IndexedSearchCluster extends SearchCluster DispatchNodesConfig.Producer, ConfigInstance.Producer { - private final IndexingDocproc indexingDocproc; private Tuning tuning; private SearchCoverage searchCoverage; @@ -68,7 +67,6 @@ public class IndexedSearchCluster extends SearchCluster public IndexedSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, ModelContext.FeatureFlags featureFlags) { super(parent, clusterName, index); - indexingDocproc = new IndexingDocproc(); documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); dispatchWarmup = featureFlags.queryDispatchWarmup(); @@ -78,9 +76,6 @@ public class IndexedSearchCluster extends SearchCluster @Override protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } - public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } - - public void addSearcher(SearchNode searcher) { searchNodes.add(searcher); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java index 4f301f6df9a..2032720db96 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.search; import com.yahoo.vespa.model.container.docproc.DocprocChain; +import java.util.Objects; + /** * Utility class to track configuration for which indexing docproc to use by a search cluster. */ @@ -51,7 +53,7 @@ public class IndexingDocproc { } public String getServiceName() { - return chain.getServiceName(); + return Objects.requireNonNull(chain).getServiceName(); } /** |