Merge pull request #16812 from vespa-engine/balder/better-scale-numdocs-for-storeonly

Balder/better scale numdocs for storeonly

4 files changed, 14 insertions, 45 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 0de111e459e..6f7709efc24 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -325,10 +325,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
             indexedCluster.setSearchableCopies(redundancy.readyCopies());
         }
         this.redundancy = redundancy;
-        for (SearchNode node : getSearchNodes()) {
-            node.setRedundancy(redundancy.finalRedundancy());
-            node.setSearchableCopies(redundancy.readyCopies());
-        }
     }
 
     private Optional<StreamingSearchCluster> findStreamingCluster(String docType) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
index 7367850bb42..e8d5a24cf73 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
@@ -16,9 +16,9 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
 
     final static long MB = 1024 * 1024;
     public final static long GB = MB * 1024;
+    // This is an approximate number base on observation of a node using 33G memory with 765M docs
+    private final static long MEMORY_COST_PER_DOCUMENT_STORE_ONLY = 46L;
     private final NodeResources resources;
-    private final int redundancy;
-    private final int searchableCopies;
     private final int threadsPerSearch;
     private final boolean combined;
 
@@ -26,13 +26,9 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
     public static final double reservedMemoryGb = 1;
 
     public NodeResourcesTuning(NodeResources resources,
-                               int redundancy,
-                               int searchableCopies,
                                int threadsPerSearch,
                                boolean combined) {
         this.resources = resources;
-        this.redundancy = redundancy;
-        this.searchableCopies = searchableCopies;
         this.threadsPerSearch = threadsPerSearch;
         this.combined = combined;
     }
@@ -56,8 +52,8 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
     private void getConfig(ProtonConfig.Documentdb.Builder builder) {
         ProtonConfig.Documentdb dbCfg = builder.build();
         if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) {
-            long numDocs = (long)usableMemoryGb() * GB / 64L;
-            builder.allocation.initialnumdocs(numDocs/Math.max(searchableCopies, redundancy));
+            long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_STORE_ONLY;
+            builder.allocation.initialnumdocs(numDocs);
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index 16302ddff49..9f129f65281 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -62,8 +62,6 @@ public class SearchNode extends AbstractService implements
     private final boolean flushOnShutdown;
     private NodeSpec nodeSpec;
     private int distributionKey;
-    private int redundancy = 1;
-    private int searchableCopies = 1;
     private final String clusterName;
     private TransactionLogServer tls;
     private AbstractService serviceLayerService;
@@ -160,16 +158,6 @@ public class SearchNode extends AbstractService implements
     private String getBaseDir() {
         return getDefaults().underVespaHome("var/db/vespa/search/cluster." + getClusterName()) + "/n" + distributionKey;
     }
-    public void setSearchableCopies(int searchableCopies) {
-        this.searchableCopies = searchableCopies;
-    }
-    public void setRedundancy(int redundancy) {
-        this.redundancy = redundancy;
-    }
-
-    void updatePartition(int partitionId) {
-        nodeSpec = new NodeSpec(nodeSpec.groupIndex(), partitionId);
-    }
 
     @Override
     public NodeSpec getNodeSpec() {
@@ -286,8 +274,6 @@ public class SearchNode extends AbstractService implements
         }
         if (getHostResource() != null && ! getHostResource().realResources().isUnspecified()) {
             var nodeResourcesTuning = new NodeResourcesTuning(getHostResource().realResources(),
-                                                              redundancy,
-                                                              searchableCopies,
                                                               tuning.map(Tuning::threadsPerSearch).orElse(1),
                                                               combined);
             nodeResourcesTuning.getConfig(builder);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
index 1f9c9b1e07a..9ea25326bd6 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
@@ -48,18 +48,17 @@ public class NodeResourcesTuningTest {
                                    .configid("some/config/id/" + sdMode.getFirst())
                                    .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond())));
         }
-        return configFromMemorySetting(gb, builder, redundancy, searchableCopies);
+        return configFromMemorySetting(gb, builder);
     }
 
     private void verify_that_initial_numdocs_is_dependent_of_mode(int redundancy, int searchablecopies) {
-        int divisor = Math.max(redundancy, searchablecopies);
         ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + reservedMemoryGb, redundancy, searchablecopies);
         assertEquals(3, cfg.documentdb().size());
         assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs());
         assertEquals("a", cfg.documentdb(0).inputdoctypename());
-        assertEquals(24 * GB / 64 / divisor, cfg.documentdb(1).allocation().initialnumdocs());
+        assertEquals(24 * GB / 46, cfg.documentdb(1).allocation().initialnumdocs());
         assertEquals("b", cfg.documentdb(1).inputdoctypename());
-        assertEquals(24 * GB / 64 / divisor, cfg.documentdb(2).allocation().initialnumdocs());
+        assertEquals(24 * GB / 46, cfg.documentdb(2).allocation().initialnumdocs());
         assertEquals("c", cfg.documentdb(2).inputdoctypename());
     }
 
@@ -210,9 +209,9 @@ public class NodeResourcesTuningTest {
         return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), combined);
     }
 
-    private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder, int redundancy, int searchableCopies) {
+    private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder) {
         return getConfig(new FlavorsConfig.Flavor.Builder()
-                                 .minMainMemoryAvailableGb(memoryGb), builder, redundancy, searchableCopies, false);
+                                 .minMainMemoryAvailableGb(memoryGb), builder, false);
     }
 
     private static ProtonConfig configFromNumCoresSetting(double numCores) {
@@ -221,7 +220,7 @@ public class NodeResourcesTuningTest {
 
     private static ProtonConfig configFromNumCoresSetting(double numCores, int numThreadsPerSearch) {
         return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores),
-                         new ProtonConfig.Builder(), 1, 1, numThreadsPerSearch, false);
+                         new ProtonConfig.Builder(), numThreadsPerSearch, false);
     }
 
     private static ProtonConfig configFromEnvironmentType(boolean docker) {
@@ -233,25 +232,17 @@ public class NodeResourcesTuningTest {
         return getConfig(flavorBuilder, new ProtonConfig.Builder(), combined);
     }
 
-    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder,
-                                          ProtonConfig.Builder protonBuilder, boolean combined) {
-        return getConfig(flavorBuilder, protonBuilder, 1, 1, combined);
-    }
-
-    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
-                                          int redundancy, int searchableCopies, boolean combined) {
+    private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, boolean combined) {
         flavorBuilder.name("my_flavor");
-        NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(),
-                                                             redundancy, searchableCopies, 1, combined);
+        NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), 1, combined);
         tuning.getConfig(protonBuilder);
         return new ProtonConfig(protonBuilder);
     }
 
     private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
-                                          int redundancy, int searchableCopies, int numThreadsPerSearch, boolean combined) {
+                                          int numThreadsPerSearch, boolean combined) {
         flavorBuilder.name("my_flavor");
-        NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(),
-                                                             redundancy, searchableCopies, numThreadsPerSearch, combined);
+        NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, combined);
         tuning.getConfig(protonBuilder);
         return new ProtonConfig(protonBuilder);
     }