diff options
26 files changed, 402 insertions, 321 deletions
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java index 8f588ffa5b0..dcca60e1be8 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -262,7 +262,7 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo { .sorted().toList(); if (! violations.isEmpty()) { - warnOrThrow("Artifacts provided from Jdisc runtime are included in compile scope: " + violations + warnOrThrow("Artifacts provided from Vespa runtime are included in compile scope: " + violations + ". Direct dependencies should be removed." + " For transitive dependencies, run 'mvn dependency:tree' and add necessary exclusions."); } diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java index 30f9cd202ff..78c0ea0ddef 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.utils.FreezableMap; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -74,7 +75,8 @@ public abstract class TreeConfigProducer<CHILD extends AnyConfigProducer> errorMsgClassName() + " '" + getSubId() + "'. (This is commonly caused by service/node index " + "collisions in the config.)." + "\nExisting instance: " + childrenBySubId.get(child.getSubId()) + - "\nAttempted to add: " + child); + "\nAttempted to add: " + child + + "\nStack trace: " + Arrays.toString(Thread.currentThread().getStackTrace())); } childrenBySubId.put(child.getSubId(), child); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index e2166b263ee..538a1b49d24 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -12,10 +12,11 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.utils.Duration; import org.w3c.dom.Element; + import java.util.Optional; /** - * Config generation for common parameters for all fleet controllers. + * Config generation for parameters for fleet controllers. */ public class ClusterControllerConfig extends AnyConfigProducer implements FleetcontrollerConfig.Producer { @@ -48,25 +49,13 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc clusterControllerTuning = tuning.child("cluster-controller"); } + var numberOfLeafGroups = ((ContentCluster) ancestor).getRootGroup().getNumberOfLeafGroups(); var tuningConfig = new ClusterControllerTuningBuilder(clusterControllerTuning, - minNodeRatioPerGroup, - bucketSplittingMinimumBits) + minNodeRatioPerGroup, + bucketSplittingMinimumBits, + allowMoreThanOneContentGroupDown, + numberOfLeafGroups) .build(); - if (ancestor instanceof ContentCluster) { - int numberOfLeafGroups = ((ContentCluster) ancestor).getRootGroup().getNumberOfLeafGroups(); - if (tuningConfig.maxGroupsAllowedDown().isPresent()) { - Integer maxGroupsAllowedDown = tuningConfig.maxGroupsAllowedDown().get(); - if (deployState.zone().environment().isProduction() && (maxGroupsAllowedDown > numberOfLeafGroups)) - throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + maxGroupsAllowedDown + - ") larger than number of groups (" + numberOfLeafGroups + ")"); - } else { - // Reduce to numberOfLeafGroups for tests or in environments where number of groups are reduced by policy (dev, test, staging, perf) - tuningConfig = tuningConfig.withMaxGroupsAllowedDown(numberOfLeafGroups); - } - } else { - // Reduce to 1 for tests (ancestor is a mock class) - tuningConfig = tuningConfig.withMaxGroupsAllowedDown(1); - } return new ClusterControllerConfig(ancestor, clusterName, @@ -74,6 +63,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc resourceLimits, allowMoreThanOneContentGroupDown); } + } private final String clusterName; @@ -122,79 +112,83 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc public ClusterControllerTuning tuning() {return tuning;} -private static class ClusterControllerTuningBuilder { - - private final Optional<Double> minNodeRatioPerGroup; - private final Optional<Duration> initProgressTime; - private final Optional<Duration> transitionTime; - private final Optional<Long> maxPrematureCrashes; - private final Optional<Duration> stableStateTimePeriod; - private final Optional<Double> minDistributorUpRatio; - private final Optional<Double> minStorageUpRatio; - private final Optional<Integer> minSplitBits; - final Optional<Integer> maxGroupsAllowedDown; - - ClusterControllerTuningBuilder(ModelElement tuning, - Optional<Double> minNodeRatioPerGroup, - Optional<Integer> bucketSplittingMinimumBits) { - this.minSplitBits = bucketSplittingMinimumBits; - this.minNodeRatioPerGroup = minNodeRatioPerGroup; - if (tuning == null) { - this.initProgressTime = Optional.empty(); - this.transitionTime = Optional.empty(); - this.maxPrematureCrashes = Optional.empty(); - this.stableStateTimePeriod = Optional.empty(); - this.minDistributorUpRatio = Optional.empty(); - this.minStorageUpRatio = Optional.empty(); - this.maxGroupsAllowedDown = Optional.empty(); - } else { - this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time")); - this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time")); - this.maxPrematureCrashes = Optional.ofNullable(tuning.childAsLong("max-premature-crashes")); - this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period")); - this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio")); - this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio")); - this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down")); + private static class ClusterControllerTuningBuilder { + + private final Optional<Double> minNodeRatioPerGroup; + private final Optional<Duration> initProgressTime; + private final Optional<Duration> transitionTime; + private final Optional<Long> maxPrematureCrashes; + private final Optional<Duration> stableStateTimePeriod; + private final Optional<Double> minDistributorUpRatio; + private final Optional<Double> minStorageUpRatio; + private final Optional<Integer> minSplitBits; + private final Optional<Integer> maxGroupsAllowedDown; + + ClusterControllerTuningBuilder(ModelElement tuning, + Optional<Double> minNodeRatioPerGroup, + Optional<Integer> bucketSplittingMinimumBits, + boolean maxGroupsAllowedDown, + int numberOfLeafGroups) { + this.minSplitBits = bucketSplittingMinimumBits; + this.minNodeRatioPerGroup = minNodeRatioPerGroup; + if (tuning == null) { + this.initProgressTime = Optional.empty(); + this.transitionTime = Optional.empty(); + this.maxPrematureCrashes = Optional.empty(); + this.stableStateTimePeriod = Optional.empty(); + this.minDistributorUpRatio = Optional.empty(); + this.minStorageUpRatio = Optional.empty(); + this.maxGroupsAllowedDown = Optional.empty(); + } + else { + this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time")); + this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time")); + this.maxPrematureCrashes = Optional.ofNullable(tuning.childAsLong("max-premature-crashes")); + this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period")); + this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio")); + this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio")); + this.maxGroupsAllowedDown = maxGroupsAllowedDown(tuning, maxGroupsAllowedDown, numberOfLeafGroups); + } } - } - private ClusterControllerTuning build() { - return new ClusterControllerTuning(initProgressTime, - transitionTime, - maxPrematureCrashes, - stableStateTimePeriod, - minDistributorUpRatio, - minStorageUpRatio, - maxGroupsAllowedDown, - minNodeRatioPerGroup, - minSplitBits); - } -} + private static Optional<Integer> maxGroupsAllowedDown(ModelElement tuning, boolean allowMoreThanOneContentGroupDown, int numberOfLeafGroups) { + var groupsAllowedDownRatio = tuning.childAsDouble("groups-allowed-down-ratio"); + + if (groupsAllowedDownRatio != null) { + if (groupsAllowedDownRatio < 0 || groupsAllowedDownRatio > 1) + throw new IllegalArgumentException("groups-allowed-down-ratio must be between 0 and 1, got " + groupsAllowedDownRatio); + + var maxGroupsAllowedDown = Math.max(1, (int) Math.floor(groupsAllowedDownRatio * numberOfLeafGroups)); + return allowMoreThanOneContentGroupDown ? Optional.of(maxGroupsAllowedDown) : Optional.empty(); + } + + return Optional.empty(); + } + + private ClusterControllerTuning build() { + return new ClusterControllerTuning(initProgressTime, + transitionTime, + maxPrematureCrashes, + stableStateTimePeriod, + minDistributorUpRatio, + minStorageUpRatio, + maxGroupsAllowedDown, + minNodeRatioPerGroup, + minSplitBits); + } -private record ClusterControllerTuning(Optional<Duration> initProgressTime, - Optional<Duration> transitionTime, - Optional<Long> maxPrematureCrashes, - Optional<Duration> stableStateTimePeriod, - Optional<Double> minDistributorUpRatio, - Optional<Double> minStorageUpRatio, - Optional<Integer> maxGroupsAllowedDown, - Optional<Double> minNodeRatioPerGroup, - Optional<Integer> minSplitBits) { - - public ClusterControllerTuning withMaxGroupsAllowedDown(int maxGroupsAllowedDown) { - return new ClusterControllerConfig.ClusterControllerTuning( - initProgressTime, - transitionTime, - maxPrematureCrashes, - stableStateTimePeriod, - minDistributorUpRatio, - minStorageUpRatio, - Optional.of(maxGroupsAllowedDown), - minNodeRatioPerGroup, - minSplitBits); } -} + private record ClusterControllerTuning(Optional<Duration> initProgressTime, + Optional<Duration> transitionTime, + Optional<Long> maxPrematureCrashes, + Optional<Duration> stableStateTimePeriod, + Optional<Double> minDistributorUpRatio, + Optional<Double> minStorageUpRatio, + Optional<Integer> maxGroupsAllowedDown, + Optional<Double> minNodeRatioPerGroup, + Optional<Integer> minSplitBits) { + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index f2d55b4d49f..985cef3a5ad 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -211,7 +211,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem docprocChain = docprocChain.trim(); } if (docprocCluster != null && !docprocCluster.isEmpty()) { - if (!c.getSearch().hasIndexedCluster() && !c.getSearch().getIndexingDocproc().isPresent() && + if (!c.getSearch().hasIndexedCluster() && c.getSearch().getIndexingDocproc().isEmpty() && docprocChain != null && !docprocChain.isEmpty()) { c.getSearch().setupStreamingSearchIndexingDocProc(); } @@ -459,7 +459,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem @Override public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) { - if ( ! getSearch().getIndexingDocproc().isPresent()) return; + if (getSearch().getIndexingDocproc().isEmpty()) return; DocumentProtocol.getConfig(builder, getConfigId()); } diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 4d8b44f3fc1..bb63dcd73ff 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -82,7 +82,7 @@ ClusterControllerTuning = element cluster-controller { element stable-state-period { xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" } }? & element min-distributor-up-ratio { xsd:double }? & element min-storage-up-ratio { xsd:double }? & - element max-groups-allowed-down { xsd:nonNegativeInteger }? + element groups-allowed-down-ratio { xsd:double }? } DispatchTuning = element dispatch { diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7273053db5c..b1f47c54d54 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -2373,7 +2373,7 @@ public class ModelProvisioningTest { " <nodes count='4' groups='4'/>" + " <tuning>" + " <cluster-controller>" + - " <max-groups-allowed-down>2</max-groups-allowed-down>" + + " <groups-allowed-down-ratio>0.5</groups-allowed-down-ratio>" + " </cluster-controller>" + " </tuning>" + " </content>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 73bbd6ee464..ef2767249a5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1422,51 +1422,36 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - void testAllow2GroupsDown() { - String services = "<?xml version='1.0' encoding='UTF-8' ?>" + - "<services version='1.0'>" + - " <container id='default' version='1.0' />" + - " <content id='storage' version='1.0'>" + - " <redundancy>4</redundancy>" + - " <documents>" + - " <document mode='index' type='type1' />" + - " </documents>" + - " <group name='root'>" + - " <distribution partitions='1|1|1|*'/>" + - " <group name='g-1' distribution-key='0'>" + - " <node hostalias='mockhost' distribution-key='0'/>" + - " </group>" + - " <group name='g-2' distribution-key='1'>" + - " <node hostalias='mockhost' distribution-key='1'/>" + - " </group>" + - " <group name='g-3' distribution-key='2'>" + - " <node hostalias='mockhost' distribution-key='2'/>" + - " </group>" + - " <group name='g-4' distribution-key='3'>" + - " <node hostalias='mockhost' distribution-key='3'/>" + - " </group>" + - " </group>" + - " <tuning>" + - " <cluster-controller>" + - " <max-groups-allowed-down>2</max-groups-allowed-down>" + - " </cluster-controller>" + - " </tuning>" + - " <engine>" + - " <proton>" + - " <searchable-copies>4</searchable-copies>" + - " </proton>" + - " </engine>" + - " </content>" + - " </services>"; - VespaModel model = createEnd2EndOneNode(new TestProperties().setAllowMoreThanOneContentGroupDown(true), services); - - var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder(); - model.getConfig(fleetControllerConfigBuilder, "admin/cluster-controllers/0/components/clustercontroller-storage-configurer"); - assertEquals(2, fleetControllerConfigBuilder.build().max_number_of_groups_allowed_to_be_down()); - } - - private void assertIndexingDocprocEnabled(boolean indexed, boolean force, boolean expEnabled) - { + void testGroupsAllowedToBeDown() { + assertGroupsAllowedsDown(1, 0.5, 1); + assertGroupsAllowedsDown(2, 0.5, 1); + assertGroupsAllowedsDown(3, 0.5, 1); + assertGroupsAllowedsDown(4, 0.5, 2); + assertGroupsAllowedsDown(5, 0.5, 2); + assertGroupsAllowedsDown(6, 0.5, 3); + + assertGroupsAllowedsDown(1, 0.33, 1); + assertGroupsAllowedsDown(2, 0.33, 1); + assertGroupsAllowedsDown(3, 0.33, 1); + assertGroupsAllowedsDown(4, 0.33, 1); + assertGroupsAllowedsDown(5, 0.33, 1); + assertGroupsAllowedsDown(6, 0.33, 1); + + assertGroupsAllowedsDown(1, 0.67, 1); + assertGroupsAllowedsDown(2, 0.67, 1); + assertGroupsAllowedsDown(3, 0.67, 2); + assertGroupsAllowedsDown(4, 0.67, 2); + assertGroupsAllowedsDown(5, 0.67, 3); + assertGroupsAllowedsDown(6, 0.67, 4); + + assertGroupsAllowedsDown(1, 0, 1); + assertGroupsAllowedsDown(2, 0, 1); + + assertGroupsAllowedsDown(1, 1, 1); + assertGroupsAllowedsDown(2, 1, 2); + } + + private void assertIndexingDocprocEnabled(boolean indexed, boolean force, boolean expEnabled) { String services = "<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + " <container id='default' version='1.0'>" + @@ -1503,4 +1488,56 @@ public class ContentClusterTest extends ContentBaseTest { assertIndexingDocprocEnabled(false, true, true); } + private void assertGroupsAllowedsDown(int groupCount, double groupsAllowedDown, int expectedGroupsAllowedDown) { + var services = servicesWithGroups(groupCount, groupsAllowedDown); + var model = createEnd2EndOneNode(new TestProperties().setAllowMoreThanOneContentGroupDown(true), services); + + var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder(); + model.getConfig(fleetControllerConfigBuilder, "admin/cluster-controllers/0/components/clustercontroller-storage-configurer"); + var config = fleetControllerConfigBuilder.build(); + + assertEquals(expectedGroupsAllowedDown, config.max_number_of_groups_allowed_to_be_down()); + } + + private String servicesWithGroups(int groupCount, double minGroupUpRatio) { + String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" + + "<services version='1.0'>" + + " <container id='default' version='1.0' />" + + " <content id='storage' version='1.0'>" + + " <redundancy>%d</redundancy>" + + " <documents>" + + " <document mode='index' type='type1' />" + + " </documents>" + + " <group name='root'>", groupCount); + String distribution = switch (groupCount) { + case 1, 2 -> " <distribution partitions='1|*'/>"; + case 3 -> " <distribution partitions='1|1|*'/>"; + case 4 -> " <distribution partitions='1|1|1|*'/>"; + case 5 -> " <distribution partitions='1|1|1|1|*'/>"; + case 6 -> " <distribution partitions='1|1|1|1|1|*'/>"; + default -> throw new IllegalArgumentException("Does not support groupCount > 6"); + }; + services += distribution; + for (int i = 0; i < groupCount; i++) { + services += String.format(" <group name='g-%d' distribution-key='%d'>" + + " <node hostalias='mockhost' distribution-key='%d'/>" + + " </group>", + i, i, i); + } + return services + + String.format(" </group>" + + " <tuning>" + + " <cluster-controller>" + + " <groups-allowed-down-ratio>%f</groups-allowed-down-ratio>" + + " </cluster-controller>" + + " </tuning>" + + " <engine>" + + " <proton>" + + " <searchable-copies>%d</searchable-copies>" + + " </proton>" + + " </engine>" + + " </content>" + + " </services>", minGroupUpRatio, groupCount); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index 138852e1c5c..2f7ed875226 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -3,44 +3,40 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.text.XML; import com.yahoo.vespa.config.content.FleetcontrollerConfig; -import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Test; -import org.w3c.dom.Document; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; public class FleetControllerClusterTest { - private ClusterControllerConfig parse(String xml, TestProperties props) { - Document doc = XML.getDocument(xml); - var deployState = new DeployState.Builder().properties(props).build(); - MockRoot root = new MockRoot("", deployState); - var clusterElement = new ModelElement(doc.getDocumentElement()); - return new ClusterControllerConfig.Builder("storage", - clusterElement, - new ClusterResourceLimits.Builder(false, - props.resourceLimitDisk(), - props.resourceLimitMemory()) - .build(clusterElement).getClusterControllerLimits(), - props.allowMoreThanOneContentGroupDown(new ClusterSpec.Id("default"))) - .build(root.getDeployState(), root, clusterElement.getXml()); + private FleetcontrollerConfig parse(String xml, TestProperties props) { + var deployStateBuilder = new DeployState.Builder().properties(props); + props.allowMoreThanOneContentGroupDown(new ClusterSpec.Id("default")); + var mockPkg = new VespaModelCreatorWithMockPkg(null, xml, ApplicationPackageUtils.generateSchemas("type1")); + var model = mockPkg.create(deployStateBuilder); + var builder = new FleetcontrollerConfig.Builder(); + model.getConfig(builder, "admin/cluster-controllers/0/components/clustercontroller-storage-configurer"); + return builder.build(); } - private ClusterControllerConfig parse(String xml) { + private FleetcontrollerConfig parse(String xml) { return parse(xml, new TestProperties()); } @Test void testParameters() { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); - parse(""" - <cluster id="storage"> - <documents/> <tuning> + var config = parse(""" + <content id="storage" version="1.0"> + <documents> + <document type="type1" mode="index"/> + </documents> + <redundancy>2</redundancy> + <tuning> <bucket-splitting minimum-bits="7" /> <cluster-controller> <init-progress-time>13</init-progress-time> @@ -51,11 +47,9 @@ public class FleetControllerClusterTest { <min-storage-up-ratio>0.3</min-storage-up-ratio> </cluster-controller> </tuning> - </cluster>""", - new TestProperties().setAllowMoreThanOneContentGroupDown(true)). - getConfig(builder); + </content>""", + new TestProperties().setAllowMoreThanOneContentGroupDown(true)); - FleetcontrollerConfig config = new FleetcontrollerConfig(builder); assertEquals(13 * 1000, config.init_progress_time()); assertEquals(27 * 1000, config.storage_transition_time()); assertEquals(4, config.max_premature_crashes()); @@ -67,33 +61,34 @@ public class FleetControllerClusterTest { @Test void testDurationParameters() { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); - parse("<cluster id=\"storage\">\n" + - " <documents/>" + - " <tuning>\n" + - " <cluster-controller>\n" + - " <init-progress-time>13ms</init-progress-time>\n" + - " </cluster-controller>\n" + - " </tuning>\n" + - "</cluster>"). - getConfig(builder); - - FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + var config = parse( + "<content id='storage' version='1.0'>\n" + + "<documents>\n" + + " <document type='type1' mode='index'/>" + + "</documents>\n" + + "<redundancy>2</redundancy>\n" + + " <tuning>\n" + + " <cluster-controller>\n" + + " <init-progress-time>13ms</init-progress-time>\n" + + " </cluster-controller>\n" + + " </tuning>\n" + + "</content>"); + assertEquals(13, config.init_progress_time()); } @Test void min_node_ratio_per_group_tuning_config_is_propagated() { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); - parse("<cluster id=\"storage\">\n" + - " <documents/>\n" + - " <tuning>\n" + - " <min-node-ratio-per-group>0.75</min-node-ratio-per-group>\n" + - " </tuning>\n" + - "</cluster>"). - getConfig(builder); - - FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + var config = parse("<content id='storage' version='1.0'>" + + "<documents>" + + "<document type='type1' mode='index'/>" + + "</documents>" + + "<redundancy>2</redundancy>" + + " <tuning>\n" + + " <min-node-ratio-per-group>0.75</min-node-ratio-per-group>\n" + + " </tuning>\n" + + "</content>"); + assertEquals(0.75, config.min_node_ratio_per_group(), 0.01); } @@ -126,18 +121,18 @@ public class FleetControllerClusterTest { } private FleetcontrollerConfig getConfigForResourceLimitsTuning(Double diskLimit, Double memoryLimit) { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); - parse(joinLines("<cluster id=\"test\">", - "<documents/>", - "<tuning>", - " <resource-limits>", - (diskLimit != null ? (" <disk>" + diskLimit + "</disk>") : ""), - (memoryLimit != null ? (" <memory>" + memoryLimit + "</memory>") : ""), - " </resource-limits>", - "</tuning>" + - "</cluster>")). - getConfig(builder); - return new FleetcontrollerConfig(builder); + return parse(joinLines("<content id='storage' version='1.0'>" + + "<documents>" + + "<document type='type1' mode='index'/>" + + "</documents>" + + "<redundancy>2</redundancy>" + + "<tuning>", + " <resource-limits>", + (diskLimit != null ? (" <disk>" + diskLimit + "</disk>") : ""), + (memoryLimit != null ? (" <memory>" + memoryLimit + "</memory>") : ""), + " </resource-limits>", + "</tuning>" + + "</content>")); } @Test @@ -153,12 +148,12 @@ public class FleetControllerClusterTest { } private FleetcontrollerConfig getConfigForBasicCluster(TestProperties props) { - var builder = new FleetcontrollerConfig.Builder(); - parse("<cluster id=\"storage\">\n" + - " <documents/>\n" + - "</cluster>", props). - getConfig(builder); - return new FleetcontrollerConfig(builder); + return parse("<content id='storage' version='1.0'>" + + "<documents>" + + "<document type='type1' mode='index'/>" + + "</documents>" + + "<redundancy>2</redundancy>" + + "</content>", props); } private FleetcontrollerConfig getConfigForBasicCluster() { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 94ed28778d7..b1edc189295 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -145,7 +145,7 @@ public class ZmsClientMock implements ZmsClient { .map(d -> d.attributes) .map(attrs -> { if (attrs.containsKey("account")) { - return new AthenzDomainMeta((String)attrs.get("account"), domain.getName()); + return new AthenzDomainMeta((String) attrs.get("account"), (String) attrs.get("gcpProject"), domain.getName()); } return null; }) diff --git a/searchcore/src/tests/grouping/grouping.cpp b/searchcore/src/tests/grouping/grouping.cpp index a9d3ffe26db..015aec73999 100644 --- a/searchcore/src/tests/grouping/grouping.cpp +++ b/searchcore/src/tests/grouping/grouping.cpp @@ -10,6 +10,7 @@ #include <vespa/searchcore/grouping/groupingmanager.h> #include <vespa/searchcore/grouping/groupingsession.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> +#include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/searchlib/test/mock_attribute_context.h> #include <vespa/vespalib/util/testclock.h> #include <iostream> @@ -35,8 +36,13 @@ const uint32_t NUM_DOCS = 1000; struct MyWorld { MockAttributeContext attributeContext; + search::AllocatedBitVector bv; - void basicSetup() { + MyWorld() + : attributeContext(), + bv(NUM_DOCS+1) + { + bv.setInterval(0, NUM_DOCS); // attribute context { SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr0"); @@ -170,7 +176,8 @@ TEST_F("testGroupingContextInitialization", DoomFixture()) { nos << (uint32_t)1; baseRequest.serialize(nos); - GroupingContext context(f1.clock.clock(), f1.timeOfDoom, os.data(), os.size(), true); + AllocatedBitVector bv(1); + GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom, os.data(), os.size(), true); ASSERT_TRUE(!context.empty()); GroupingContext::GroupingList list = context.getGroupingList(); ASSERT_TRUE(list.size() == 1); @@ -200,7 +207,8 @@ TEST_F("testGroupingContextUsage", DoomFixture()) { auto r1 = std::make_shared<Grouping>(request1); auto r2 = std::make_shared<Grouping>(request2); - GroupingContext context(f1.clock.clock(), f1.timeOfDoom); + AllocatedBitVector bv(1); + GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom); ASSERT_TRUE(context.empty()); context.addGrouping(r1); ASSERT_TRUE(context.getGroupingList().size() == 1); @@ -222,7 +230,8 @@ TEST_F("testGroupingContextSerializing", DoomFixture()) { nos << (uint32_t)1; baseRequest.serialize(nos); - GroupingContext context(f1.clock.clock(), f1.timeOfDoom); + AllocatedBitVector bv(1); + GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom); auto bp = std::make_shared<Grouping>(baseRequest); context.addGrouping(bp); context.serialize(); @@ -240,7 +249,8 @@ TEST_F("testGroupingManager", DoomFixture()) { .addLevel(createGL(MU<AttributeNode>("attr1"), MU<AttributeNode>("attr2"))) .addLevel(createGL(MU<AttributeNode>("attr2"), MU<AttributeNode>("attr3"))); - GroupingContext context(f1.clock.clock(), f1.timeOfDoom); + AllocatedBitVector bv(1); + GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom); auto bp = std::make_shared<Grouping>(request1); context.addGrouping(bp); GroupingManager manager(context); @@ -249,7 +259,6 @@ TEST_F("testGroupingManager", DoomFixture()) { TEST_F("testGroupingSession", DoomFixture()) { MyWorld world; - world.basicSetup(); vespalib::nbostream os; Grouping request1; request1.setId(0) @@ -275,7 +284,7 @@ TEST_F("testGroupingSession", DoomFixture()) { auto r1 = std::make_shared<Grouping>(request1); auto r2 = std::make_shared<Grouping>(request2); - GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom); + GroupingContext initContext(world.bv, f1.clock.clock(), f1.timeOfDoom); initContext.addGrouping(r1); initContext.addGrouping(r2); SessionId id("foo"); @@ -307,7 +316,7 @@ TEST_F("testGroupingSession", DoomFixture()) { } // Test second pass { - GroupingContext context(f1.clock.clock(), f1.timeOfDoom); + GroupingContext context(world.bv, f1.clock.clock(), f1.timeOfDoom); auto r = std::make_shared<Grouping>(request1); r->setFirstLevel(1); r->setLastLevel(1); @@ -318,7 +327,7 @@ TEST_F("testGroupingSession", DoomFixture()) { } // Test last pass. Session should be marked as finished { - GroupingContext context(f1.clock.clock(), f1.timeOfDoom); + GroupingContext context(world.bv, f1.clock.clock(), f1.timeOfDoom); auto r = std::make_shared<Grouping>(request1); r->setFirstLevel(2); r->setLastLevel(2); @@ -332,7 +341,6 @@ TEST_F("testGroupingSession", DoomFixture()) { TEST_F("testEmptySessionId", DoomFixture()) { MyWorld world; - world.basicSetup(); vespalib::nbostream os; Grouping request1; request1.setId(0) @@ -342,7 +350,7 @@ TEST_F("testEmptySessionId", DoomFixture()) { .addLevel(createGL(MU<AttributeNode>("attr2"), MU<AttributeNode>("attr3"))); auto r1 = std::make_shared<Grouping>(request1); - GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom); + GroupingContext initContext(world.bv, f1.clock.clock(), f1.timeOfDoom); initContext.addGrouping(r1); SessionId id; @@ -362,7 +370,6 @@ TEST_F("testEmptySessionId", DoomFixture()) { TEST_F("testSessionManager", DoomFixture()) { MyWorld world; - world.basicSetup(); vespalib::nbostream os; Grouping request1; request1.setId(0) @@ -375,7 +382,7 @@ TEST_F("testSessionManager", DoomFixture()) { .setResult(Int64ResultNode(0)))); auto r1 = std::make_shared<Grouping>(request1); - GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom); + GroupingContext initContext(world.bv, f1.clock.clock(), f1.timeOfDoom); initContext.addGrouping(r1); SessionManager mgr(2); @@ -424,7 +431,6 @@ void doGrouping(GroupingContext &ctx, TEST_F("test grouping fork/join", DoomFixture()) { MyWorld world; - world.basicSetup(); Grouping request; request.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("attr0")))) @@ -433,7 +439,7 @@ TEST_F("test grouping fork/join", DoomFixture()) { .setLastLevel(1); auto g1 = std::make_shared<Grouping>(request); - GroupingContext context(f1.clock.clock(), f1.timeOfDoom); + GroupingContext context(world.bv, f1.clock.clock(), f1.timeOfDoom); context.addGrouping(g1); GroupingSession session(SessionId(), context, world.attributeContext); session.prepareThreadContextCreation(4); @@ -470,13 +476,12 @@ TEST_F("test grouping fork/join", DoomFixture()) { TEST_F("test session timeout", DoomFixture()) { MyWorld world; - world.basicSetup(); SessionManager mgr(2); SessionId id1("foo"); SessionId id2("bar"); - GroupingContext initContext1(f1.clock.clock(), steady_time(duration(10))); - GroupingContext initContext2(f1.clock.clock(), steady_time(duration(20))); + GroupingContext initContext1(world.bv, f1.clock.clock(), steady_time(duration(10))); + GroupingContext initContext2(world.bv, f1.clock.clock(), steady_time(duration(20))); auto s1 = std::make_unique<GroupingSession>(id1, initContext1, world.attributeContext); auto s2 = std::make_unique<GroupingSession>(id2, initContext2, world.attributeContext); mgr.insert(std::move(s1)); diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp index f76fe873b03..b85dda6a9f2 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp @@ -6,6 +6,9 @@ #include <vespa/searchlib/aggregation/hitsaggregationresult.h> #include <vespa/searchlib/common/bitvector.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchcore/grouping.groupingcontext"); + namespace search::grouping { using aggregation::CountFS4Hits; @@ -49,9 +52,10 @@ GroupingContext::setDistributionKey(uint32_t distributionKey) } } -GroupingContext::GroupingContext(const vespalib::Clock & clock, vespalib::steady_time timeOfDoom, +GroupingContext::GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom, const char *groupSpec, uint32_t groupSpecLen, bool enableNested) - : _clock(clock), + : _validLids(validLids), + _clock(clock), _timeOfDoom(timeOfDoom), _os(), _groupingList(), @@ -60,8 +64,9 @@ GroupingContext::GroupingContext(const vespalib::Clock & clock, vespalib::steady deserialize(groupSpec, groupSpecLen); } -GroupingContext::GroupingContext(const vespalib::Clock & clock, vespalib::steady_time timeOfDoom) - : _clock(clock), +GroupingContext::GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom) + : _validLids(validLids), + _clock(clock), _timeOfDoom(timeOfDoom), _os(), _groupingList(), @@ -69,7 +74,8 @@ GroupingContext::GroupingContext(const vespalib::Clock & clock, vespalib::steady { } GroupingContext::GroupingContext(const GroupingContext & rhs) - : _clock(rhs._clock), + : _validLids(rhs._validLids), + _clock(rhs._clock), _timeOfDoom(rhs._timeOfDoom), _os(), _groupingList(), @@ -102,27 +108,32 @@ GroupingContext::needRanking() const return true; } -using DocId = uint32_t; +void +GroupingContext::aggregate(Grouping & grouping, uint32_t docId, HitRank rank) const { + if (_validLids.testBit(docId)) { + grouping.aggregate(docId, rank); + } +} unsigned int GroupingContext::aggregateRanked(Grouping &grouping, const RankedHit *rankedHit, unsigned int len) const { unsigned int i(0); for(; (i < len) && !hasExpired(); i++) { - grouping.aggregate(rankedHit[i].getDocId(), rankedHit[i].getRank()); + aggregate(grouping, rankedHit[i].getDocId(), rankedHit[i].getRank()); } return i; } void GroupingContext::aggregate(Grouping & grouping, const BitVector * bVec, unsigned int lidLimit) const { - for (DocId d(bVec->getFirstTrueBit()); (d < lidLimit) && !hasExpired(); d = bVec->getNextTrueBit(d+1)) { - grouping.aggregate(d, 0.0); + for (uint32_t d(bVec->getFirstTrueBit()); (d < lidLimit) && !hasExpired(); d = bVec->getNextTrueBit(d+1)) { + aggregate(grouping, d, 0.0); } } void GroupingContext::aggregate(Grouping & grouping, const BitVector * bVec, unsigned int lidLimit, unsigned int topN) const { - for(DocId d(bVec->getFirstTrueBit()), i(0); (d < lidLimit) && (i < topN) && !hasExpired(); d = bVec->getNextTrueBit(d+1), i++) { - grouping.aggregate(d, 0.0); + for(uint32_t d(bVec->getFirstTrueBit()), i(0); (d < lidLimit) && (i < topN) && !hasExpired(); d = bVec->getNextTrueBit(d+1), i++) { + aggregate(grouping, d, 0.0); } } @@ -153,4 +164,30 @@ GroupingContext::aggregate(Grouping & grouping, const RankedHit * rankedHit, uns grouping.postProcess(); } +void +GroupingContext::groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow) +{ + for (const auto & g : _groupingList) { + if ( g->needResort() ) { + aggregate(*g, searchResults, binSize, overflow); + LOG(debug, "groupUnordered: %s", g->asString().c_str()); + g->cleanTemporary(); + g->cleanupAttributeReferences(); + } + } +} + +void +GroupingContext::groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize) +{ + for (const auto & g : _groupingList) { + if ( ! g->needResort() ) { + aggregate(*g, searchResults, binSize); + LOG(debug, "groupInRelevanceOrder: %s", g->asString().c_str()); + g->cleanTemporary(); + g->cleanupAttributeReferences(); + } + } +} + } diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h index 880ffe5f767..f37046a8b3b 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h @@ -33,7 +33,7 @@ public: * @param groupSpec The grouping specification to use for initialization. * @param groupSpecLen The length of the grouping specification, in bytes. **/ - GroupingContext(const vespalib::Clock & clock, vespalib::steady_time timeOfDoom, + GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom, const char *groupSpec, uint32_t groupSpecLen, bool enableNestedMultivalueGrouping); /** @@ -41,7 +41,7 @@ public: * @param groupSpec The grouping specification to use for initialization. * @param groupSpecLen The length of the grouping specification, in bytes. **/ - GroupingContext(const vespalib::Clock & clock, vespalib::steady_time timeOfDoom); + GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom); /** * Shallow copy of references @@ -107,13 +107,18 @@ public: */ bool needRanking() const; bool enableNestedMultivalueGrouping() const noexcept { return _enableNestedMultivalueGrouping; } + const search::BitVector & getValidLids() const { return _validLids; } + void groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow); + void groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize); +private: void aggregate(Grouping & grouping, const RankedHit * rankedHit, unsigned int len, const BitVector * bv) const; void aggregate(Grouping & grouping, const RankedHit * rankedHit, unsigned int len) const; -private: + void aggregate(Grouping & grouping, uint32_t docId, HitRank rank) const; unsigned int aggregateRanked(Grouping & grouping, const RankedHit * rankedHit, unsigned int len) const; void aggregate(Grouping & grouping, const BitVector * bv, unsigned int lidLimit) const; void aggregate(Grouping & grouping, const BitVector * bv, unsigned int , unsigned int topN) const; + const BitVector & _validLids; const vespalib::Clock & _clock; vespalib::steady_time _timeOfDoom; vespalib::nbostream _os; diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp index 3eb410ad2c2..e3348f13e18 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp @@ -69,27 +69,13 @@ GroupingManager::init(const IAttributeContext &attrCtx) void GroupingManager::groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize) { - for (const auto & g : _groupingContext.getGroupingList()) { - if ( ! g->needResort() ) { - _groupingContext.aggregate(*g, searchResults, binSize); - LOG(debug, "groupInRelevanceOrder: %s", g->asString().c_str()); - g->cleanTemporary(); - g->cleanupAttributeReferences(); - } - } + _groupingContext.groupInRelevanceOrder(searchResults, binSize); } void GroupingManager::groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow) { - for (const auto & g : _groupingContext.getGroupingList()) { - if ( g->needResort() ) { - _groupingContext.aggregate(*g, searchResults, binSize, overflow); - LOG(debug, "groupUnordered: %s", g->asString().c_str()); - g->cleanTemporary(); - g->cleanupAttributeReferences(); - } - } + _groupingContext.groupUnordered(searchResults, binSize, overflow); } void diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index f0d345b4b5a..d8f42968147 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -202,6 +202,7 @@ public: DocId getNumActiveLids() const override { return _lidAlloc.getNumActiveLids(); } search::LidUsageStats getLidUsageStats() const override; std::unique_ptr<search::queryeval::Blueprint> createWhiteListBlueprint() const override; + const search::BitVector & getValidLids() const override { return _lidAlloc.getUsedLids(); } /** * Implements search::AttributeVector diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index 67f5f01ba60..6b4d8844263 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -85,6 +85,7 @@ public: } const search::BitVector &getActiveLids() const { return _activeLids.getBitVector(); } + const search::BitVector &getUsedLids() const { return _usedLids.getBitVector(); } }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 0bb183d1dc0..26555a0b9f0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -6,12 +6,12 @@ #include "match_thread.h" #include "match_tools.h" #include "extract_features.h" +#include "partial_result.h" #include <vespa/searchlib/engine/trace.h> #include <vespa/searchlib/engine/searchreply.h> #include <vespa/vespalib/util/thread_bundle.h> #include <vespa/vespalib/util/issue.h> #include <vespa/vespalib/data/slime/inserter.h> -#include <vespa/vespalib/data/slime/inject.h> #include <vespa/vespalib/data/slime/cursor.h> namespace proton::matching { diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index acbd069aac7..38336009f14 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -3,6 +3,7 @@ #include "match_thread.h" #include "document_scorer.h" #include "match_tools.h" +#include "partial_result.h" #include <vespa/searchcore/grouping/groupingmanager.h> #include <vespa/searchcore/grouping/groupingcontext.h> #include <vespa/searchlib/engine/trace.h> @@ -57,6 +58,37 @@ LazyValue get_score_feature(const RankProgram &rankProgram) { return resolver.resolve(0); } +void +fillPartialResult(ResultProcessor::Context & context, size_t totalHits, size_t numHits, + const search::RankedHit *hits, const search::BitVector * bits) __attribute__((noinline)); + +void +fillPartialResult(ResultProcessor::Context & context, size_t totalHits, size_t numHits, + const search::RankedHit *hits, const search::BitVector * bits) { + PartialResult &pr = *context.result; + pr.totalHits(totalHits); + size_t maxHits = std::min(numHits, pr.maxSize()); + //TODO :const search::BitVector & validLids = context._validLids; + if (pr.hasSortData()) { + FastS_SortSpec &spec = context.sort->sortSpec; + for (size_t i = 0; i < maxHits; ++i) { + pr.add(hits[i], spec.getSortRef(i)); + } + } else { + for (size_t i = 0; i < maxHits; ++i) { + pr.add(hits[i]); + } + if ((bits != nullptr) && (pr.size() < pr.maxSize())) { + for (unsigned int bitId = bits->getFirstTrueBit(); + (bitId < bits->size()) && (pr.size() < pr.maxSize()); + bitId = bits->getNextTrueBit(bitId + 1)) + { + pr.add(search::RankedHit(bitId)); + } + } + } +} + } // namespace proton::matching::<unnamed> //----------------------------------------------------------------------------- @@ -328,9 +360,7 @@ MatchThread::findMatches(MatchTools &tools) } void -MatchThread::processResult(const Doom & doom, - search::ResultSet::UP result, - ResultProcessor::Context &context) +MatchThread::processResult(const Doom & doom, search::ResultSet::UP result, ResultProcessor::Context &context) { if (doom.hard_doom()) return; bool hasGrouping = bool(context.grouping); @@ -338,7 +368,6 @@ MatchThread::processResult(const Doom & doom, result->mergeWithBitOverflow(fallback_rank_value()); } if (doom.hard_doom()) return; - size_t totalHits = result->getNumHits(); const search::RankedHit *hits = result->getArray(); size_t numHits = result->getArrayUsed(); search::BitVector *bits = result->getBitOverflow(); @@ -359,27 +388,7 @@ MatchThread::processResult(const Doom & doom, man.groupInRelevanceOrder(hits, numHits); } if (doom.hard_doom()) return; - PartialResult &pr = *context.result; - pr.totalHits(totalHits); - size_t maxHits = std::min(numHits, pr.maxSize()); - if (pr.hasSortData()) { - FastS_SortSpec &spec = context.sort->sortSpec; - for (size_t i = 0; i < maxHits; ++i) { - pr.add(hits[i], spec.getSortRef(i)); - } - } else { - for (size_t i = 0; i < maxHits; ++i) { - pr.add(hits[i]); - } - if ((bits != nullptr) && (pr.size() < pr.maxSize())) { - for (unsigned int bitId = bits->getFirstTrueBit(); - (bitId < bits->size()) && (pr.size() < pr.maxSize()); - bitId = bits->getNextTrueBit(bitId + 1)) - { - pr.add(search::RankedHit(bitId)); - } - } - } + fillPartialResult(context, result->getNumHits(), numHits, hits, bits); if (auto task = matchToolsFactory.createOnMatchTask()) { task->run(result->copyResult()); @@ -481,4 +490,9 @@ MatchThread::run() } } +std::unique_ptr<PartialResult> +MatchThread::extract_result() { + return std::move(resultContext->result); +} + } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index 757caae0e75..03ba34eca1f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -5,7 +5,6 @@ #include "i_match_loop_communicator.h" #include "match_params.h" #include "matching_stats.h" -#include "partial_result.h" #include "result_processor.h" #include "docid_range_scheduler.h" #include <vespa/vespalib/util/runnable.h> @@ -60,7 +59,7 @@ private: uint32_t _distributionKey; ResultProcessor &resultProcessor; vespalib::DualMergeDirector &mergeDirector; - ResultProcessor::Context::UP resultContext; + std::unique_ptr<ResultProcessor::Context> resultContext; MatchingStats::Partition thread_stats; double total_time_s; double match_time_s; @@ -135,7 +134,7 @@ public: void run() override; const MatchingStats::Partition &get_thread_stats() const { return thread_stats; } double get_match_time() const { return match_time_s; } - PartialResult::UP extract_result() { return std::move(resultContext->result); } + std::unique_ptr<PartialResult> extract_result(); const Trace & getTrace() const { return *trace; } const UniqueIssues &get_issues() const { return my_issues; } }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h index 607a9d6aab0..db30ea8d2b2 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h @@ -120,18 +120,18 @@ private: using RankSetup = search::fef::RankSetup; using IIndexEnvironment = search::fef::IIndexEnvironment; using IDiversifier = search::queryeval::IDiversifier; - QueryLimiter & _queryLimiter; - AttributeBlueprintParams _global_filter_params; - Query _query; - MaybeMatchPhaseLimiter::UP _match_limiter; + QueryLimiter & _queryLimiter; + AttributeBlueprintParams _global_filter_params; + Query _query; + MaybeMatchPhaseLimiter::UP _match_limiter; std::unique_ptr<RangeQueryLocator> _rangeLocator; - QueryEnvironment _queryEnv; - RequestContext _requestContext; - MatchDataLayout _mdl; - const RankSetup & _rankSetup; - const Properties & _featureOverrides; - DiversityParams _diversityParams; - bool _valid; + QueryEnvironment _queryEnv; + RequestContext _requestContext; + MatchDataLayout _mdl; + const RankSetup & _rankSetup; + const Properties & _featureOverrides; + DiversityParams _diversityParams; + bool _valid; std::unique_ptr<AttributeOperationTask> createTask(vespalib::stringref attribute, vespalib::stringref operation) const; @@ -172,7 +172,7 @@ public: std::unique_ptr<AttributeOperationTask> createOnSummaryTask() const; const Query & query() const { return _query; } - const RequestContext & getRequestContext() const { return _requestContext; } + const RequestContext & get_request_context() const { return _requestContext; } const StringStringMap & get_feature_rename_map() const; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 26cf9901f57..aedfde2521c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -16,6 +16,7 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/ranksetup.h> #include <vespa/searchlib/fef/test/plugin/setup.h> +#include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/vespalib/data/slime/inserter.h> #include <cinttypes> @@ -51,6 +52,8 @@ constexpr vespalib::duration TIME_BEFORE_ALLOWING_SOFT_TIMEOUT_FACTOR_ADJUSTMENT // used to give out empty whitelist blueprints struct StupidMetaStore : search::IDocumentMetaStore { + static const search::AllocatedBitVector _dummy; + const search::BitVector & getValidLids() const override { return _dummy; } bool getGid(DocId, GlobalId &) const override { return false; } bool getGidEvenIfMoved(DocId, GlobalId &) const override { return false; } bool getLid(const GlobalId &, DocId &) const override { return false; } @@ -65,6 +68,8 @@ struct StupidMetaStore : search::IDocumentMetaStore { void foreach(const search::IGidToLidMapperVisitor &) const override { } }; +const search::AllocatedBitVector StupidMetaStore::_dummy(1); + size_t numThreads(size_t hits, size_t minHits) { return static_cast<size_t>(std::ceil(double(hits) / double(minHits))); @@ -245,7 +250,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl bool isDoomExplicit = false; { // we want to measure full set-up and tear-down time as part of // collateral time - GroupingContext groupingContext(_clock, request.getTimeOfDoom(), + GroupingContext groupingContext(metaStore.getValidLids(), _clock, request.getTimeOfDoom(), request.groupSpec.data(), request.groupSpec.size(), _rankSetup->enableNestedMultivalueGrouping()); SessionId sessionId(request.sessionId.data(), request.sessionId.size()); bool shouldCacheSearchSession = false; @@ -270,7 +275,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl MatchToolsFactory::UP mtf = create_match_tools_factory(request, searchContext, attrContext, metaStore, *feature_overrides, threadBundle, &owned_objects.readGuard, true); - isDoomExplicit = mtf->getRequestContext().getDoom().isExplicitSoftDoom(); + isDoomExplicit = mtf->get_request_context().getDoom().isExplicitSoftDoom(); traceQuery(6, request.trace(), mtf->query()); if (!mtf->valid()) { return reply; diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h index 314fefa3cc0..d031e1893ce 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h +++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h @@ -5,7 +5,6 @@ #include <vespa/vespalib/util/dual_merge_director.h> #include <vespa/searchlib/common/rankedhit.h> #include <vector> -#include <cassert> namespace proton::matching { @@ -16,19 +15,10 @@ namespace proton::matching { class PartialResult : public vespalib::DualMergeDirector::Source { public: - using UP = std::unique_ptr<PartialResult>; using SortRef = std::pair<const char *, size_t>; - -private: - std::vector<search::RankedHit> _hits; - std::vector<SortRef> _sortData; - size_t _maxSize; - size_t _totalHits; - bool _hasSortData; - size_t _sortDataSize; - -public: PartialResult(size_t maxSize_in, bool hasSortData_in); + PartialResult(const PartialResult &) = delete; + PartialResult & operator =(const PartialResult &) = delete; ~PartialResult() override; size_t size() const { return _hits.size(); } size_t maxSize() const { return _maxSize; } @@ -39,16 +29,21 @@ public: const SortRef &sortData(size_t i) const { return _sortData[i]; } void totalHits(size_t th) { _totalHits = th; } void add(const search::RankedHit &h) { - assert(!_hasSortData); _hits.push_back(h); } void add(const search::RankedHit &h, const SortRef &sd) { - assert(_hasSortData); _hits.push_back(h); _sortData.push_back(sd); _sortDataSize += sd.second; } void merge(Source &rhs) override; +private: + std::vector<search::RankedHit> _hits; + std::vector<SortRef> _sortData; + size_t _maxSize; + size_t _totalHits; + bool _hasSortData; + size_t _sortDataSize; }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp index 1608c633124..a973e264269 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp @@ -36,8 +36,9 @@ ResultProcessor::Sort::Sort(uint32_t partitionId, const vespalib::Doom & doom, I } } -ResultProcessor::Context::Context(Sort::UP s, PartialResult::UP r, GroupingContext::UP g) - : sort(std::move(s)), +ResultProcessor::Context::Context(const search::BitVector & validLids, Sort::UP s, PartialResultUP r, GroupingContext::UP g) + : _validLids(validLids), + sort(std::move(s)), result(std::move(r)), grouping(std::move(g)), groupingSource(grouping.get()) @@ -90,7 +91,7 @@ ResultProcessor::prepareThreadContextCreation(size_t num_threads) } } -ResultProcessor::Context::UP +std::unique_ptr<ResultProcessor::Context> ResultProcessor::createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id, uint32_t distributionKey) { auto sort = std::make_unique<Sort>(distributionKey, hardDoom, _attrContext, _sortSpec); @@ -99,7 +100,7 @@ ResultProcessor::createThreadContext(const vespalib::Doom & hardDoom, size_t thr if (_groupingSession) { groupingContext = _groupingSession->createThreadContext(thread_id, _attrContext); } - return std::make_unique<Context>(std::move(sort), std::move(result), std::move(groupingContext)); + return std::make_unique<Context>(_metaStore.getValidLids(), std::move(sort), std::move(result), std::move(groupingContext)); } std::vector<std::pair<uint32_t,uint32_t>> diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h index 54b9adc4723..49fd9f37063 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h @@ -14,6 +14,7 @@ namespace search { class GroupingSession; } struct IDocumentMetaStore; + class BitVector; } namespace proton::matching { @@ -58,15 +59,15 @@ public: * Context per thread used for result processing. **/ struct Context { - using UP = std::unique_ptr<Context>; using GroupingContextUP = std::unique_ptr<GroupingContext>; + const search::BitVector & _validLids; Sort::UP sort; PartialResultUP result; GroupingContextUP grouping; GroupingSource groupingSource; - Context(Sort::UP s, PartialResultUP r, GroupingContextUP g); + Context(const search::BitVector & validLids, Sort::UP s, PartialResultUP r, GroupingContextUP g); ~Context(); }; @@ -101,7 +102,7 @@ public: ~ResultProcessor(); void prepareThreadContextCreation(size_t num_threads); - Context::UP createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id, uint32_t distributionKey); + std::unique_ptr<Context> createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id, uint32_t distributionKey); std::vector<std::pair<uint32_t,uint32_t>> extract_docid_ordering(const PartialResult &result) const; std::unique_ptr<Result> makeReply(PartialResultUP full_result); }; diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h index b21cc809794..89ed8312ec4 100644 --- a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h @@ -51,6 +51,9 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore uint64_t getCurrentGeneration() const override { return _store.getCurrentGeneration(); } + const search::BitVector & getValidLids() const override { + return _store.getValidLids(); + } /** diff --git a/searchlib/src/vespa/searchlib/common/idocumentmetastore.h b/searchlib/src/vespa/searchlib/common/idocumentmetastore.h index 66b4905ea64..729bede1e71 100644 --- a/searchlib/src/vespa/searchlib/common/idocumentmetastore.h +++ b/searchlib/src/vespa/searchlib/common/idocumentmetastore.h @@ -55,13 +55,10 @@ struct DocumentMetaData { } }; -namespace queryeval { - -class Blueprint; - -} +namespace queryeval { class Blueprint; } class IGidToLidMapperVisitor; +class BitVector; /** @@ -77,6 +74,8 @@ struct IDocumentMetaStore { virtual ~IDocumentMetaStore() = default; + virtual const BitVector & getValidLids() const = 0; + /** * Retrieves the gid associated with the given lid. * Returns true if found, false otherwise. diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzDomainMeta.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzDomainMeta.java index 7808a689982..c0f4f8958bf 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzDomainMeta.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzDomainMeta.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties(ignoreUnknown = true) public record AthenzDomainMeta( @JsonProperty("account") String account, + @JsonProperty("gcpProject") String gcpProject, @JsonProperty("name") String name ) { } |