diff options
95 files changed, 640 insertions, 1632 deletions
diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index 6a5b936434c..10fb2743c63 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -33,6 +33,8 @@ type visitArgs struct { to string slices int sliceId int + bucketSpace string + bucketSpaces []string cli *CLI } @@ -132,6 +134,7 @@ $ vespa visit --field-set "[id]" # list document IDs cmd.Flags().StringVar(&vArgs.to, "to", "", `Timestamp to visit up to, in seconds`) cmd.Flags().IntVar(&vArgs.sliceId, "slice-id", -1, `The number of the slice this visit invocation should fetch`) cmd.Flags().IntVar(&vArgs.slices, "slices", -1, `Split the document corpus into this number of independent slices`) + cmd.Flags().StringSliceVar(&vArgs.bucketSpaces, "bucket-space", []string{"global", "default"}, `"default" or "global" bucket space`) return cmd } @@ -157,6 +160,16 @@ func checkArguments(vArgs visitArgs) (res util.OperationResult) { return util.Failure("Invalid 'to' argument: '" + vArgs.to + "': " + err.Error()) } } + for _, b := range vArgs.bucketSpaces { + switch b { + case + "default", + "global": + // Do nothing + default: + return util.Failure("Invalid 'bucket-space' argument '" + b + "', must be 'default' or 'global'") + } + } return util.Success("") } @@ -226,13 +239,16 @@ func visitClusters(vArgs *visitArgs, service *vespa.Service) (res util.Operation if vArgs.makeFeed { vArgs.writeString("[\n") } - for _, c := range clusters { - vArgs.contentCluster = c - res = runVisit(vArgs, service) - if !res.Success { - return res + for _, b := range vArgs.bucketSpaces { + for _, c := range clusters { + vArgs.bucketSpace = b + vArgs.contentCluster = c + res = runVisit(vArgs, service) + if !res.Success { + return res + } + vArgs.debugPrint("Success: " + res.Message) } - vArgs.debugPrint("Success: " + res.Message) } if vArgs.makeFeed { vArgs.writeString("{}\n]\n") @@ -330,6 +346,9 @@ func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*V if vArgs.slices > 0 { urlPath = urlPath + fmt.Sprintf("&slices=%d&sliceId=%d", vArgs.slices, vArgs.sliceId) } + if vArgs.bucketSpace != "" { + urlPath = urlPath + "&bucketSpace=" + vArgs.bucketSpace + } url, urlParseError := url.Parse(urlPath) if urlParseError != nil { return nil, util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index b6e5b893e0b..9bb8f61554a 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -105,6 +105,7 @@ func TestVisitCommand(t *testing.T) { assertVisitResults( []string{ "visit", + "--bucket-space", "default", "--json-lines", }, t, @@ -118,7 +119,7 @@ func TestVisitCommand(t *testing.T) { document3 + `],"documentCount":2}`, }, - "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000", + "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000&bucketSpace=default", document1+"\n"+ document2+"\n"+ document3+"\n") diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 180c8e6012f..93bec4975a6 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -709,8 +709,17 @@ public class Schema implements ImmutableSchema { public FieldSets fieldSets() { return fieldSets; } + private Schema inheritedSchema = null; + + public void setInheritedSchema(Schema value) { + inheritedSchema = value; + } + /** Returns the schema inherited by this, or throws if none */ - private Schema requireInherited() { return owner.schemas().get(inherited.get()); } + private Schema requireInherited() { + if (inheritedSchema != null) return inheritedSchema; + return owner.schemas().get(inherited.get()); + } /** * For adding structs defined in document scope diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index 0abcc9e890a..40ec84ec8bc 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -98,6 +98,13 @@ public class ConvertParsedSchemas { Schema schema = parsed.getDocumentWithoutSchema() ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); + inherited.ifPresent(parentName -> { + for (var possibleParent : resultList) { + if (possibleParent.getName().equals(parentName)) { + schema.setInheritedSchema(possibleParent); + } + } + }); convertSchema(schema, parsed); resultList.add(schema); } @@ -145,7 +152,23 @@ public class ConvertParsedSchemas { docsum.setOmitSummaryFeatures(true); } for (var parsedField : parsed.getSummaryFields()) { - DataType dataType = typeContext.resolveType(parsedField.getType()); + var parsedType = parsedField.getType(); + DataType dataType = (parsedType != null) ? typeContext.resolveType(parsedType) : null; + var existingField = schema.getField(parsedField.name()); + if (existingField != null) { + var existingType = existingField.getDataType(); + if (dataType == null) { + dataType = existingType; + } else if (!dataType.equals(existingType)) { + if (dataType.getValueClass().equals(com.yahoo.document.datatypes.WeightedSet.class)) { + // "adjusting type for field " + parsedField.name() + " in document-summary " + parsed.name() + " field already has: " + existingType + " but declared type was: " + dataType + dataType = existingType; + } + } + } + if (dataType == null) { + throw new IllegalArgumentException("Missing data-type for summary field " + parsedField.name() + " in document-summary " + parsed.name()); + } var summaryField = new SummaryField(parsedField.name(), dataType); // XXX does not belong here: summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java deleted file mode 100644 index d9bf82980d7..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidator.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; - -/** - * Validates that a Vespa Cloud application has at least one container cluster. - * - * @author jonmv - */ -public class ContainerInCloudValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - if (deployState.isHosted() && model.getContainerClusters().isEmpty()) - throw new IllegalArgumentException("Vespa Cloud applications must have at least one container cluster"); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 4f2f8e7932c..2576d9cb392 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -76,7 +76,6 @@ public class Validation { new StreamingValidator().validate(model, deployState); new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); - new ContainerInCloudValidator().validate(model, deployState); new DeploymentSpecValidator().validate(model, deployState); new ValidationOverridesValidator().validate(model, deployState); new ConstantValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 5a96e33c522..e2166b263ee 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.utils.Duration; import org.w3c.dom.Element; import java.util.Optional; @@ -47,9 +48,29 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc clusterControllerTuning = tuning.child("cluster-controller"); } + var tuningConfig = new ClusterControllerTuningBuilder(clusterControllerTuning, + minNodeRatioPerGroup, + bucketSplittingMinimumBits) + .build(); + if (ancestor instanceof ContentCluster) { + int numberOfLeafGroups = ((ContentCluster) ancestor).getRootGroup().getNumberOfLeafGroups(); + if (tuningConfig.maxGroupsAllowedDown().isPresent()) { + Integer maxGroupsAllowedDown = tuningConfig.maxGroupsAllowedDown().get(); + if (deployState.zone().environment().isProduction() && (maxGroupsAllowedDown > numberOfLeafGroups)) + throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + maxGroupsAllowedDown + + ") larger than number of groups (" + numberOfLeafGroups + ")"); + } else { + // Reduce to numberOfLeafGroups for tests or in environments where number of groups are reduced by policy (dev, test, staging, perf) + tuningConfig = tuningConfig.withMaxGroupsAllowedDown(numberOfLeafGroups); + } + } else { + // Reduce to 1 for tests (ancestor is a mock class) + tuningConfig = tuningConfig.withMaxGroupsAllowedDown(1); + } + return new ClusterControllerConfig(ancestor, clusterName, - new ClusterControllerTuning(clusterControllerTuning, minNodeRatioPerGroup, bucketSplittingMinimumBits), + tuningConfig, resourceLimits, allowMoreThanOneContentGroupDown); } @@ -99,45 +120,81 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc resourceLimits.getConfig(builder); } - public ClusterControllerTuning tuning() { return tuning; } - - public static class ClusterControllerTuning { - - private final Optional<Double> minNodeRatioPerGroup; - private final Optional<Duration> initProgressTime; - private final Optional<Duration> transitionTime; - private final Optional<Long> maxPrematureCrashes; - private final Optional<Duration> stableStateTimePeriod; - private final Optional<Double> minDistributorUpRatio; - private final Optional<Double> minStorageUpRatio; - private final Optional<Integer> minSplitBits; - final Optional<Integer> maxGroupsAllowedDown; - - ClusterControllerTuning(ModelElement tuning, - Optional<Double> minNodeRatioPerGroup, - Optional<Integer> bucketSplittingMinimumBits) { - this.minSplitBits = bucketSplittingMinimumBits; - this.minNodeRatioPerGroup = minNodeRatioPerGroup; - if (tuning == null) { - this.initProgressTime = Optional.empty(); - this.transitionTime = Optional.empty(); - this.maxPrematureCrashes = Optional.empty(); - this.stableStateTimePeriod = Optional.empty(); - this.minDistributorUpRatio = Optional.empty(); - this.minStorageUpRatio = Optional.empty(); - this.maxGroupsAllowedDown = Optional.empty(); - } else { - this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time")); - this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time")); - this.maxPrematureCrashes = Optional.ofNullable(tuning.childAsLong("max-premature-crashes")); - this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period")); - this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio")); - this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio")); - this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down")); - } + public ClusterControllerTuning tuning() {return tuning;} + +private static class ClusterControllerTuningBuilder { + + private final Optional<Double> minNodeRatioPerGroup; + private final Optional<Duration> initProgressTime; + private final Optional<Duration> transitionTime; + private final Optional<Long> maxPrematureCrashes; + private final Optional<Duration> stableStateTimePeriod; + private final Optional<Double> minDistributorUpRatio; + private final Optional<Double> minStorageUpRatio; + private final Optional<Integer> minSplitBits; + final Optional<Integer> maxGroupsAllowedDown; + + ClusterControllerTuningBuilder(ModelElement tuning, + Optional<Double> minNodeRatioPerGroup, + Optional<Integer> bucketSplittingMinimumBits) { + this.minSplitBits = bucketSplittingMinimumBits; + this.minNodeRatioPerGroup = minNodeRatioPerGroup; + if (tuning == null) { + this.initProgressTime = Optional.empty(); + this.transitionTime = Optional.empty(); + this.maxPrematureCrashes = Optional.empty(); + this.stableStateTimePeriod = Optional.empty(); + this.minDistributorUpRatio = Optional.empty(); + this.minStorageUpRatio = Optional.empty(); + this.maxGroupsAllowedDown = Optional.empty(); + } else { + this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time")); + this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time")); + this.maxPrematureCrashes = Optional.ofNullable(tuning.childAsLong("max-premature-crashes")); + this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period")); + this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio")); + this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio")); + this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down")); } + } - public Optional<Integer> maxGroupsAllowedDown() { return maxGroupsAllowedDown; } + private ClusterControllerTuning build() { + return new ClusterControllerTuning(initProgressTime, + transitionTime, + maxPrematureCrashes, + stableStateTimePeriod, + minDistributorUpRatio, + minStorageUpRatio, + maxGroupsAllowedDown, + minNodeRatioPerGroup, + minSplitBits); } } + +private record ClusterControllerTuning(Optional<Duration> initProgressTime, + Optional<Duration> transitionTime, + Optional<Long> maxPrematureCrashes, + Optional<Duration> stableStateTimePeriod, + Optional<Double> minDistributorUpRatio, + Optional<Double> minStorageUpRatio, + Optional<Integer> maxGroupsAllowedDown, + Optional<Double> minNodeRatioPerGroup, + Optional<Integer> minSplitBits) { + + public ClusterControllerTuning withMaxGroupsAllowedDown(int maxGroupsAllowedDown) { + return new ClusterControllerConfig.ClusterControllerTuning( + initProgressTime, + transitionTime, + maxPrematureCrashes, + stableStateTimePeriod, + minDistributorUpRatio, + minStorageUpRatio, + Optional.of(maxGroupsAllowedDown), + minNodeRatioPerGroup, + minSplitBits); + } + +} + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index f1f210b013c..f1d5c7c9220 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -164,18 +164,12 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem DeployState deployState, ContentCluster c, ClusterResourceLimits resourceLimits) { - var config = new ClusterControllerConfig.Builder(c.clusterId, - contentElement, - resourceLimits.getClusterControllerLimits(), - deployState.featureFlags() - .allowMoreThanOneContentGroupDown(new ClusterSpec.Id(c.clusterId))) + return new ClusterControllerConfig.Builder(c.clusterId, + contentElement, + resourceLimits.getClusterControllerLimits(), + deployState.featureFlags() + .allowMoreThanOneContentGroupDown(new ClusterSpec.Id(c.clusterId))) .build(deployState, c, contentElement.getXml()); - config.tuning().maxGroupsAllowedDown().ifPresent(m -> { - int numberOfLeafGroups = c.getRootGroup().getNumberOfLeafGroups(); - if (m > numberOfLeafGroups) - throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + m + ") larger than number of groups (" + numberOfLeafGroups + ")"); - }); - return config; } private void setupSearchCluster(ContentSearchCluster csc, ModelElement element, DeployLogger logger) { diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 9d6e16b3f67..9a38fdc673e 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -1075,15 +1075,16 @@ void attributeSetting(ParsedAttribute attribute) : void summaryInDocument(ParsedDocumentSummary docsum) : { String name; - ParsedType type; + ParsedType type = null; ParsedSummaryField psf; } { <SUMMARY> name = identifierWithDash() { } - <TYPE> type = dataType() { + (<TYPE> type = dataType())? + lbrace() { psf = new ParsedSummaryField(name, type); } - lbrace() (summaryItem(psf) (<NL>)*)* <RBRACE> + (summaryItem(psf) (<NL>)*)* <RBRACE> { var old = docsum.addField(psf); if (old != null) { diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index 085a9eb232f..50f2419fc58 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -82,8 +82,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "h" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "h" -indexinfo[].command[].command "string" -indexinfo[].command[].indexname "h" indexinfo[].command[].command "type WeightedSet<string>" indexinfo[].command[].indexname "loc" indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd index 51259802a3a..5f93a6e512b 100644 --- a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd +++ b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd @@ -62,6 +62,7 @@ schema multiplesummaries { field h type weightedset<string> { indexing: summary + weightedset: create-if-nonexistent } field loc type string { @@ -91,7 +92,7 @@ schema multiplesummaries { summary e type string { } - summary f type array<string> { + summary f { } summary g type array<int> { @@ -209,7 +210,7 @@ schema multiplesummaries { bolding: on } - summary c type string { + summary c { } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 90b4625a282..f1dffe53ad7 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -15,6 +15,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorStatusConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -2352,6 +2353,38 @@ public class ModelProvisioningTest { } @Test + public void testAllow2ContentGroupsDown() { + String servicesXml = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='qrs'>" + + " <nodes count='1'/>" + + " </container>" + + " <content version='1.0' id='content'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='4' groups='4'/>" + + " <tuning>" + + " <cluster-controller>" + + " <max-groups-allowed-down>2</max-groups-allowed-down>" + + " </cluster-controller>" + + " </tuning>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.setModelProperties(new TestProperties().setAllowMoreThanOneContentGroupDown(true)); + tester.addHosts(9); + VespaModel model = tester.createModel(servicesXml, true, new DeployState.Builder() + .properties(new TestProperties().setAllowMoreThanOneContentGroupDown(true))); + + var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder(); + model.getConfig(fleetControllerConfigBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-content-configurer"); + assertEquals(2, fleetControllerConfigBuilder.build().max_number_of_groups_allowed_to_be_down()); + } + + @Test public void containerWithZooKeeperSuboptimalNodeCountDuringRetirement() { String servicesXml = "<?xml version='1.0' encoding='utf-8' ?>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java deleted file mode 100644 index 3feb8888821..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.vespa.model.VespaModel; -import org.junit.jupiter.api.Test; -import org.xml.sax.SAXException; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -public class ContainerInCloudValidatorTest { - - @Test - void failsWhenNoContainerInCloud() throws IOException, SAXException { - String noContainer = ""; - String container = """ - <container id='default' version='1.0'> - <nodes count='2' /> - </container> - """; - runValidatorOnApp(false, container); - runValidatorOnApp(false, noContainer); - runValidatorOnApp(true, container); - assertEquals("Vespa Cloud applications must have at least one container cluster", - assertThrows(IllegalArgumentException.class, - () -> runValidatorOnApp(true, noContainer)) - .getMessage()); - } - - private static void runValidatorOnApp(boolean isHosted, String container) throws IOException, SAXException { - String servicesXml = """ - <services version='1.0'> - %s - <content version='1.0'> - <redundancy>2</redundancy> - <documents> - </documents> - <nodes count='2' /> - </content> - </services> - """.formatted(container); - ApplicationPackage app = new MockApplicationPackage.Builder() - .withServices(servicesXml) - .build(); - DeployState deployState = new DeployState.Builder() - .applicationPackage(app) - .properties(new TestProperties().setHostedVespa(isHosted).setAllowUserFilters(false)) - .build(); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new ContainerInCloudValidator().validate(model, deployState); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index 78d3838d39d..cb535380b18 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -35,7 +35,7 @@ public class ValidationTester { /** Creates a validation tester with 1 node available (in addition to cluster controllers) */ public ValidationTester() { - this(5); + this(4); } /** Creates a validation tester with number of nodes available and the given test properties */ diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java index 784174a35a0..5d0a1704a1d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -79,9 +79,6 @@ public class IndexingModeChangeValidatorTest { private static String getServices(String indexingMode) { return "<services version='1.0'>" + - " <container id='default-container' version='1.0'>" + - " <nodes count='1'/>" + - " </container>" + " <content id='default' version='1.0'>" + " <redundancy>1</redundancy>" + " <documents>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index a33b30f7d93..12a6ac00f48 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1331,14 +1331,10 @@ public class ContentClusterTest extends ContentBaseTest { " </engine>" + " </content>" + " </services>"; - VespaModel model = createEnd2EndOneNode(new TestProperties() - .setHostedVespa(false) - .setMultitenant(true) - .setAllowMoreThanOneContentGroupDown(true), - services); + VespaModel model = createEnd2EndOneNode(new TestProperties().setAllowMoreThanOneContentGroupDown(true), services); var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder(); - model.getConfig(fleetControllerConfigBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-storage-configurer"); + model.getConfig(fleetControllerConfigBuilder, "admin/cluster-controllers/0/components/clustercontroller-storage-configurer"); assertEquals(2, fleetControllerConfigBuilder.build().max_number_of_groups_allowed_to_be_down()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index ae22542de6c..138852e1c5c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -41,14 +41,14 @@ public class FleetControllerClusterTest { parse(""" <cluster id="storage"> <documents/> <tuning> - <bucket-splitting minimum-bits="7" /> <cluster-controller> + <bucket-splitting minimum-bits="7" /> + <cluster-controller> <init-progress-time>13</init-progress-time> <transition-time>27</transition-time> <max-premature-crashes>4</max-premature-crashes> <stable-state-period>72</stable-state-period> <min-distributor-up-ratio>0.7</min-distributor-up-ratio> <min-storage-up-ratio>0.3</min-storage-up-ratio> - <max-groups-allowed-down>2</max-groups-allowed-down> </cluster-controller> </tuning> </cluster>""", @@ -63,7 +63,6 @@ public class FleetControllerClusterTest { assertEquals(0.7, config.min_distributor_up_ratio(), 0.01); assertEquals(0.3, config.min_storage_up_ratio(), 0.01); assertEquals(7, config.ideal_distribution_bits()); - assertEquals(2, config.max_number_of_groups_allowed_to_be_down()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 48ddf6b8a82..500fb0838e1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -21,7 +21,6 @@ import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; - import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -50,6 +49,7 @@ public class VespaModelTester { private final ConfigModelRegistry configModelRegistry; private boolean hosted = true; + private TestProperties modelProperties = new TestProperties(); private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; @@ -101,6 +101,9 @@ public class VespaModelTester { /** Sets whether this sets up a model for a hosted system. Default: true */ public void setHosted(boolean hosted) { this.hosted = hosted; } + /** Sets whether this sets up a model for a hosted system. Default: true */ + public void setModelProperties(TestProperties testProperties) { this.modelProperties = testProperties; } + /** Sets architecture to use for admin clusters. Default: x86_64 */ public void setAdminClusterArchitecture(Architecture architecture) { this.adminClusterArchitecture = architecture; @@ -206,7 +209,7 @@ public class VespaModelTester { provisioner = new SingleNodeProvisioner(); } - TestProperties properties = new TestProperties() + TestProperties properties = modelProperties .setMultitenant(hosted) // Note: system tests are multitenant but not hosted .setHostedVespa(hosted) .setApplicationId(applicationId) diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 70dff6730ff..409b807c833 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -9,9 +9,7 @@ import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; -import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.messagebus.Message; @@ -300,12 +298,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public void onMessage(Message m, AckToken token) { if (m instanceof QueryResultMessage qm) { onQueryResult(qm.getResult(), qm.getSummary()); - } else if (m instanceof SearchResultMessage) { - onSearchResult(((SearchResultMessage) m).getResult()); - } else if (m instanceof DocumentSummaryMessage dsm) { - onDocumentSummary(dsm.getResult()); } else { - throw new UnsupportedOperationException("Received unsupported message " + m + ". VdsVisitor can only accept query result, search result, and documentsummary messages."); + throw new UnsupportedOperationException("Received unsupported message " + m + ". VdsVisitor can only accept query result messages."); } ack(token); } @@ -320,13 +314,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { handleSummary(summary); } - public void onSearchResult(SearchResult sr) { - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, "Got SearchResult for query with selection " + params.getDocumentSelection()); - } - handleSearchResult(sr); - } - private void handleSearchResult(SearchResult sr) { final int hitCountTotal = sr.getTotalHitCount(); final int hitCount = sr.getHitCount(); @@ -377,13 +364,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } } - public void onDocumentSummary(DocumentSummary ds) { - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); - } - handleSummary(ds); - } - private void handleSummary(DocumentSummary ds) { int summaryCount = ds.getSummaryCount(); if (log.isLoggable(Level.FINE)) { diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index 0e3ecf1c8cc..28a34ff2f6d 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -5,9 +5,7 @@ import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.*; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; -import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.routing.Route; @@ -63,18 +61,6 @@ public class VdsVisitorTestCase { return qrm; } - private SearchResultMessage createSRM(String docId, double rank) { - SearchResultMessage srm = new SearchResultMessage(); - srm.setSearchResult(createSR(docId, rank)); - return srm; - } - - private DocumentSummaryMessage createDSM(String docId) { - DocumentSummaryMessage dsm = new DocumentSummaryMessage(); - dsm.setDocumentSummary(createDS(docId)); - return dsm; - } - private Message createM() { return new Message() { @Override @@ -357,15 +343,13 @@ public class VdsVisitorTestCase { private void supplyResults(VdsVisitor visitor) { AckToken ackToken = null; visitor.onMessage(createQRM("id:ns:type::0", 0.3), ackToken); - visitor.onMessage(createSRM("id:ns:type::1", 1.0), ackToken); - visitor.onMessage(createSRM("id:ns:type::2", 0.5), ackToken); - visitor.onMessage(createDSM("id:ns:type::1"), ackToken); - visitor.onMessage(createDSM("id:ns:type::2"), ackToken); + visitor.onMessage(createQRM("id:ns:type::1", 1.0), ackToken); + visitor.onMessage(createQRM("id:ns:type::2", 0.5), ackToken); try { visitor.onMessage(createM(), ackToken); assertTrue(false, "Unsupported message did not cause exception"); } catch (UnsupportedOperationException uoe) { - assertTrue(uoe.getMessage().contains("VdsVisitor can only accept query result, search result, and documentsummary messages")); + assertTrue(uoe.getMessage().contains("VdsVisitor can only accept query result messages")); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java index 796ce5da449..65c8e8390c8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import java.util.Objects; @@ -22,15 +23,20 @@ public class NodeFilter { private final Set<HostName> hostnames; private final Set<HostName> parentHostnames; private final Set<ApplicationId> applications; + private final Set<ClusterSpec.Id> clusterIds; + private final Set<Node.ClusterType> clusterTypes; private NodeFilter(boolean includeDeprovisioned, Set<Node.State> states, Set<HostName> hostnames, - Set<HostName> parentHostnames, Set<ApplicationId> applications) { + Set<HostName> parentHostnames, Set<ApplicationId> applications, + Set<ClusterSpec.Id> clusterIds, Set<Node.ClusterType> clusterTypes) { this.includeDeprovisioned = includeDeprovisioned; // Uses Guava Set to preserve insertion order this.states = ImmutableSet.copyOf(Objects.requireNonNull(states)); this.hostnames = ImmutableSet.copyOf(Objects.requireNonNull(hostnames)); this.parentHostnames = ImmutableSet.copyOf(Objects.requireNonNull(parentHostnames)); this.applications = ImmutableSet.copyOf(Objects.requireNonNull(applications)); + this.clusterIds = ImmutableSet.copyOf(Objects.requireNonNull(clusterIds)); + this.clusterTypes = ImmutableSet.copyOf(Objects.requireNonNull(clusterTypes)); if (!includeDeprovisioned && states.contains(Node.State.deprovisioned)) { throw new IllegalArgumentException("Must include deprovisioned nodes when matching deprovisioned state"); } @@ -56,8 +62,16 @@ public class NodeFilter { return applications; } + public Set<ClusterSpec.Id> clusterIds() { + return clusterIds; + } + + public Set<Node.ClusterType> clusterTypes() { + return clusterTypes; + } + public NodeFilter includeDeprovisioned(boolean includeDeprovisioned) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } public NodeFilter states(Node.State... states) { @@ -65,7 +79,7 @@ public class NodeFilter { } public NodeFilter states(Set<Node.State> states) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } public NodeFilter hostnames(HostName... hostnames) { @@ -73,7 +87,7 @@ public class NodeFilter { } public NodeFilter hostnames(Set<HostName> hostnames) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } public NodeFilter parentHostnames(HostName... parentHostnames) { @@ -81,7 +95,7 @@ public class NodeFilter { } public NodeFilter parentHostnames(Set<HostName> parentHostnames) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } public NodeFilter applications(ApplicationId... applications) { @@ -89,12 +103,28 @@ public class NodeFilter { } public NodeFilter applications(Set<ApplicationId> applications) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); + } + + public NodeFilter clusterIds(ClusterSpec.Id... clusterIds) { + return clusterIds(ImmutableSet.copyOf(clusterIds)); + } + + public NodeFilter clusterIds(Set<ClusterSpec.Id> clusterIds) { + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); + } + + public NodeFilter clusterTypes(Node.ClusterType... clusterTypes) { + return clusterTypes(ImmutableSet.copyOf(clusterTypes)); + } + + public NodeFilter clusterTypes(Set<Node.ClusterType> clusterTypes) { + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } /** A filter which matches all nodes, except deprovisioned ones */ public static NodeFilter all() { - return new NodeFilter(false, Set.of(), Set.of(), Set.of(), Set.of()); + return new NodeFilter(false, Set.of(), Set.of(), Set.of(), Set.of(), Set.of(), Set.of()); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 4c5a67626ea..485bf627c87 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; @@ -11,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.Applicat import java.util.List; import java.util.Map; +import java.util.Optional; /** * Node repository interface intended for use by the controller. @@ -67,6 +69,9 @@ public interface NodeRepository { /** Retire given node */ void retire(ZoneId zone, String hostname, boolean wantToRetire, boolean wantToDeprovision); + /** Drop all documents on content nodes in the given zone, application and cluster */ + void dropDocuments(ZoneId zoneId, ApplicationId applicationId, Optional<ClusterSpec.Id> clusterId); + /** Update reports for given node. A key with null value clears that report */ void updateReports(ZoneId zone, String hostname, Map<String, String> reports); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index fef29a99a47..ac895022130 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -173,6 +173,10 @@ enum PathGroup { Matcher.application, "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/node/{node}/service-dump"), + dropDocuments(Matcher.tenant, + Matcher.application, + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/drop-documents"), + /** Paths used for development deployments. */ developmentDeployment(Matcher.tenant, Matcher.application, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index bb53ae61525..9a28226c921 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -87,7 +87,7 @@ enum Policy { /** Read access to application information and settings. */ applicationRead(Privilege.grant(Action.read) - .on(PathGroup.application, PathGroup.applicationInfo, PathGroup.reindexing, PathGroup.serviceDump) + .on(PathGroup.application, PathGroup.applicationInfo, PathGroup.reindexing, PathGroup.serviceDump, PathGroup.dropDocuments) .in(SystemName.all())), /** Update access to application information and settings. */ @@ -102,7 +102,7 @@ enum Policy { /** Full access to application information and settings. */ applicationOperations(Privilege.grant(Action.write()) - .on(PathGroup.applicationInfo, PathGroup.productionRestart, PathGroup.reindexing, PathGroup.serviceDump) + .on(PathGroup.applicationInfo, PathGroup.productionRestart, PathGroup.reindexing, PathGroup.serviceDump, PathGroup.dropDocuments) .in(SystemName.all())), /** Access to create and delete developer and deploy keys under a tenant. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index ded27ee1060..d5029d2d8a5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -131,6 +131,7 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -281,6 +282,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/private-services")) return getPrivateServiceInfo(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/drop-documents")) return dropDocumentsStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return supportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return getServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/scaling")) return scaling(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -346,6 +348,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return enableReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspend")) return suspend(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/drop-documents")) return dropDocuments(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return allowSupportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return requestServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploySystemApplication(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -2017,6 +2020,65 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } + private HttpResponse dropDocumentsStatus(String tenant, String application, String instance, String environment, String region, Optional<ClusterSpec.Id> clusterId) { + ZoneId zone = ZoneId.from(environment, region); + if (!zone.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Drop documents status is only available for manually deployed environments"); + + ApplicationId applicationId = ApplicationId.from(tenant, application, instance); + NodeFilter filters = NodeFilter.all() + .states(Node.State.active) + .applications(applicationId) + .clusterTypes(Node.ClusterType.content, Node.ClusterType.combined); + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, clusterId.map(filters::clusterIds).orElse(filters)); + if (nodes.isEmpty()) { + throw new NotExistsException("No content nodes found for %s%s in %s".formatted( + applicationId.toFullString(), clusterId.map(id -> " cluster " + id).orElse(""), zone)); + } + + Instant readiedAt = null; + int numNoReport = 0, numInitial = 0, numDropped = 0, numReadied = 0, numStarted = 0; + for (Node node : nodes) { + Inspector report = Optional.ofNullable(node.reports().get("dropDocuments")) + .map(json -> SlimeUtils.jsonToSlime(json).get()).orElse(null); + if (report == null) numNoReport++; + else if (report.field("startedAt").valid()) { + numStarted++; + readiedAt = SlimeUtils.instant(report.field("readiedAt")); + } else if (report.field("readiedAt").valid()) numReadied++; + else if (report.field("droppedAt").valid()) numDropped++; + else numInitial++; + } + + if (numInitial + numDropped > 0 && numNoReport + numReadied + numStarted > 0) + return ErrorResponse.conflict("Last dropping of documents may have failed to clear all documents due " + + "to concurrent topology changes, consider retrying"); + + Slime slime = new Slime(); + Cursor root = slime.setObject(); + if (numStarted + numNoReport == nodes.size()) { + if (readiedAt != null) root.setLong("lastDropped", readiedAt.toEpochMilli()); + } else { + Cursor progress = root.setObject("progress"); + progress.setLong("total", nodes.size()); + progress.setLong("dropped", numDropped); + progress.setLong("started", numStarted + numNoReport); + } + + return new SlimeJsonResponse(slime); + } + + private HttpResponse dropDocuments(String tenant, String application, String instance, String environment, String region, Optional<ClusterSpec.Id> clusterId) { + ZoneId zone = ZoneId.from(environment, region); + if (!zone.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Drop documents status is only available for manually deployed environments"); + + ApplicationId applicationId = ApplicationId.from(tenant, application, instance); + controller.serviceRegistry().configServer().nodeRepository().dropDocuments(zone, applicationId, clusterId); + return new MessageResponse("Triggered drop documents for " + applicationId.toFullString() + + clusterId.map(id -> " and cluster " + id).orElse("") + " in " + zone); + } + private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 297997365b0..7004028c072 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -88,6 +88,8 @@ public class NodeRepositoryMock implements NodeRepository { (node.owner().isPresent() && filter.applications().contains(node.owner().get()))) .filter(node -> filter.hostnames().isEmpty() || filter.hostnames().contains(node.hostname())) .filter(node -> filter.states().isEmpty() || filter.states().contains(node.state())) + .filter(node -> filter.clusterIds().isEmpty() || filter.clusterIds().contains(ClusterSpec.Id.from(node.clusterId()))) + .filter(node -> filter.clusterTypes().isEmpty() || filter.clusterTypes().contains(node.clusterType())) .toList(); } @@ -201,6 +203,10 @@ public class NodeRepositoryMock implements NodeRepository { } @Override + public void dropDocuments(ZoneId zoneId, ApplicationId applicationId, Optional<ClusterSpec.Id> clusterId) { + } + + @Override public void updateReports(ZoneId zone, String hostname, Map<String, String> reports) { Map<String, String> trimmedReports = reports.entrySet().stream() // Null value clears a report diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 76bcbe078ff..5b75d8cb914 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -40,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; @@ -55,6 +57,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -90,6 +93,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Supplier; import static com.yahoo.application.container.handler.Request.Method.DELETE; @@ -510,6 +514,42 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/bar/file.json?query=param", GET).userIdentity(USER_ID), "{\"path\":\"/bar/file.json\"}"); + // Drop documents + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/drop-documents", POST) + .userIdentity(USER_ID), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Drop documents status is only available for manually deployed environments\"}", 400); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", POST) + .userIdentity(USER_ID), + "{\"message\":\"Triggered drop documents for tenant2.application1.default in dev.us-east-1\"}"); + + ZoneId zone = ZoneId.from("dev", "us-east-1"); + ApplicationId application = ApplicationId.from("tenant2", "application1", "default"); + BiFunction<Integer, String, Node> nodeBuilder = (index, dropDocumentsReport) -> Node.builder().hostname("node" + index + ".dev.us-east-1.test") + .state(Node.State.active).type(NodeType.tenant).owner(application).clusterId("c1").clusterType(Node.ClusterType.content) + .reports(dropDocumentsReport == null ? Map.of() : Map.of("dropDocuments", dropDocumentsReport)).build(); + NodeRepositoryMock nodeRepository = deploymentTester.controllerTester().serviceRegistry().configServer().nodeRepository(); + + // 2 nodes, neither ever dropped any documents + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, null), nodeBuilder.apply(2, null))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{}"); + + // 1 node previously dropped documents, 1 node without any report + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, "{\"droppedAt\":1,\"readiedAt\":2,\"startedAt\":3}"), nodeBuilder.apply(2, null))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{\"lastDropped\":2}"); + + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, "{}"), nodeBuilder.apply(2, null))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{\"error-code\":\"CONFLICT\",\"message\":\"Last dropping of documents may have failed to clear all documents due to concurrent topology changes, consider retrying\"}", 409); + + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, "{}"), nodeBuilder.apply(2, "{\"droppedAt\":1}"))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{\"progress\":{\"total\":2,\"dropped\":1,\"started\":0}}"); + + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, "{\"startedAt\":3}"), nodeBuilder.apply(2, "{\"readiedAt\":1}"))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{\"progress\":{\"total\":2,\"dropped\":0,\"started\":1}}"); updateMetrics(); diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index dbea284e39e..73ad4b1d121 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -1828,8 +1828,6 @@ "public static final int MESSAGE_CREATEVISITOR", "public static final int MESSAGE_DESTROYVISITOR", "public static final int MESSAGE_VISITORINFO", - "public static final int MESSAGE_SEARCHRESULT", - "public static final int MESSAGE_DOCUMENTSUMMARY", "public static final int MESSAGE_MAPVISITOR", "public static final int MESSAGE_GETBUCKETSTATE", "public static final int MESSAGE_STATBUCKET", @@ -1846,8 +1844,6 @@ "public static final int REPLY_CREATEVISITOR", "public static final int REPLY_DESTROYVISITOR", "public static final int REPLY_VISITORINFO", - "public static final int REPLY_SEARCHRESULT", - "public static final int REPLY_DOCUMENTSUMMARY", "public static final int REPLY_MAPVISITOR", "public static final int REPLY_GETBUCKETSTATE", "public static final int REPLY_STATBUCKET", @@ -2093,21 +2089,6 @@ ], "fields" : [ ] }, - "com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.VisitorMessage", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "public void setDocumentSummary(com.yahoo.vdslib.DocumentSummary)", - "public com.yahoo.vdslib.DocumentSummary getResult()", - "public com.yahoo.documentapi.messagebus.protocol.DocumentReply createReply()", - "public int getType()" - ], - "fields" : [ ] - }, "com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig$Builder" : { "superClass" : "java.lang.Object", "interfaces" : [ @@ -2459,8 +2440,8 @@ "public int getType()", "public com.yahoo.document.TestAndSetCondition getCondition()", "public void setCondition(com.yahoo.document.TestAndSetCondition)", - "public boolean getCreateIfNonExistent()", - "public void setCreateIfNonExistent(boolean)" + "public void setCreateIfNonExistent(boolean)", + "public boolean getCreateIfNonExistent()" ], "fields" : [ ] }, @@ -2672,32 +2653,6 @@ ], "fields" : [ ] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentSummaryMessageFactory" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "protected com.yahoo.documentapi.messagebus.protocol.DocumentMessage doDecode(com.yahoo.document.serialization.DocumentDeserializer)", - "protected boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentMessage, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields" : [ ] - }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentSummaryReplyFactory" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "protected com.yahoo.documentapi.messagebus.protocol.DocumentReply doDecode(com.yahoo.document.serialization.DocumentDeserializer)", - "protected boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentReply, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields" : [ ] - }, "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$EmptyBucketsMessageFactory" : { "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces" : [ ], @@ -2936,32 +2891,6 @@ ], "fields" : [ ] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$SearchResultMessageFactory" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "protected com.yahoo.documentapi.messagebus.protocol.DocumentMessage doDecode(com.yahoo.document.serialization.DocumentDeserializer)", - "protected boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentMessage, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields" : [ ] - }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$SearchResultReplyFactory" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "protected com.yahoo.documentapi.messagebus.protocol.DocumentReply doDecode(com.yahoo.document.serialization.DocumentDeserializer)", - "protected boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentReply, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields" : [ ] - }, "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$StatBucketMessageFactory" : { "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces" : [ ], @@ -3095,21 +3024,6 @@ ], "fields" : [ ] }, - "com.yahoo.documentapi.messagebus.protocol.SearchResultMessage" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.VisitorMessage", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "public com.yahoo.vdslib.SearchResult getResult()", - "public void setSearchResult(com.yahoo.vdslib.SearchResult)", - "public com.yahoo.documentapi.messagebus.protocol.DocumentReply createReply()", - "public int getType()" - ], - "fields" : [ ] - }, "com.yahoo.documentapi.messagebus.protocol.SlobrokPolicy" : { "superClass" : "java.lang.Object", "interfaces" : [ diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 0ff578b64d7..6185437a48f 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -50,8 +50,9 @@ public class DocumentProtocol implements Protocol { public static final int MESSAGE_CREATEVISITOR = DOCUMENT_MESSAGE + 7; public static final int MESSAGE_DESTROYVISITOR = DOCUMENT_MESSAGE + 8; public static final int MESSAGE_VISITORINFO = DOCUMENT_MESSAGE + 9; - public static final int MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11; - public static final int MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14; + // SearchResult and DocumentSummary messages were replaced by QueryResult message in 2010. + // public static final int MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11; + // public static final int MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14; public static final int MESSAGE_MAPVISITOR = DOCUMENT_MESSAGE + 15; public static final int MESSAGE_GETBUCKETSTATE = DOCUMENT_MESSAGE + 18; public static final int MESSAGE_STATBUCKET = DOCUMENT_MESSAGE + 19; @@ -70,8 +71,9 @@ public class DocumentProtocol implements Protocol { public static final int REPLY_CREATEVISITOR = DOCUMENT_REPLY + 7; public static final int REPLY_DESTROYVISITOR = DOCUMENT_REPLY + 8; public static final int REPLY_VISITORINFO = DOCUMENT_REPLY + 9; - public static final int REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11; - public static final int REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14; + // SearchResult and DocumentSummary replies were replaced by QueryResult reply in 2010. + // public static final int REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11; + // public static final int REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14; public static final int REPLY_MAPVISITOR = DOCUMENT_REPLY + 15; public static final int REPLY_GETBUCKETSTATE = DOCUMENT_REPLY + 18; public static final int REPLY_STATBUCKET = DOCUMENT_REPLY + 19; @@ -282,7 +284,6 @@ public class DocumentProtocol implements Protocol { putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories60.CreateVisitorMessageFactory(), from6); putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories60.DestroyVisitorMessageFactory(), from6); putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories60.DocumentListMessageFactory(), from6); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories60.DocumentSummaryMessageFactory(), from6); putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories60.EmptyBucketsMessageFactory(), from6); putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories60.GetBucketListMessageFactory(), from6); putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories60.GetBucketStateMessageFactory(), from6); @@ -292,7 +293,6 @@ public class DocumentProtocol implements Protocol { putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories60.QueryResultMessageFactory(), from6); putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories60.RemoveDocumentMessageFactory(), from6); putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories60.RemoveLocationMessageFactory(), from6); - putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories60.SearchResultMessageFactory(), from6); putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories60.StatBucketMessageFactory(), from6); putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentMessageFactory(), from6); putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories60.VisitorInfoMessageFactory(), from6); @@ -300,7 +300,6 @@ public class DocumentProtocol implements Protocol { putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories60.DestroyVisitorReplyFactory(), from6); putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories60.DocumentIgnoredReplyFactory(), from6); putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories60.DocumentListReplyFactory(), from6); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories60.DocumentSummaryReplyFactory(), from6); putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories60.EmptyBucketsReplyFactory(), from6); putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories60.GetBucketListReplyFactory(), from6); putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories60.GetBucketStateReplyFactory(), from6); @@ -310,7 +309,6 @@ public class DocumentProtocol implements Protocol { putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories60.QueryResultReplyFactory(), from6); putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories60.RemoveDocumentReplyFactory(), from6); putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories60.RemoveLocationReplyFactory(), from6); - putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories60.SearchResultReplyFactory(), from6); putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories60.StatBucketReplyFactory(), from6); putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentReplyFactory(), from6); putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentReplyFactory(), from6); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java deleted file mode 100644 index 4866579a977..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.vdslib.DocumentSummary; - -public class DocumentSummaryMessage extends VisitorMessage { - - private DocumentSummary documentSummary = null; - - public void setDocumentSummary(DocumentSummary summary) { - documentSummary = summary; - } - - public DocumentSummary getResult() { - return documentSummary; - } - - @Override - public DocumentReply createReply() { - return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - - @Override - public int getType() { - return DocumentProtocol.MESSAGE_DOCUMENTSUMMARY; - } -} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index 9812f214066..3824da32d4e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -344,34 +344,6 @@ public abstract class RoutableFactories60 { } } - public static class DocumentSummaryMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - DocumentSummaryMessage msg = new DocumentSummaryMessage(); - msg.setDocumentSummary(new DocumentSummary(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - - public static class DocumentSummaryReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - public static class EmptyBucketsMessageFactory extends DocumentMessageFactory { @Override @@ -728,21 +700,6 @@ public abstract class RoutableFactories60 { } } - public static class SearchResultMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - SearchResultMessage msg = new SearchResultMessage(); - msg.setSearchResult(new SearchResult(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - public static class QueryResultMessageFactory extends DocumentMessageFactory { @Override @@ -759,19 +716,6 @@ public abstract class RoutableFactories60 { } } - public static class SearchResultReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - public static class QueryResultReplyFactory extends DocumentReplyFactory { @Override diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java deleted file mode 100644 index 570aafd49e6..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.vdslib.SearchResult; - -public class SearchResultMessage extends VisitorMessage { - - private SearchResult searchResult = null; - - public SearchResult getResult() { - return searchResult; - } - - public void setSearchResult(SearchResult result) { - searchResult = result; - } - - @Override - public DocumentReply createReply() { - return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); - } - - @Override - public int getType() { - return DocumentProtocol.MESSAGE_SEARCHRESULT; - } -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index 97cef53695a..22650fcdbf8 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -20,7 +20,6 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentReply; import com.yahoo.documentapi.messagebus.protocol.DocumentState; -import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage; import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; @@ -34,7 +33,6 @@ import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; -import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage; import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; @@ -69,7 +67,6 @@ public class Messages60TestCase extends MessagesTestBase { out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage()); out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage()); - out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage()); out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage()); out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage()); out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage()); @@ -79,7 +76,6 @@ public class Messages60TestCase extends MessagesTestBase { out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage()); out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage()); - out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage()); out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage()); out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage()); @@ -87,7 +83,6 @@ public class Messages60TestCase extends MessagesTestBase { out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply()); out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply()); out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply()); - out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply()); out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply()); out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply()); out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply()); @@ -97,7 +92,6 @@ public class Messages60TestCase extends MessagesTestBase { out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply()); out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply()); out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply()); - out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply()); out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply()); out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply()); out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply()); @@ -324,14 +318,6 @@ public class Messages60TestCase extends MessagesTestBase { } } - public class testDocumentSummaryReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - } - public class testEmptyBucketsReply implements RunnableTest { @Override @@ -392,65 +378,6 @@ public class Messages60TestCase extends MessagesTestBase { } } - public class testDocumentSummaryMessage implements RunnableTest { - - @Override - public void run() { - Routable routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - DocumentSummaryMessage msg = (DocumentSummaryMessage) routable; - assertEquals(0, msg.getResult().getSummaryCount()); - - routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - msg = (DocumentSummaryMessage) routable; - assertEquals(2, msg.getResult().getSummaryCount()); - com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0); - assertEquals("doc1", s.getDocId()); - byte[] b = s.getSummary(); - assertEquals(8, b.length); - byte[] c = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - - s = msg.getResult().getSummary(1); - assertEquals("aoc17", s.getDocId()); - b = s.getSummary(); - assertEquals(9, b.length); - byte[] d = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - msg = (DocumentSummaryMessage) routable; - assertEquals(2, msg.getResult().getSummaryCount()); - - s = msg.getResult().getSummary(0); - assertEquals("aoc17", s.getDocId()); - b = s.getSummary(); - assertEquals(9, b.length); - byte[] e = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - - s = msg.getResult().getSummary(1); - assertEquals("doc1", s.getDocId()); - b = s.getSummary(); - assertEquals(8, b.length); - byte[] f = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; - for (int i = 0; i < b.length; i++) { - assertEquals(f[i], b[i]); - } - } - } - - public class testGetDocumentMessage implements RunnableTest { @Override @@ -521,78 +448,6 @@ public class Messages60TestCase extends MessagesTestBase { } } - public class testSearchResultMessage implements RunnableTest { - - @Override - public void run() throws Exception { - Routable routable = deserialize("SearchResultMessage-1", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - SearchResultMessage msg = (SearchResultMessage)routable; - assertEquals(0, msg.getResult().getHitCount()); - - routable = deserialize("SearchResultMessage-2", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - - routable = deserialize("SearchResultMessage-3", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - - routable = deserialize("SearchResultMessage-4", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(3, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - h = msg.getResult().getHit(1); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - h = msg.getResult().getHit(2); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(90.0, h.getRank(), 1E-6); - assertEquals("doc18", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - } - } - private static String CONDITION_STRING = "There's just one condition"; public class testPutDocumentMessage implements RunnableTest { @@ -750,14 +605,6 @@ public class Messages60TestCase extends MessagesTestBase { } } - public class testSearchResultReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT); - } - } - public class testStatBucketReply implements RunnableTest { @Override diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index 93c7b5ef7cb..12cecefb072 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -39,7 +39,6 @@ Messages60Test::Messages60Test() putTest(DocumentProtocol::MESSAGE_CREATEVISITOR, TEST_METHOD(Messages60Test::testCreateVisitorMessage)); putTest(DocumentProtocol::MESSAGE_DESTROYVISITOR, TEST_METHOD(Messages60Test::testDestroyVisitorMessage)); putTest(DocumentProtocol::MESSAGE_DOCUMENTLIST, TEST_METHOD(Messages60Test::testDocumentListMessage)); - putTest(DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, TEST_METHOD(Messages60Test::testDocumentSummaryMessage)); putTest(DocumentProtocol::MESSAGE_EMPTYBUCKETS, TEST_METHOD(Messages60Test::testEmptyBucketsMessage)); putTest(DocumentProtocol::MESSAGE_GETBUCKETLIST, TEST_METHOD(Messages60Test::testGetBucketListMessage)); putTest(DocumentProtocol::MESSAGE_GETBUCKETSTATE, TEST_METHOD(Messages60Test::testGetBucketStateMessage)); @@ -49,7 +48,6 @@ Messages60Test::Messages60Test() putTest(DocumentProtocol::MESSAGE_QUERYRESULT, TEST_METHOD(Messages60Test::testQueryResultMessage)); putTest(DocumentProtocol::MESSAGE_REMOVEDOCUMENT, TEST_METHOD(Messages60Test::testRemoveDocumentMessage)); putTest(DocumentProtocol::MESSAGE_REMOVELOCATION, TEST_METHOD(Messages60Test::testRemoveLocationMessage)); - putTest(DocumentProtocol::MESSAGE_SEARCHRESULT, TEST_METHOD(Messages60Test::testSearchResultMessage)); putTest(DocumentProtocol::MESSAGE_STATBUCKET, TEST_METHOD(Messages60Test::testStatBucketMessage)); putTest(DocumentProtocol::MESSAGE_UPDATEDOCUMENT, TEST_METHOD(Messages60Test::testUpdateDocumentMessage)); putTest(DocumentProtocol::MESSAGE_VISITORINFO, TEST_METHOD(Messages60Test::testVisitorInfoMessage)); @@ -58,7 +56,6 @@ Messages60Test::Messages60Test() putTest(DocumentProtocol::REPLY_DESTROYVISITOR, TEST_METHOD(Messages60Test::testDestroyVisitorReply)); putTest(DocumentProtocol::REPLY_DOCUMENTIGNORED, TEST_METHOD(Messages60Test::testDocumentIgnoredReply)); putTest(DocumentProtocol::REPLY_DOCUMENTLIST, TEST_METHOD(Messages60Test::testDocumentListReply)); - putTest(DocumentProtocol::REPLY_DOCUMENTSUMMARY, TEST_METHOD(Messages60Test::testDocumentSummaryReply)); putTest(DocumentProtocol::REPLY_EMPTYBUCKETS, TEST_METHOD(Messages60Test::testEmptyBucketsReply)); putTest(DocumentProtocol::REPLY_GETBUCKETLIST, TEST_METHOD(Messages60Test::testGetBucketListReply)); putTest(DocumentProtocol::REPLY_GETBUCKETSTATE, TEST_METHOD(Messages60Test::testGetBucketStateReply)); @@ -68,7 +65,6 @@ Messages60Test::Messages60Test() putTest(DocumentProtocol::REPLY_QUERYRESULT, TEST_METHOD(Messages60Test::testQueryResultReply)); putTest(DocumentProtocol::REPLY_REMOVEDOCUMENT, TEST_METHOD(Messages60Test::testRemoveDocumentReply)); putTest(DocumentProtocol::REPLY_REMOVELOCATION, TEST_METHOD(Messages60Test::testRemoveLocationReply)); - putTest(DocumentProtocol::REPLY_SEARCHRESULT, TEST_METHOD(Messages60Test::testSearchResultReply)); putTest(DocumentProtocol::REPLY_STATBUCKET, TEST_METHOD(Messages60Test::testStatBucketReply)); putTest(DocumentProtocol::REPLY_UPDATEDOCUMENT, TEST_METHOD(Messages60Test::testUpdateDocumentReply)); putTest(DocumentProtocol::REPLY_VISITORINFO, TEST_METHOD(Messages60Test::testVisitorInfoReply)); @@ -269,65 +265,6 @@ Messages60Test::testRemoveLocationMessage() bool -Messages60Test::testDocumentSummaryMessage() -{ - DocumentSummaryMessage srm; - EXPECT_EQUAL(srm.hasSequenceId(), false); - EXPECT_EQUAL(srm.getSummaryCount(), size_t(0)); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), serialize("DocumentSummaryMessage-1", srm)); - - mbus::Routable::UP routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - DocumentSummaryMessage * dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(0)); - - srm.addSummary("doc1", "summary1", 8); - srm.addSummary("aoc17", "summary45", 9); - - const void *summary(NULL); - const char *docId(NULL); - size_t sz(0); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-2", srm)); - routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); - dm->getSummary(0, docId, summary, sz); - EXPECT_EQUAL(sz, 8u); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); - dm->getSummary(1, docId, summary, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(strcmp("aoc17", docId), 0); - EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); - - srm.sort(); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-3", srm)); - routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); - dm->getSummary(0, docId, summary, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(strcmp("aoc17", docId), 0); - EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); - dm->getSummary(1, docId, summary, sz); - EXPECT_EQUAL(sz, 8u); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); - return true; -} - -bool Messages60Test::testGetDocumentMessage() { GetDocumentMessage tmp(document::DocumentId("id:ns:testdoc::"), "foo bar"); @@ -561,133 +498,6 @@ Messages60Test::testRemoveDocumentReply() } bool -Messages60Test::testSearchResultMessage() -{ - SearchResultMessage srm; - EXPECT_EQUAL(srm.getSequenceId(), 0u); - EXPECT_EQUAL(srm.getHitCount(), 0u); - EXPECT_EQUAL(srm.getAggregatorList().getSerializedSize(), 4u); - EXPECT_EQUAL(srm.vdslib::SearchResult::getSerializedSize(), 20u); - EXPECT_EQUAL(srm.getSerializedSize(), 20u); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), serialize("SearchResultMessage-1", srm)); - - mbus::Routable::UP routable = deserialize("SearchResultMessage-1", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - SearchResultMessage * dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); - EXPECT_EQUAL(dm->getHitCount(), size_t(0)); - - srm.addHit(0, "doc1", 89); - srm.addHit(1, "doc17", 109); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-2", srm)); - routable = deserialize("SearchResultMessage-2", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(2)); - const char *docId; - SearchResultMessage::RankType rank; - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - - srm.sort(); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-3", srm)); - routable = deserialize("SearchResultMessage-3", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(2)); - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - - SearchResultMessage srm2; - srm2.addHit(0, "doc1", 89, "sortdata2", 9); - srm2.addHit(1, "doc17", 109, "sortdata1", 9); - srm2.addHit(2, "doc18", 90, "sortdata3", 9); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-4", srm2)); - routable = deserialize("SearchResultMessage-4", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(3)); - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(2, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - - srm2.sort(); - const void *buf; - size_t sz; - srm2.getHit(0, docId, rank); - srm2.getSortBlob(0, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - srm2.getHit(1, docId, rank); - srm2.getSortBlob(1, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - srm2.getHit(2, docId, rank); - srm2.getSortBlob(2, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-5", srm2)); - routable = deserialize("SearchResultMessage-5", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(3)); - dm->getHit(0, docId, rank); - dm->getSortBlob(0, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(1, docId, rank); - dm->getSortBlob(1, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(2, docId, rank); - dm->getSortBlob(2, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - return true; -} - -bool Messages60Test::testUpdateDocumentMessage() { const DocumentTypeRepo & repo = getTypeRepo(); @@ -913,12 +723,6 @@ Messages60Test::testDocumentListReply() } bool -Messages60Test::testDocumentSummaryReply() -{ - return tryVisitorReply("DocumentSummaryReply", DocumentProtocol::REPLY_DOCUMENTSUMMARY); -} - -bool Messages60Test::testGetDocumentReply() { document::Document::SP doc = @@ -947,12 +751,6 @@ Messages60Test::testMapVisitorReply() } bool -Messages60Test::testSearchResultReply() -{ - return tryVisitorReply("SearchResultReply", DocumentProtocol::REPLY_SEARCHRESULT); -} - -bool Messages60Test::testStatBucketReply() { StatBucketReply msg; diff --git a/documentapi/src/tests/messages/messages60test.h b/documentapi/src/tests/messages/messages60test.h index 1eb3e8e248f..4a2a3f98fad 100644 --- a/documentapi/src/tests/messages/messages60test.h +++ b/documentapi/src/tests/messages/messages60test.h @@ -24,7 +24,6 @@ public: bool testDocumentListMessage(); bool testDocumentListReply(); bool testDocumentSummaryMessage(); - bool testDocumentSummaryReply(); bool testEmptyBucketsMessage(); bool testEmptyBucketsReply(); bool testGetBucketListMessage(); @@ -44,7 +43,6 @@ public: bool testRemoveLocationMessage(); bool testRemoveLocationReply(); bool testSearchResultMessage(); - bool testSearchResultReply(); bool testStatBucketMessage(); bool testStatBucketReply(); bool testUpdateDocumentMessage(); diff --git a/documentapi/src/vespa/documentapi/documentapi.h b/documentapi/src/vespa/documentapi/documentapi.h index 311199c94cd..b784a63a642 100644 --- a/documentapi/src/vespa/documentapi/documentapi.h +++ b/documentapi/src/vespa/documentapi/documentapi.h @@ -11,9 +11,7 @@ #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> #include <vespa/documentapi/messagebus/messages/feedreply.h> #include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> -#include <vespa/documentapi/messagebus/messages/searchresultmessage.h> #include <vespa/documentapi/messagebus/messages/visitor.h> -#include <vespa/documentapi/messagebus/messages/documentsummarymessage.h> #include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> #include <vespa/documentapi/messagebus/messages/getbucketlistmessage.h> #include <vespa/documentapi/messagebus/messages/getbucketlistreply.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index d751d11177c..1eb50bba714 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -49,7 +49,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, putRoutableFactory(MESSAGE_CREATEVISITOR, std::make_shared<RoutableFactories60::CreateVisitorMessageFactory>(), from6); putRoutableFactory(MESSAGE_DESTROYVISITOR, std::make_shared<RoutableFactories60::DestroyVisitorMessageFactory>(), from6); putRoutableFactory(MESSAGE_DOCUMENTLIST, std::make_shared<RoutableFactories60::DocumentListMessageFactory>(*_repo), from6); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, std::make_shared<RoutableFactories60::DocumentSummaryMessageFactory>(), from6); putRoutableFactory(MESSAGE_EMPTYBUCKETS, std::make_shared<RoutableFactories60::EmptyBucketsMessageFactory>(), from6); putRoutableFactory(MESSAGE_GETBUCKETLIST, std::make_shared<RoutableFactories60::GetBucketListMessageFactory>(), from6); putRoutableFactory(MESSAGE_GETBUCKETSTATE, std::make_shared<RoutableFactories60::GetBucketStateMessageFactory>(), from6); @@ -59,7 +58,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, putRoutableFactory(MESSAGE_QUERYRESULT, std::make_shared<RoutableFactories60::QueryResultMessageFactory>(), from6); putRoutableFactory(MESSAGE_REMOVEDOCUMENT, std::make_shared<RoutableFactories60::RemoveDocumentMessageFactory>(), from6); putRoutableFactory(MESSAGE_REMOVELOCATION, std::make_shared<RoutableFactories60::RemoveLocationMessageFactory>(*_repo), from6); - putRoutableFactory(MESSAGE_SEARCHRESULT, std::make_shared<RoutableFactories60::SearchResultMessageFactory>(), from6); putRoutableFactory(MESSAGE_STATBUCKET, std::make_shared<RoutableFactories60::StatBucketMessageFactory>(), from6); putRoutableFactory(MESSAGE_UPDATEDOCUMENT, std::make_shared<RoutableFactories60::UpdateDocumentMessageFactory>(*_repo), from6); putRoutableFactory(MESSAGE_VISITORINFO, std::make_shared<RoutableFactories60::VisitorInfoMessageFactory>(), from6); @@ -67,7 +65,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, putRoutableFactory(REPLY_DESTROYVISITOR, std::make_shared<RoutableFactories60::DestroyVisitorReplyFactory>(), from6); putRoutableFactory(REPLY_DOCUMENTIGNORED, std::make_shared<RoutableFactories60::DocumentIgnoredReplyFactory>(), from6); putRoutableFactory(REPLY_DOCUMENTLIST, std::make_shared<RoutableFactories60::DocumentListReplyFactory>(), from6); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, std::make_shared<RoutableFactories60::DocumentSummaryReplyFactory>(), from6); putRoutableFactory(REPLY_EMPTYBUCKETS, std::make_shared<RoutableFactories60::EmptyBucketsReplyFactory>(), from6); putRoutableFactory(REPLY_GETBUCKETLIST, std::make_shared<RoutableFactories60::GetBucketListReplyFactory>(), from6); putRoutableFactory(REPLY_GETBUCKETSTATE, std::make_shared<RoutableFactories60::GetBucketStateReplyFactory>(), from6); @@ -77,7 +74,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, putRoutableFactory(REPLY_QUERYRESULT, std::make_shared<RoutableFactories60::QueryResultReplyFactory>(), from6); putRoutableFactory(REPLY_REMOVEDOCUMENT, std::make_shared<RoutableFactories60::RemoveDocumentReplyFactory>(), from6); putRoutableFactory(REPLY_REMOVELOCATION, std::make_shared<RoutableFactories60::RemoveLocationReplyFactory>(), from6); - putRoutableFactory(REPLY_SEARCHRESULT, std::make_shared<RoutableFactories60::SearchResultReplyFactory>(), from6); putRoutableFactory(REPLY_STATBUCKET, std::make_shared<RoutableFactories60::StatBucketReplyFactory>(), from6); putRoutableFactory(REPLY_UPDATEDOCUMENT, std::make_shared<RoutableFactories60::UpdateDocumentReplyFactory>(), from6); putRoutableFactory(REPLY_VISITORINFO, std::make_shared<RoutableFactories60::VisitorInfoReplyFactory>(), from6); diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index 9f0d7253335..d91d355c567 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -55,9 +55,10 @@ public: MESSAGE_CREATEVISITOR = DOCUMENT_MESSAGE + 7, MESSAGE_DESTROYVISITOR = DOCUMENT_MESSAGE + 8, MESSAGE_VISITORINFO = DOCUMENT_MESSAGE + 9, - MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11, + // SearchResult and DocumentSummary messages were replaced by QueryResult message in 2010. + // MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11, //MESSAGE_MULTIOPERATION = DOCUMENT_MESSAGE + 13, - MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14, + // MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14, MESSAGE_MAPVISITOR = DOCUMENT_MESSAGE + 15, MESSAGE_GETBUCKETSTATE = DOCUMENT_MESSAGE + 18, MESSAGE_STATBUCKET = DOCUMENT_MESSAGE + 19, @@ -78,9 +79,10 @@ public: REPLY_CREATEVISITOR = DOCUMENT_REPLY + 7, REPLY_DESTROYVISITOR = DOCUMENT_REPLY + 8, REPLY_VISITORINFO = DOCUMENT_REPLY + 9, - REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11, + // SearchResult and DocumentSummary replies were replaced by QueryResult reply in 2010. + // REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11, //REPLY_MULTIOPERATION = DOCUMENT_REPLY + 13, - REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14, + // REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14, REPLY_MAPVISITOR = DOCUMENT_REPLY + 15, REPLY_GETBUCKETSTATE = DOCUMENT_REPLY + 18, REPLY_STATBUCKET = DOCUMENT_REPLY + 19, diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt index ddbe66fc22a..d906166b6df 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(documentapi_documentapimessages OBJECT documentmessage.cpp documentreply.cpp documentstate.cpp - documentsummarymessage.cpp emptybucketsmessage.cpp feedanswer.cpp feedmessage.cpp @@ -21,7 +20,6 @@ vespa_add_library(documentapi_documentapimessages OBJECT removedocumentmessage.cpp removedocumentreply.cpp removelocationmessage.cpp - searchresultmessage.cpp statbucketmessage.cpp statbucketreply.cpp testandsetmessage.cpp diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp deleted file mode 100644 index 6be241b2e1d..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "documentsummarymessage.h" - -using vdslib::DocumentSummary; - -namespace documentapi { - -DocumentSummaryMessage::DocumentSummaryMessage(const DocumentSummary & sr) : - VisitorMessage(), - DocumentSummary(sr) -{ - // empty -} - -DocumentSummaryMessage::DocumentSummaryMessage() : - VisitorMessage(), - DocumentSummary() -{ - // empty -} - -DocumentReply::UP -DocumentSummaryMessage::doCreateReply() const -{ - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTSUMMARY)); -} - -uint32_t -DocumentSummaryMessage::getApproxSize() const -{ - return DocumentSummary::getSerializedSize(); -} - -uint32_t -DocumentSummaryMessage::getType() const -{ - return DocumentProtocol::MESSAGE_DOCUMENTSUMMARY; -} - -} - diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h deleted file mode 100644 index 2c8149ab058..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include <vespa/vdslib/container/documentsummary.h> - -namespace documentapi { - -class DocumentSummaryMessage : public VisitorMessage, - public vdslib::DocumentSummary { -protected: - // Implements VisitorMessage. - DocumentReply::UP doCreateReply() const override; - -public: - /** - * Convenience typedef. - */ - using UP = std::unique_ptr<DocumentSummaryMessage>; - using SP = std::shared_ptr<DocumentSummaryMessage>; - - /** - * Constructs a new document message with no content. - */ - DocumentSummaryMessage(); - - /** - * Constructs a new document message with summary comment. - * - * @param summary The document summary to contain. - */ - DocumentSummaryMessage(const vdslib::DocumentSummary &summary); - uint32_t getApproxSize() const override; - uint32_t getType() const override; - string toString() const override { return "documentsummarymessage"; } -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp deleted file mode 100644 index 8e25e70d749..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "searchresultmessage.h" - -using vdslib::SearchResult; - -namespace documentapi { - -SearchResultMessage::SearchResultMessage() : - VisitorMessage(), - SearchResult() -{ - // empty -} - -SearchResultMessage::SearchResultMessage(SearchResult &&result) : - VisitorMessage(), - SearchResult(std::move(result)) -{ - // empty -} - -DocumentReply::UP -SearchResultMessage::doCreateReply() const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); -} - -uint32_t -SearchResultMessage::getApproxSize() const -{ - return SearchResult::getSerializedSize(); -} - -uint32_t -SearchResultMessage::getType() const -{ - return DocumentProtocol::MESSAGE_SEARCHRESULT; -} - -} - diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h deleted file mode 100644 index d22f8197534..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include <vespa/vdslib/container/searchresult.h> - -namespace documentapi { - -class SearchResultMessage : public VisitorMessage, - public vdslib::SearchResult { -protected: - DocumentReply::UP doCreateReply() const override; - -public: - using UP = std::unique_ptr<SearchResultMessage>; - using SP = std::shared_ptr<SearchResultMessage>; - - SearchResultMessage(); - SearchResultMessage(vdslib::SearchResult &&result); - - uint32_t getApproxSize() const override; - uint32_t getType() const override; - string toString() const override { return "searchresultmessage"; } -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 1dce7ff281f..508bb25c907 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -258,37 +258,6 @@ RoutableFactories60::DocumentListReplyFactory::doEncode(const DocumentReply &, v } DocumentMessage::UP -RoutableFactories60::DocumentSummaryMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<DocumentSummaryMessage>(); - - msg->deserialize(buf); - - return msg; -} - -bool -RoutableFactories60::DocumentSummaryMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const DocumentSummaryMessage &msg = static_cast<const DocumentSummaryMessage&>(obj); - msg.serialize(buf); - - return true; -} - -DocumentReply::UP -RoutableFactories60::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTSUMMARY); -} - -bool -RoutableFactories60::DocumentSummaryReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentMessage::UP RoutableFactories60::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer &buf) const { auto msg = std::make_unique<EmptyBucketsMessage>(); @@ -642,23 +611,6 @@ RoutableFactories60::RemoveLocationReplyFactory::doEncode(const DocumentReply &, } DocumentMessage::UP -RoutableFactories60::SearchResultMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<SearchResultMessage>(); - msg->deserialize(buf); - return msg; -} - -bool -RoutableFactories60::SearchResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const auto & msg = static_cast<const SearchResultMessage&>(obj); - msg.serialize(buf); - - return true; -} - -DocumentMessage::UP RoutableFactories60::QueryResultMessageFactory::doDecode(document::ByteBuffer &buf) const { auto msg = std::make_unique<QueryResultMessage>(); @@ -680,18 +632,6 @@ RoutableFactories60::QueryResultMessageFactory::doEncode(const DocumentMessage & } DocumentReply::UP -RoutableFactories60::SearchResultReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); -} - -bool -RoutableFactories60::SearchResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentReply::UP RoutableFactories60::QueryResultReplyFactory::doDecode(document::ByteBuffer &) const { return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_QUERYRESULT); diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h index b618d92a145..c0cbc4868eb 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h @@ -172,16 +172,6 @@ public: DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; }; - class DocumentSummaryMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class DocumentSummaryReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; class EmptyBucketsMessageFactory : public DocumentMessageFactory { protected: DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; @@ -282,16 +272,6 @@ public: DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; }; - class SearchResultMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class SearchResultReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; class StatBucketMessageFactory : public DocumentMessageFactory { virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; virtual string decodeBucketSpace(document::ByteBuffer&) const; diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat Binary files differdeleted file mode 100644 index 0107dd5f350..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat Binary files differdeleted file mode 100644 index 57187093f28..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat Binary files differdeleted file mode 100644 index 6a516d38d17..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat Binary files differdeleted file mode 100644 index 988f9fdab1f..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat Binary files differdeleted file mode 100644 index ac277d09643..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat Binary files differdeleted file mode 100644 index 03b49c8a0ac..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat Binary files differdeleted file mode 100644 index d52e574ea44..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat Binary files differdeleted file mode 100644 index e68654e9941..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/6.221-java-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-java-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/6.221-java-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-java-SearchResultReply.dat +++ /dev/null diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index ea17d9967ae..f90d2855478 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -402,7 +402,7 @@ public class Flags { "allow-more-than-one-content-group-down", false, List.of("hmusum"), "2023-04-14", "2023-06-14", "Whether to enable possible configuration of letting more than one content group down", "Takes effect at redeployment", - HOSTNAME); + ZONE_ID, APPLICATION_ID); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java deleted file mode 100644 index b98ad7a11bc..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; - -import static com.yahoo.vespa.hosted.node.admin.container.ContainerStatsCollector.userHzToMicroSeconds; - -/** - * Read and write interface to the CGroup of a podman container. - * - * @author freva - */ -public interface CGroup { - - /** - * Returns quota and period values used for CPU scheduling. This serves as hard cap on CPU usage by allowing - * the CGroup to use up to {@code quota} each {@code period}. If uncapped, quota will be negative. - * - * @param containerId full container ID. - * @return CPU quota and period for the given container. Empty if CGroup for this container is not found. - */ - Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId); - - /** @return number of shares allocated to this CGroup for purposes of CPU time scheduling, empty if CGroup not found */ - OptionalInt cpuShares(ContainerId containerId); - - /** Update CPU quota and period for the given container ID, set quota to -1 value for unlimited */ - boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs); - - boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares); - - Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException; - - /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ - long memoryLimitInBytes(ContainerId containerId) throws IOException; - - /** @return The total amount of memory currently being used by the cgroup and its descendants. */ - long memoryUsageInBytes(ContainerId containerId) throws IOException; - - /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ - long memoryCacheInBytes(ContainerId containerId) throws IOException; - - enum CpuStatField { - TOTAL_USAGE_USEC(null/* in a dedicated file */, "usage_usec"), - USER_USAGE_USEC("user", "user_usec"), - SYSTEM_USAGE_USEC("system", "system_usec"), - TOTAL_PERIODS("nr_periods", "nr_periods"), - THROTTLED_PERIODS("nr_throttled", "nr_throttled"), - THROTTLED_TIME_USEC("throttled_time", "throttled_usec"); - - private final String v1Name; - private final String v2Name; - CpuStatField(String v1Name, String v2Name) { - this.v1Name = v1Name; - this.v2Name = v2Name; - } - - long parseValueV1(String value) { - long longValue = Long.parseLong(value); - return switch (this) { - case THROTTLED_TIME_USEC, TOTAL_USAGE_USEC -> longValue / 1000; // Value in ns - case USER_USAGE_USEC, SYSTEM_USAGE_USEC -> userHzToMicroSeconds(longValue); - default -> longValue; - }; - } - - long parseValueV2(String value) { - return Long.parseLong(value); - } - - static Optional<CpuStatField> fromV1Field(String name) { - return Arrays.stream(values()) - .filter(field -> name.equals(field.v1Name)) - .findFirst(); - } - - static Optional<CpuStatField> fromV2Field(String name) { - return Arrays.stream(values()) - .filter(field -> name.equals(field.v2Name)) - .findFirst(); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java deleted file mode 100644 index 7607858ec85..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.logging.Logger; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.parseLong; - -/** - * Read and write interface to the CGroup V1 of a Podman container. - * - * @see <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/index.html">CGroups V1</a> - * @author freva - */ -public class CGroupV1 implements CGroup { - - private static final Logger logger = Logger.getLogger(CGroupV1.class.getName()); - - private final FileSystem fileSystem; - - public CGroupV1(FileSystem fileSystem) { - this.fileSystem = fileSystem; - } - - @Override - public Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId) { - OptionalInt quota = readCgroupsCpuInt(cfsQuotaPath(containerId)); - if (quota.isEmpty()) return Optional.empty(); - OptionalInt period = readCgroupsCpuInt(cfsPeriodPath(containerId)); - if (period.isEmpty()) return Optional.empty(); - return Optional.of(new Pair<>(quota.getAsInt(), period.getAsInt())); - } - - @Override - public OptionalInt cpuShares(ContainerId containerId) { - return readCgroupsCpuInt(sharesPath(containerId)); - } - - @Override - public boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs) { - return writeCgroupsCpuInt(context, cfsQuotaPath(containerId), cpuQuotaUs) | - writeCgroupsCpuInt(context, cfsPeriodPath(containerId), periodUs); - } - - @Override - public boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares) { - return writeCgroupsCpuInt(context, sharesPath(containerId), shares); - } - - @Override - public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { - Map<CpuStatField, Long> stats = new HashMap<>(); - stats.put(CpuStatField.TOTAL_USAGE_USEC, parseLong(cpuacctPath(containerId).resolve("cpuacct.usage")) / 1000); - Stream.concat(Files.readAllLines(cpuacctPath(containerId).resolve("cpuacct.stat")).stream(), - Files.readAllLines(cpuacctPath(containerId).resolve("cpu.stat")).stream()) - .forEach(line -> { - String[] parts = line.split("\\s+"); - if (parts.length != 2) return; - CpuStatField.fromV1Field(parts[0]).ifPresent(field -> stats.put(field, field.parseValueV1(parts[1]))); - }); - return stats; - } - - @Override - public long memoryLimitInBytes(ContainerId containerId) throws IOException { - return parseLong(memoryPath(containerId).resolve("memory.limit_in_bytes")); - } - - @Override - public long memoryUsageInBytes(ContainerId containerId) throws IOException { - return parseLong(memoryPath(containerId).resolve("memory.usage_in_bytes")); - } - - @Override - public long memoryCacheInBytes(ContainerId containerId) throws IOException { - return parseLong(memoryPath(containerId).resolve("memory.stat"), "cache"); - } - - private Path cpuacctPath(ContainerId containerId) { - return fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-" + containerId + ".scope"); - } - - private Path cpuPath(ContainerId containerId) { - return fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-" + containerId + ".scope"); - } - - private Path memoryPath(ContainerId containerId) { - return fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-" + containerId + ".scope"); - } - - private UnixPath cfsQuotaPath(ContainerId containerId) { - return new UnixPath(cpuPath(containerId).resolve("cpu.cfs_quota_us")); - } - - private UnixPath cfsPeriodPath(ContainerId containerId) { - return new UnixPath(cpuPath(containerId).resolve("cpu.cfs_period_us")); - } - - private UnixPath sharesPath(ContainerId containerId) { - return new UnixPath(cpuPath(containerId).resolve("cpu.shares")); - } - - private static OptionalInt readCgroupsCpuInt(UnixPath unixPath) { - return unixPath.readUtf8FileIfExists() - .map(s -> OptionalInt.of(Integer.parseInt(s.strip()))) - .orElseGet(OptionalInt::empty); - } - - private static boolean writeCgroupsCpuInt(NodeAgentContext context, UnixPath unixPath, int value) { - int currentValue = readCgroupsCpuInt(unixPath).orElseThrow(); - if (currentValue == value) return false; - - context.recordSystemModification(logger, "Updating " + unixPath + " from " + currentValue + " to " + value); - unixPath.writeUtf8File(Integer.toString(value)); - return true; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java index 0c86829b96d..fef7f4bbd4c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -17,15 +18,16 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Read and write interface to the CGroup V2 of a Podman container. + * Read and write interface to the cgroup v2 of a Podman container. * * @see <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">CGroups V2</a> * @author freva */ -public class CGroupV2 implements CGroup { +public class CGroupV2 { private static final Logger logger = Logger.getLogger(CGroupV2.class.getName()); private static final String MAX = "max"; + public static final String VESPA_CGEXEC_PATH = "/opt/vespa/bin/vespa-cgexec"; private final FileSystem fileSystem; @@ -33,7 +35,13 @@ public class CGroupV2 implements CGroup { this.fileSystem = fileSystem; } - @Override + /** + * Returns quota and period values used for CPU scheduling. This serves as hard cap on CPU usage by allowing + * the CGroupV2 to use up to {@code quota} each {@code period}. If uncapped, quota will be negative. + * + * @param containerId full container ID. + * @return CPU quota and period for the given container. Empty if CGroupV2 for this container is not found. + */ public Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId) { return cpuMaxPath(containerId).readUtf8FileIfExists() .map(s -> { @@ -42,45 +50,68 @@ public class CGroupV2 implements CGroup { }); } - @Override + /** @return number of shares allocated to this CGroupV2 for purposes of CPU time scheduling, empty if CGroupV2 not found */ public OptionalInt cpuShares(ContainerId containerId) { return cpuWeightPath(containerId).readUtf8FileIfExists() .map(s -> OptionalInt.of(weightToShares(Integer.parseInt(s.strip())))) .orElseGet(OptionalInt::empty); } - @Override + /** Update CPU quota and period for the given container ID, set quota to -1 value for unlimited */ public boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs) { String wanted = String.format("%s %d", cpuQuotaUs < 0 ? MAX : cpuQuotaUs, periodUs); return writeCGroupsValue(context, cpuMaxPath(containerId), wanted); } - @Override public boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares) { return writeCGroupsValue(context, cpuWeightPath(containerId), Integer.toString(sharesToWeight(shares))); } - @Override + enum CpuStatField { + TOTAL_USAGE_USEC("usage_usec"), + USER_USAGE_USEC("user_usec"), + SYSTEM_USAGE_USEC("system_usec"), + TOTAL_PERIODS("nr_periods"), + THROTTLED_PERIODS("nr_throttled"), + THROTTLED_TIME_USEC("throttled_usec"); + + private final String name; + + CpuStatField(String name) { + this.name = name; + } + + long parseValue(String value) { + return Long.parseLong(value); + } + + static Optional<CpuStatField> fromField(String fieldName) { + return Arrays.stream(values()) + .filter(field -> fieldName.equals(field.name)) + .findFirst(); + } + } + public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { return Files.readAllLines(cgroupRoot(containerId).resolve("cpu.stat")).stream() .map(line -> line.split("\\s+")) .filter(parts -> parts.length == 2) - .flatMap(parts -> CpuStatField.fromV2Field(parts[0]).stream().map(field -> new Pair<>(field, field.parseValueV2(parts[1])))) + .flatMap(parts -> CpuStatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); } - @Override + /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ public long memoryLimitInBytes(ContainerId containerId) throws IOException { String limit = Files.readString(cgroupRoot(containerId).resolve("memory.max")).strip(); return MAX.equals(limit) ? -1L : Long.parseLong(limit); } - @Override + /** @return The total amount of memory currently being used by the cgroup and its descendants. */ public long memoryUsageInBytes(ContainerId containerId) throws IOException { return parseLong(cgroupRoot(containerId).resolve("memory.current")); } - @Override + /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ public long memoryCacheInBytes(ContainerId containerId) throws IOException { return parseLong(cgroupRoot(containerId).resolve("memory.stat"), "file"); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java index fb789874acf..e76a46b1c3b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java @@ -51,6 +51,16 @@ public class Container extends PartialContainer { } @Override + public String toString() { + return "Container{" + + "hostname='" + hostname + '\'' + + ", resources=" + resources + + ", conmonPid=" + conmonPid + + ", networks=" + networks + + '}'; + } + + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index f131aca2db0..ce2a6bb22ac 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -33,7 +33,7 @@ public class ContainerOperations { private final ContainerImagePruner imagePruner; private final ContainerStatsCollector containerStatsCollector; - public ContainerOperations(ContainerEngine containerEngine, CGroup cgroup, FileSystem fileSystem) { + public ContainerOperations(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem) { this.containerEngine = Objects.requireNonNull(containerEngine); this.imageDownloader = new ContainerImageDownloader(containerEngine); this.imagePruner = new ContainerImagePruner(containerEngine, Clock.systemUTC()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java index c17f98b9c9d..870809123a9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java @@ -27,15 +27,15 @@ import java.util.stream.Stream; class ContainerStatsCollector { private final ContainerEngine containerEngine; - private final CGroup cgroup; + private final CGroupV2 cgroup; private final FileSystem fileSystem; private final int onlineCpus; - ContainerStatsCollector(ContainerEngine containerEngine, CGroup cgroup, FileSystem fileSystem) { + ContainerStatsCollector(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem) { this(containerEngine, cgroup, fileSystem, Runtime.getRuntime().availableProcessors()); } - ContainerStatsCollector(ContainerEngine containerEngine, CGroup cgroup, FileSystem fileSystem, int onlineCpus) { + ContainerStatsCollector(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem, int onlineCpus) { this.containerEngine = Objects.requireNonNull(containerEngine); this.cgroup = Objects.requireNonNull(cgroup); this.fileSystem = Objects.requireNonNull(fileSystem); @@ -83,14 +83,14 @@ class ContainerStatsCollector { } private ContainerStats.CpuStats collectCpuStats(ContainerId containerId) throws IOException { - Map<CGroup.CpuStatField, Long> cpuStats = cgroup.cpuStats(containerId); + Map<CGroupV2.CpuStatField, Long> cpuStats = cgroup.cpuStats(containerId); return new ContainerStats.CpuStats(onlineCpus, systemCpuUsage(), - cpuStats.get(CGroup.CpuStatField.TOTAL_USAGE_USEC), - cpuStats.get(CGroup.CpuStatField.SYSTEM_USAGE_USEC), - cpuStats.get(CGroup.CpuStatField.THROTTLED_TIME_USEC), - cpuStats.get(CGroup.CpuStatField.TOTAL_PERIODS), - cpuStats.get(CGroup.CpuStatField.THROTTLED_PERIODS)); + cpuStats.get(CGroupV2.CpuStatField.TOTAL_USAGE_USEC), + cpuStats.get(CGroupV2.CpuStatField.SYSTEM_USAGE_USEC), + cpuStats.get(CGroupV2.CpuStatField.THROTTLED_TIME_USEC), + cpuStats.get(CGroupV2.CpuStatField.TOTAL_PERIODS), + cpuStats.get(CGroupV2.CpuStatField.THROTTLED_PERIODS)); } private ContainerStats.MemoryStats collectMemoryStats(ContainerId containerId) throws IOException { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java deleted file mode 100644 index f25001d77cd..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; - -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - */ -public class CGroupV1Test { - - private static final ContainerId containerId = new ContainerId("4aec78cc"); - - private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroup cgroup = new CGroupV1(fileSystem); - private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); - - @Test - public void updates_cpu_quota_and_period() { - assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); - - UnixPath cpu = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-4aec78cc.scope")).createDirectories(); - cpu.resolve("cpu.cfs_period_us").writeUtf8File("123456\n"); - cpu.resolve("cpu.cfs_quota_us").writeUtf8File("-1\n"); - assertEquals(Optional.of(new Pair<>(-1, 123456)), cgroup.cpuQuotaPeriod(containerId)); - - cpu.resolve("cpu.cfs_quota_us").writeUtf8File("456\n"); - assertEquals(Optional.of(new Pair<>(456, 123456)), cgroup.cpuQuotaPeriod(containerId)); - - assertFalse(cgroup.updateCpuQuotaPeriod(context, containerId, 456, 123456)); - - assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, 654, 123456)); - assertEquals(Optional.of(new Pair<>(654, 123456)), cgroup.cpuQuotaPeriod(containerId)); - } - - @Test - public void updates_cpu_shares() { - assertEquals(OptionalInt.empty(), cgroup.cpuShares(containerId)); - - UnixPath cpuPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-4aec78cc.scope")).createDirectories(); - cpuPath.resolve("cpu.shares").writeUtf8File("987\n"); - assertEquals(OptionalInt.of(987), cgroup.cpuShares(containerId)); - - assertFalse(cgroup.updateCpuShares(context, containerId, 987)); - - assertTrue(cgroup.updateCpuShares(context, containerId, 789)); - assertEquals(OptionalInt.of(789), cgroup.cpuShares(containerId)); - } - - @Test - public void reads_cpu_stats() throws IOException { - UnixPath cpuacctPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-4aec78cc.scope")).createDirectories(); - cpuacctPath.resolve("cpuacct.usage").writeUtf8File("91623711445\n"); - cpuacctPath.resolve("cpuacct.stat").writeUtf8File("user 7463\n" + - "system 1741\n"); - cpuacctPath.resolve("cpu.stat").writeUtf8File("nr_periods 2361\n" + - "nr_throttled 342\n" + - "throttled_time 131033468519\n"); - - assertEquals(Map.of(TOTAL_USAGE_USEC, 91623711L, SYSTEM_USAGE_USEC, 17410000L, USER_USAGE_USEC, 74630000L, - TOTAL_PERIODS, 2361L, THROTTLED_PERIODS, 342L, THROTTLED_TIME_USEC, 131033468L), cgroup.cpuStats(containerId)); - } - - @Test - public void reads_memory_metrics() throws IOException { - UnixPath memoryPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-4aec78cc.scope")).createDirectories(); - memoryPath.resolve("memory.usage_in_bytes").writeUtf8File("2525093888\n"); - assertEquals(2525093888L, cgroup.memoryUsageInBytes(containerId)); - - memoryPath.resolve("memory.limit_in_bytes").writeUtf8File("4322885632\n"); - assertEquals(4322885632L, cgroup.memoryLimitInBytes(containerId)); - - memoryPath.resolve("memory.stat").writeUtf8File("cache 296828928\n" + - "rss 2152587264\n" + - "rss_huge 1107296256\n" + - "shmem 135168\n" + - "mapped_file 270336\n"); - assertEquals(296828928L, cgroup.memoryCacheInBytes(containerId)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java index 909979342ea..789f31f75c6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java @@ -14,12 +14,12 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.USER_USAGE_USEC; import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.sharesToWeight; import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.weightToShares; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -34,7 +34,7 @@ public class CGroupV2Test { private static final ContainerId containerId = new ContainerId("4aec78cc"); private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroup cgroup = new CGroupV2(fileSystem); + private final CGroupV2 cgroup = new CGroupV2(fileSystem); private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java index f852eb6235d..72c5d016a47 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java @@ -17,12 +17,12 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.USER_USAGE_USEC; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; @@ -37,7 +37,7 @@ public class ContainerStatsCollectorTest { private final TestTerminal testTerminal = new TestTerminal(); private final ContainerEngineMock containerEngine = new ContainerEngineMock(testTerminal); private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroup cgroup = mock(CGroup.class); + private final CGroupV2 cgroup = mock(CGroupV2.class); private final NodeAgentContext context = NodeAgentContextImpl.builder(NodeSpec.Builder.testSpec("c1").build()) .fileSystem(TestFileSystem.create()) .build(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 6dc3b2b3193..fd6b15609d6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -5,6 +5,7 @@ import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; @@ -42,6 +43,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.hosted.provision.restapi.NodePatcher.DROP_DOCUMENTS_REPORT; + /** * The nodes in the node repo and their state transitions * @@ -727,6 +730,23 @@ public class Nodes { return resultingNodes; } + public List<Node> dropDocuments(ApplicationId applicationId, Optional<ClusterSpec.Id> clusterId) { + try (Mutex lock = applications.lock(applicationId)) { + Instant now = clock.instant(); + List<Node> nodes = list(Node.State.active, Node.State.reserved) + .owner(applicationId) + .matching(node -> { + ClusterSpec cluster = node.allocation().get().membership().cluster(); + if (!cluster.type().isContent()) return false; + return clusterId.isEmpty() || clusterId.get().equals(cluster.id()); + }) + .mapToList(node -> node.with(node.reports().withReport(Report.basicReport(DROP_DOCUMENTS_REPORT, Report.Type.UNSPECIFIED, now, "")))); + if (nodes.isEmpty()) + throw new NoSuchNodeException("No content nodes found for " + applicationId + clusterId.map(id -> " and cluster " + id).orElse("")); + return db.writeTo(nodes, Agent.operator, Optional.empty()); + } + } + public boolean canAllocateTenantNodeTo(Node host) { return canAllocateTenantNodeTo(host, zone.cloud().dynamicProvisioning()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java index 2a4e320fd6a..401554f940d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java @@ -36,16 +36,8 @@ public class ApplicationFilter { public static Predicate<Node> from(String applicationIds) { return makePredicate(StringUtilities.split(applicationIds).stream() - .map(ApplicationFilter::toApplicationId) + .map(ApplicationId::fromFullString) .collect(Collectors.toUnmodifiableSet())); } - public static ApplicationId toApplicationId(String applicationIdString) { - String[] parts = applicationIdString.split("\\."); - if (parts.length != 3) - throw new IllegalArgumentException("Application id must be on the form tenant.application.instance, got '" + - applicationIdString + "'"); - return ApplicationId.from(parts[0], parts[1], parts[2]); - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index bf5b735c4a0..09f947503f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; -import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import java.util.List; import java.util.Optional; @@ -32,7 +31,7 @@ public class LoadBalancersResponse extends SlimeJsonResponse { private Optional<ApplicationId> application() { return Optional.ofNullable(request.getProperty("application")) - .map(ApplicationFilter::toApplicationId); + .map(ApplicationId::fromFullString); } private List<LoadBalancer> loadBalancers() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index bbe287fc034..4dc48459ec9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -58,7 +58,7 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote; public class NodePatcher { // Same as in DropDocumentsReport.java - private static final String DROP_DOCUMENTS_REPORT = "dropDocuments"; + public static final String DROP_DOCUMENTS_REPORT = "dropDocuments"; private static final String WANT_TO_RETIRE = "wantToRetire"; private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index eb935ba6a5c..e04d21d3012 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostName; @@ -222,6 +223,11 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { } if (path.matches("/nodes/v2/maintenance/run/{job}")) return runJob(path.get("job")); if (path.matches("/nodes/v2/upgrade/firmware")) return requestFirmwareCheckResponse(); + if (path.matches("/nodes/v2/application/{applicationId}/drop-documents")) { + int count = nodeRepository.nodes().dropDocuments(ApplicationId.fromFullString(path.get("applicationId")), + Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)).size(); + return new MessageResponse("Triggered dropping of documents on " + count + " nodes"); + } throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); } @@ -482,14 +488,14 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new SlimeJsonResponse(slime); } - private void toSlime(Load load, Cursor object) { + private static void toSlime(Load load, Cursor object) { object.setDouble("cpu", load.cpu()); object.setDouble("memory", load.memory()); object.setDouble("disk", load.disk()); } /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ - private URI withPath(String newPath, URI uri) { + private static URI withPath(String newPath, URI uri) { try { return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 7affcfebdb3..022822fd3ec 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -647,22 +647,43 @@ public class NodesV2ApiTest { Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json"); + } + + @Test + public void drop_documents() throws IOException { + // Initially no reports + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), "reports", false); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "reports", false); + + // Initiating drop documents will set the report on all nodes + assertResponse(new Request("http://localhost:8080/nodes/v2/application/tenant3.application3.instance3/drop-documents?clusterId=id3", new byte[0], Request.Method.POST), + "{\"message\":\"Triggered dropping of documents on 2 nodes\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), + "{\"dropDocuments\":{\"createdMillis\":123}}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), + "{\"dropDocuments\":{\"createdMillis\":123}}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com", + // Host admin of the first node finishes dropping + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}}"), Request.Method.PATCH), - "{\"message\":\"Updated host1.yahoo.com\"}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + "{\"message\":\"Updated host4.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com", + // Host admin of the second node finishes dropping, node-repo will update report on both nodes to start phase 2 + assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456}}}"), Request.Method.PATCH), - "{\"message\":\"Updated host10.yahoo.com\"}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com"), + "{\"message\":\"Updated test-node-pool-102-2\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), "{\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456,\"readiedAt\":123}}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36,\"readiedAt\":123}}"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/application/does.not.exist/drop-documents", new byte[0], Request.Method.POST), + 404, + "{\"error-code\":\"NOT_FOUND\",\"message\":\"No content nodes found for does.not.exist\"}"); } @Test diff --git a/parent/pom.xml b/parent/pom.xml index 76f4ef30dda..e6ef7e873fd 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -212,6 +212,7 @@ <java.io.tmpdir>${project.build.directory}</java.io.tmpdir> </systemPropertyVariables> <trimStackTrace>false</trimStackTrace> + <argLine>-Djava.io.tmpdir=${project.build.directory}</argLine> </configuration> </plugin> <plugin> diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h index cc2cb216bbb..bc884f9020e 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h @@ -2,7 +2,7 @@ /** * @class search::SearchVisitor * - * @brief Visitor that applies a search query to visitor data and converts them to a SearchResultCommand + * @brief Visitor that applies a search query to visitor data and converts them to a QueryResultCommand */ #pragma once diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp index c515a1bec7f..0b1d1221df2 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp @@ -2,7 +2,7 @@ /** * @class search::SearchVisitor * - * @brief Visitor that applies a search query to visitor data and converts them to a SearchResultCommand + * @brief Visitor that applies a search query to visitor data and converts them to a QueryResultCommand */ #pragma once diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index c59b56eb68f..11bbf2c241a 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -19,7 +19,6 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST externaloperationhandlertest.cpp garbagecollectiontest.cpp getoperationtest.cpp - gtest_runner.cpp idealstatemanagertest.cpp joinbuckettest.cpp maintenancemocks.cpp @@ -27,6 +26,7 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST mergelimitertest.cpp mergeoperationtest.cpp multi_thread_stripe_access_guard_test.cpp + newest_replica_test.cpp node_supported_features_repo_test.cpp nodeinfotest.cpp nodemaintenancestatstrackertest.cpp @@ -57,7 +57,7 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST storage_testcommon storage_testhostreporter storage - GTest::GTest + GTest::gmock_main ) vespa_add_test( diff --git a/storage/src/tests/distributor/gtest_runner.cpp b/storage/src/tests/distributor/gtest_runner.cpp deleted file mode 100644 index 7c20f681093..00000000000 --- a/storage/src/tests/distributor/gtest_runner.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> -LOG_SETUP("storage_distributor_gtest_runner"); - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/storage/src/tests/distributor/newest_replica_test.cpp b/storage/src/tests/distributor/newest_replica_test.cpp new file mode 100644 index 00000000000..9267c6e37a2 --- /dev/null +++ b/storage/src/tests/distributor/newest_replica_test.cpp @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/storage/distributor/operations/external/newest_replica.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/gtest/matchers/elements_are_distinct.h> + +using namespace ::testing; +using storage::api::Timestamp; +using document::BucketId; + +namespace storage::distributor { + +TEST(NewestReplicaTest, equality_predicate_considers_all_fields) { + std::vector elems = { + NewestReplica::of(Timestamp(1000), BucketId(16, 1), 0, false, false), + NewestReplica::of(Timestamp(1001), BucketId(16, 1), 0, false, false), + NewestReplica::of(Timestamp(1000), BucketId(16, 2), 0, false, false), + NewestReplica::of(Timestamp(1000), BucketId(16, 1), 1, false, false), + NewestReplica::of(Timestamp(1000), BucketId(16, 1), 0, true, false), + NewestReplica::of(Timestamp(1000), BucketId(16, 1), 0, false, true) + }; + EXPECT_THAT(elems, ElementsAreDistinct()); +} + +} diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index dea215f47dc..aa5ad217ae9 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -15,13 +15,12 @@ #include <algorithm> #include <vespa/log/log.h> -LOG_SETUP(".distributor.callback.doc.put"); +LOG_SETUP(".distributor.operations.external.put"); - -using namespace storage::distributor; -using namespace storage; using document::BucketSpace; +namespace storage::distributor { + PutOperation::PutOperation(const DistributorNodeContext& node_ctx, DistributorStripeOperationContext& op_ctx, DistributorBucketSpace& bucketSpace, @@ -116,6 +115,20 @@ bool PutOperation::has_unavailable_targets_in_pending_state(const OperationTarge }); } +bool PutOperation::at_least_one_storage_node_is_available() const { + const lib::ClusterState& cluster_state = _bucketSpace.getClusterState(); + + const uint16_t storage_node_index_ubound = cluster_state.getNodeCount(lib::NodeType::STORAGE); + for (uint16_t i = 0; i < storage_node_index_ubound; i++) { + if (cluster_state.getNodeState(lib::Node(lib::NodeType::STORAGE, i)) + .getState().oneOf(storage_node_up_states())) + { + return true; + } + } + return false; +} + void PutOperation::onStart(DistributorStripeMessageSender& sender) { @@ -124,19 +137,7 @@ PutOperation::onStart(DistributorStripeMessageSender& sender) LOG(debug, "Received PUT %s for bucket %s", _msg->getDocumentId().toString().c_str(), bid.toString().c_str()); - lib::ClusterState systemState = _bucketSpace.getClusterState(); - - // Don't do anything if all nodes are down. - bool up = false; - for (uint16_t i = 0; i < systemState.getNodeCount(lib::NodeType::STORAGE); i++) { - if (systemState.getNodeState(lib::Node(lib::NodeType::STORAGE, i)) - .getState().oneOf(storage_node_up_states())) - { - up = true; - } - } - - if (up) { + if (at_least_one_storage_node_is_available()) { std::vector<document::BucketId> bucketsToCheckForSplit; OperationTargetResolverImpl targetResolver(_bucketSpace, _bucketSpace.getBucketDatabase(), @@ -145,8 +146,8 @@ PutOperation::onStart(DistributorStripeMessageSender& sender) _msg->getBucket().getBucketSpace()); OperationTargetList targets(targetResolver.getTargets(OperationTargetResolver::PUT, bid)); - for (size_t i = 0; i < targets.size(); ++i) { - if (_op_ctx.has_pending_message(targets[i].getNode().getIndex(), targets[i].getBucket(), + for (const auto& target : targets) { + if (_op_ctx.has_pending_message(target.getNode().getIndex(), target.getBucket(), api::MessageType::DELETEBUCKET_ID)) { _tracker.fail(sender, api::ReturnCode(api::ReturnCode::BUCKET_DELETED, @@ -179,13 +180,12 @@ PutOperation::onStart(DistributorStripeMessageSender& sender) std::vector<PersistenceMessageTracker::ToSend> putBatch; // Now send PUTs - for (uint32_t i = 0; i < targets.size(); i++) { - const OperationTarget& target(targets[i]); + for (const auto& target : targets) { sendPutToBucketOnNode(_msg->getBucket().getBucketSpace(), target.getBucketId(), target.getNode().getIndex(), putBatch); } - if (putBatch.size()) { + if (!putBatch.empty()) { _tracker.queueMessageBatch(putBatch); } else { const char* error = "Can't store document: No storage nodes available"; @@ -196,9 +196,9 @@ PutOperation::onStart(DistributorStripeMessageSender& sender) // Check whether buckets are large enough to be split. // TODO(vekterli): only check entries for sendToExisting? - for (uint32_t i = 0; i < entries.size(); ++i) { + for (const auto& entry : entries) { _op_ctx.send_inline_split_if_bucket_too_large(_msg->getBucket().getBucketSpace(), - entries[i], _msg->getPriority()); + entry, _msg->getPriority()); } _tracker.flushQueue(sender); @@ -235,3 +235,5 @@ PutOperation::onClose(DistributorStripeMessageSender& sender) LOG(debug, "%s", error); _tracker.fail(sender, api::ReturnCode(api::ReturnCode::ABORTED, error)); } + +} diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.h b/storage/src/vespa/storage/distributor/operations/external/putoperation.h index 9801fed0c99..283395f1406 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.h @@ -47,9 +47,10 @@ private: void sendPutToBucketOnNode(document::BucketSpace bucketSpace, const document::BucketId& bucketId, const uint16_t node, std::vector<PersistenceMessageTracker::ToSend>& putBatch); - bool shouldImplicitlyActivateReplica(const OperationTargetList& targets) const; + [[nodiscard]] bool shouldImplicitlyActivateReplica(const OperationTargetList& targets) const; - bool has_unavailable_targets_in_pending_state(const OperationTargetList& targets) const; + [[nodiscard]] bool has_unavailable_targets_in_pending_state(const OperationTargetList& targets) const; + [[nodiscard]] bool at_least_one_storage_node_is_available() const; std::shared_ptr<api::PutCommand> _msg; DistributorStripeOperationContext& _op_ctx; diff --git a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp index 584ad9de2ce..b752a7fadde 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp @@ -5,13 +5,12 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/log/log.h> -LOG_SETUP(".distributor.operation.external.remove"); +LOG_SETUP(".distributor.operations.external.remove"); - -using namespace storage::distributor; -using namespace storage; using document::BucketSpace; +namespace storage::distributor { + RemoveOperation::RemoveOperation(const DistributorNodeContext& node_ctx, DistributorStripeOperationContext& op_ctx, DistributorBucketSpace& bucketSpace, @@ -100,3 +99,5 @@ RemoveOperation::onClose(DistributorStripeMessageSender& sender) { _tracker.fail(sender, api::ReturnCode(api::ReturnCode::ABORTED, "Process is shutting down")); } + +} diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index 9ae6aaf0653..e7eb7a752fb 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -8,11 +8,9 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/storage/common/bucket_resolver.h> #include <vespa/storageapi/message/datagram.h> -#include <vespa/storageapi/message/documentsummary.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/queryresult.h> #include <vespa/storageapi/message/removelocation.h> -#include <vespa/storageapi/message/searchresult.h> #include <vespa/storageapi/message/stat.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/messagebus/error.h> @@ -237,24 +235,12 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) toMsg = std::move(to); break; } - case api::MessageType::SEARCHRESULT_ID: - { - auto & from(static_cast<api::SearchResultCommand&>(fromMsg)); - toMsg = std::make_unique<documentapi::SearchResultMessage>(std::move(from)); - break; - } case api::MessageType::QUERYRESULT_ID: { auto & from(static_cast<api::QueryResultCommand&>(fromMsg)); toMsg = std::make_unique<documentapi::QueryResultMessage>(std::move(from.getSearchResult()), from.getDocumentSummary()); break; } - case api::MessageType::DOCUMENTSUMMARY_ID: - { - auto & from(static_cast<api::DocumentSummaryCommand&>(fromMsg)); - toMsg = std::make_unique<documentapi::DocumentSummaryMessage>(from); - break; - } case api::MessageType::MAPVISITOR_ID: { auto & from(static_cast<api::MapVisitorCommand&>(fromMsg)); diff --git a/storage/src/vespa/storageapi/message/CMakeLists.txt b/storage/src/vespa/storageapi/message/CMakeLists.txt index 2a761921dff..2728b5b51ad 100644 --- a/storage/src/vespa/storageapi/message/CMakeLists.txt +++ b/storage/src/vespa/storageapi/message/CMakeLists.txt @@ -6,9 +6,7 @@ vespa_add_library(storageapi_message OBJECT bucket.cpp visitor.cpp state.cpp - searchresult.cpp bucketsplitting.cpp - documentsummary.cpp stat.cpp removelocation.cpp queryresult.cpp diff --git a/storage/src/vespa/storageapi/message/documentsummary.cpp b/storage/src/vespa/storageapi/message/documentsummary.cpp deleted file mode 100644 index 6909b4d223c..00000000000 --- a/storage/src/vespa/storageapi/message/documentsummary.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentsummary.h" -#include <ostream> - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(DocumentSummaryCommand, DocumentSummaryReply) -IMPLEMENT_REPLY(DocumentSummaryReply) - -DocumentSummaryCommand::DocumentSummaryCommand() - : StorageCommand(MessageType::DOCUMENTSUMMARY), - DocumentSummary() -{ } - -void -DocumentSummaryCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DocumentSummary(" << getSummaryCount() << " summaries)"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -DocumentSummaryReply::DocumentSummaryReply(const DocumentSummaryCommand& cmd) - : StorageReply(cmd) -{ } - -void -DocumentSummaryReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DocumentSummaryReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storage/src/vespa/storageapi/message/documentsummary.h b/storage/src/vespa/storageapi/message/documentsummary.h deleted file mode 100644 index 5e2c1af3cfd..00000000000 --- a/storage/src/vespa/storageapi/message/documentsummary.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include <vespa/vdslib/container/documentsummary.h> - -namespace storage { -namespace api { - -/** - * @class DocumentSummaryCommand - * @ingroup message - * - * @brief The result of a searchvisitor. - */ -class DocumentSummaryCommand : public StorageCommand, - public vdslib::DocumentSummary -{ -public: - explicit DocumentSummaryCommand(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(DocumentSummaryCommand, onDocumentSummary) -}; - -/** - * @class DocumentSummaryReply - * @ingroup message - * - * @brief Response to a document summary command. - */ -class DocumentSummaryReply : public StorageReply { -public: - explicit DocumentSummaryReply(const DocumentSummaryCommand& command); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(DocumentSummaryReply, onDocumentSummaryReply) -}; - -} // api -} // storage diff --git a/storage/src/vespa/storageapi/message/searchresult.cpp b/storage/src/vespa/storageapi/message/searchresult.cpp deleted file mode 100644 index b2cf04b0410..00000000000 --- a/storage/src/vespa/storageapi/message/searchresult.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "searchresult.h" -#include <ostream> - -using vdslib::SearchResult; - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(SearchResultCommand, SearchResultReply) -IMPLEMENT_REPLY(SearchResultReply) - -SearchResultCommand::SearchResultCommand() - : StorageCommand(MessageType::SEARCHRESULT), - SearchResult() -{ -} - -void -SearchResultCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SearchResultCommand(" << getHitCount() << " hits)"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -SearchResultReply::SearchResultReply(const SearchResultCommand& cmd) - : StorageReply(cmd) -{ } - -void -SearchResultReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SearchResultReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storage/src/vespa/storageapi/message/searchresult.h b/storage/src/vespa/storageapi/message/searchresult.h deleted file mode 100644 index b12fa5e1613..00000000000 --- a/storage/src/vespa/storageapi/message/searchresult.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include <vespa/vdslib/container/searchresult.h> - -namespace storage::api { - -/** - * @class SearchResultCommand - * @ingroup message - * - * @brief The result of a searchvisitor. - */ -class SearchResultCommand : public StorageCommand, public vdslib::SearchResult { -public: - SearchResultCommand(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(SearchResultCommand, onSearchResult) -}; - -/** - * @class SearchResultReply - * @ingroup message - * - * @brief Response to a search result command. - */ -class SearchResultReply : public StorageReply { -public: - explicit SearchResultReply(const SearchResultCommand& command); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(SearchResultReply, onSearchResultReply) -}; - -} diff --git a/storage/src/vespa/storageapi/messageapi/messagehandler.h b/storage/src/vespa/storageapi/messageapi/messagehandler.h index 9ba8542e9db..fa362d5380f 100644 --- a/storage/src/vespa/storageapi/messageapi/messagehandler.h +++ b/storage/src/vespa/storageapi/messageapi/messagehandler.h @@ -29,8 +29,6 @@ class CreateVisitorCommand; // Create a new visitor class DestroyVisitorCommand; // Destroy a running visitor class VisitorInfoCommand; // Sends visitor info to visitor controller class MapVisitorCommand; -class SearchResultCommand; -class DocumentSummaryCommand; class QueryResultCommand; class InternalCommand; @@ -67,8 +65,6 @@ class CreateVisitorReply; class DestroyVisitorReply; class VisitorInfoReply; class MapVisitorReply; -class SearchResultReply; -class DocumentSummaryReply; class QueryResultReply; class InternalReply; @@ -137,12 +133,8 @@ public: virtual bool onVisitorInfoReply(const std::shared_ptr<api::VisitorInfoReply>&) { return false; } virtual bool onMapVisitor(const std::shared_ptr<api::MapVisitorCommand>&) { return false; } virtual bool onMapVisitorReply(const std::shared_ptr<api::MapVisitorReply>&) { return false; } - virtual bool onSearchResult(const std::shared_ptr<api::SearchResultCommand>&) { return false; } - virtual bool onSearchResultReply(const std::shared_ptr<api::SearchResultReply>&) { return false; } virtual bool onQueryResult(const std::shared_ptr<api::QueryResultCommand>&) { return false; } virtual bool onQueryResultReply(const std::shared_ptr<api::QueryResultReply>&) { return false; } - virtual bool onDocumentSummary(const std::shared_ptr<api::DocumentSummaryCommand>&) { return false; } - virtual bool onDocumentSummaryReply(const std::shared_ptr<api::DocumentSummaryReply>&) { return false; } virtual bool onEmptyBuckets(const std::shared_ptr<api::EmptyBucketsCommand>&) { return false; } virtual bool onEmptyBucketsReply(const std::shared_ptr<api::EmptyBucketsReply>&) { return false; } virtual bool onInternal(const std::shared_ptr<api::InternalCommand>&) { return false; } diff --git a/storage/src/vespa/storageapi/messageapi/storagemessage.cpp b/storage/src/vespa/storageapi/messageapi/storagemessage.cpp index b1d68fd77e3..c72ece80476 100644 --- a/storage/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storage/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -76,10 +76,6 @@ const MessageType MessageType::APPLYBUCKETDIFF("ApplyBucketDiff", APPLYBUCKETDIF const MessageType MessageType::APPLYBUCKETDIFF_REPLY("ApplyBucketDiff reply", APPLYBUCKETDIFF_REPLY_ID, &MessageType::APPLYBUCKETDIFF); const MessageType MessageType::VISITOR_INFO("VisitorInfo", VISITOR_INFO_ID); const MessageType MessageType::VISITOR_INFO_REPLY("VisitorInfo reply", VISITOR_INFO_REPLY_ID, &MessageType::VISITOR_INFO); -const MessageType MessageType::SEARCHRESULT("SearchResult", SEARCHRESULT_ID); -const MessageType MessageType::SEARCHRESULT_REPLY("SearchResult reply", SEARCHRESULT_REPLY_ID, &MessageType::SEARCHRESULT); -const MessageType MessageType::DOCUMENTSUMMARY("DocumentSummary", DOCUMENTSUMMARY_ID); -const MessageType MessageType::DOCUMENTSUMMARY_REPLY("DocumentSummary reply", DOCUMENTSUMMARY_REPLY_ID, &MessageType::DOCUMENTSUMMARY); const MessageType MessageType::MAPVISITOR("Mapvisitor", MAPVISITOR_ID); const MessageType MessageType::MAPVISITOR_REPLY("Mapvisitor reply", MAPVISITOR_REPLY_ID, &MessageType::MAPVISITOR); const MessageType MessageType::SPLITBUCKET("SplitBucket", SPLITBUCKET_ID); diff --git a/storage/src/vespa/storageapi/messageapi/storagemessage.h b/storage/src/vespa/storageapi/messageapi/storagemessage.h index 282f110646d..4649781c1e5 100644 --- a/storage/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storage/src/vespa/storageapi/messageapi/storagemessage.h @@ -122,14 +122,14 @@ public: DOCBLOCK_REPLY_ID = 59, VISITOR_INFO_ID = 60, VISITOR_INFO_REPLY_ID = 61, - SEARCHRESULT_ID = 64, - SEARCHRESULT_REPLY_ID = 65, + // SEARCHRESULT_ID = 64, + // SEARCHRESULT_REPLY_ID = 65, SPLITBUCKET_ID = 66, SPLITBUCKET_REPLY_ID = 67, JOINBUCKETS_ID = 68, JOINBUCKETS_REPLY_ID = 69, - DOCUMENTSUMMARY_ID = 72, - DOCUMENTSUMMARY_REPLY_ID = 73, + // DOCUMENTSUMMARY_ID = 72, + // DOCUMENTSUMMARY_REPLY_ID = 73, MAPVISITOR_ID = 74, MAPVISITOR_REPLY_ID = 75, STATBUCKET_ID = 76, @@ -208,14 +208,10 @@ public: static const MessageType APPLYBUCKETDIFF_REPLY; static const MessageType VISITOR_INFO; static const MessageType VISITOR_INFO_REPLY; - static const MessageType SEARCHRESULT; - static const MessageType SEARCHRESULT_REPLY; static const MessageType SPLITBUCKET; static const MessageType SPLITBUCKET_REPLY; static const MessageType JOINBUCKETS; static const MessageType JOINBUCKETS_REPLY; - static const MessageType DOCUMENTSUMMARY; - static const MessageType DOCUMENTSUMMARY_REPLY; static const MessageType MAPVISITOR; static const MessageType MAPVISITOR_REPLY; static const MessageType STATBUCKET; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 162c031ac9e..80df69f756e 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -38,7 +38,7 @@ namespace streaming { * @class storage::SearchVisitor * * @brief Visitor that applies a search query to visitor data and - * converts them to a SearchResultCommand and a DocumentSummaryCommand. + * converts them to a QueryResultCommand. **/ class SearchVisitor : public storage::Visitor { public: diff --git a/vespa-documentgen-plugin/etc/music/music.sd b/vespa-documentgen-plugin/etc/music/music.sd index 205614db67d..8c2d324697c 100644 --- a/vespa-documentgen-plugin/etc/music/music.sd +++ b/vespa-documentgen-plugin/etc/music/music.sd @@ -32,6 +32,14 @@ search music { indexing: summary | index } + field tags type weightedset<string> { + indexing: attribute | summary + attribute: fast-search + weightedset { + create-if-nonexistent + } + } + } rank-profile default inherits default { @@ -48,6 +56,11 @@ search music { } + document-summary tags { + summary tags type weightedset<string> { + full + } + } } diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java index 2a7eb261a0b..f55b226b11b 100644 --- a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java +++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java @@ -21,9 +21,11 @@ public class DocumentGenTest { DocumentGenMojo mojo = new DocumentGenMojo(); mojo.execute(new File("etc/music/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), "com.yahoo.vespa.document"); Map<String, Schema> searches = mojo.getSearches(); - assertEquals(searches.size(),1); + assertEquals(searches.size(), 1); assertEquals(searches.get("music").getDocument("music").getField("title").getDataType(), DataType.STRING); assertEquals(searches.get("music").getDocument("music").getField("eitheror").getDataType(), DataType.BOOL); + assertEquals(searches.get("music").getDocument("music").getField("tags").getDataType(), + searches.get("music").getSummaries().get("tags").getSummaryField("tags").getDataType()); } @Test diff --git a/vespalib/src/vespa/vespalib/gtest/matchers/elements_are_distinct.h b/vespalib/src/vespa/vespalib/gtest/matchers/elements_are_distinct.h new file mode 100644 index 00000000000..374dafbf893 --- /dev/null +++ b/vespalib/src/vespa/vespalib/gtest/matchers/elements_are_distinct.h @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <gtest/gtest.h> +#include <gmock/gmock.h> +#include <ranges> + +/** + * Checks that all elements of a forward iterable range are distinct, i.e. the following must hold: + * - for any single element `foo`, foo == foo is true + * - for any two separate elements `foo` and `bar`, foo == bar is false + */ +MATCHER(ElementsAreDistinct, "") { + const auto& range = arg; + static_assert(std::ranges::forward_range<decltype(range)>); + const auto end = std::ranges::cend(range); + // Explicitly count element positions instead of comparing iterators to avoid depending + // on iterators being comparable with each other. + size_t i = 0; + for (auto lhs = std::ranges::cbegin(range); lhs != end; ++lhs, ++i) { + size_t j = 0; + for (auto rhs = std::ranges::cbegin(range); rhs != end; ++rhs, ++j) { + if (i != j) { + if (*lhs == *rhs) { + *result_listener << "Expected elements to be distinct, but element at position " + << i << " (" << *lhs << ") is equal to element at position " + << j << " (" << *rhs << ")"; + return false; + } + } else if (!(*lhs == *rhs)) { + *result_listener << "Element at position " << i << " (" << *lhs << ") does not equal itself"; + return false; + } + } + } + return true; +} |