diff options
378 files changed, 3323 insertions, 2002 deletions
diff --git a/.buildkite/Makefile b/.buildkite/Makefile index b5f730d0133..7fed05cc7bd 100644 --- a/.buildkite/Makefile +++ b/.buildkite/Makefile @@ -12,8 +12,8 @@ export FACTORY_VESPA_VERSION := $(VESPA_VERSION) export VALGRIND_UNIT_TESTS ?= false export VESPA_USE_SANITIZER ?= null -export VESPA_MAVEN_EXTRA_OPTS ?= --show-version --batch-mode --no-snapshot-updates -Dmaven.javadoc.skip=true \ - -Dmaven.source.skip=true -DaltDeploymentRepository=local-repo::default::file:$(WORKDIR)/artifacts/$(ARCH)/maven-repo +export VESPA_MAVEN_EXTRA_OPTS ?= --show-version --batch-mode --no-snapshot-updates \ + -DaltDeploymentRepository=local-repo::default::file:$(WORKDIR)/artifacts/$(ARCH)/maven-repo export NUM_CPU_LIMIT ?= $(shell nproc) export NUM_CPP_THREADS := $(shell echo $$(( $(NUM_CPU_LIMIT)*2/3 ))) @@ -25,11 +25,12 @@ ifeq ($(BUILDKITE_PULL_REQUEST),false) export VESPA_MAVEN_TARGET ?= deploy else export VESPA_MAVEN_TARGET ?= install + export VESPA_MAVEN_EXTRA_OPTS := $(VESPA_MAVEN_EXTRA_OPTS) -Dmaven.source.skip=true -Dmaven.javadoc.skip=true endif .DEFAULT_GOAL := pr -main: build-rpms cpp-test quick-start-guide publish-container publish-artifacts upload-test-results +main: build-rpms cpp-test quick-start-guide publish-container publish-artifacts pr: build-rpms cpp-test basic-search-test check: diff --git a/client/go/Makefile b/client/go/Makefile index fee92547e73..608ca02b7ce 100644 --- a/client/go/Makefile +++ b/client/go/Makefile @@ -85,7 +85,7 @@ dist-win64: GOOS=windows dist-win64: GOARCH=amd64 $(DIST_TARGETS): DIST_NAME=vespa-cli_$(VERSION)_$(GOOS)_$(GOARCH) -$(DIST_TARGETS): dist-version ci manpages +$(DIST_TARGETS): dist-version setenv manpages $(DIST_TARGETS): mkdir -p $(DIST)/$(DIST_NAME)/bin env CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $(DIST)/$(DIST_NAME)/bin $(GO_FLAGS) \ diff --git a/client/go/internal/admin/jvm/qr_start_cfg.go b/client/go/internal/admin/jvm/qr_start_cfg.go index a14ab0d1946..cbcc2c066ee 100644 --- a/client/go/internal/admin/jvm/qr_start_cfg.go +++ b/client/go/internal/admin/jvm/qr_start_cfg.go @@ -38,7 +38,7 @@ func (a *ApplicationContainer) getQrStartCfg() *QrStartConfig { var parsedJson QrStartConfig args := []string{ "-j", - "-w", "10", + "-w", "30", "-n", "search.config.qr-start", "-i", a.ConfigId(), } diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index d6bc59f5b4f..18906d1a66e 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -18,7 +18,7 @@ import ( func addFeedFlags(cli *CLI, cmd *cobra.Command, options *feedOptions) { cmd.PersistentFlags().IntVar(&options.connections, "connections", 8, "The number of connections to use") - cmd.PersistentFlags().StringVar(&options.compression, "compression", "auto", `Compression mode to use. Default is "auto" which compresses large documents. Must be "auto", "gzip" or "none"`) + cmd.PersistentFlags().StringVar(&options.compression, "compression", "auto", `Whether to compress the document data when sending the HTTP request. Default is "auto", which compresses large documents. Must be "auto", "gzip" or "none"`) cmd.PersistentFlags().IntVar(&options.timeoutSecs, "timeout", 0, "Individual feed operation timeout in seconds. 0 to disable (default 0)") cmd.Flags().StringSliceVarP(&options.headers, "header", "", nil, "Add a header to all HTTP requests, on the format 'Header: Value'. This can be specified multiple times") cmd.PersistentFlags().IntVar(&options.doomSecs, "deadline", 0, "Exit if this number of seconds elapse without any successful operations. 0 to disable (default 0)") diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 2172bac94b8..a549cb81fbc 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -876,18 +876,18 @@ eslint-visitor-keys "^3.3.0" "@eslint-community/regexpp@^4.6.1": - version "4.10.1" - resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.1.tgz#361461e5cb3845d874e61731c11cfedd664d83a0" - integrity sha512-Zm2NGpWELsQAD1xsJzGQpYfvICSsFkEpU0jxBjfdC6uNEWXcHnfs9hScFWtXVDVl+rBQJGrl4g1vcKIejpH9dA== + version "4.11.0" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.11.0.tgz#b0ffd0312b4a3fd2d6f77237e7248a5ad3a680ae" + integrity sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A== -"@eslint/config-array@^0.16.0": - version "0.16.0" - resolved "https://registry.yarnpkg.com/@eslint/config-array/-/config-array-0.16.0.tgz#bb3364fc39ee84ec3a62abdc4b8d988d99dfd706" - integrity sha512-/jmuSd74i4Czf1XXn7wGRWZCuyaUZ330NH1Bek0Pplatt4Sy1S5haN21SCLLdbeKslQ+S0wEJ+++v5YibSi+Lg== +"@eslint/config-array@^0.17.0": + version "0.17.0" + resolved "https://registry.yarnpkg.com/@eslint/config-array/-/config-array-0.17.0.tgz#ff305e1ee618a00e6e5d0485454c8d92d94a860d" + integrity sha512-A68TBu6/1mHHuc5YJL0U0VVeGNiklLAL6rRmhTCP2B5XjWLMnrX+HkO+IAXyHvks5cyyY1jjK5ITPQ1HGS2EVA== dependencies: "@eslint/object-schema" "^2.1.4" debug "^4.3.1" - minimatch "^3.0.5" + minimatch "^3.1.2" "@eslint/eslintrc@^3.1.0": version "3.1.0" @@ -904,10 +904,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@9.5.0": - version "9.5.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.5.0.tgz#0e9c24a670b8a5c86bff97b40be13d8d8f238045" - integrity sha512-A7+AOT2ICkodvtsWnxZP4Xxk3NbZ3VMHd8oihydLRGrJgqqdEz1qSeEgXYyT/Cu8h1TWWsQRejIx48mtjZ5y1w== +"@eslint/js@9.6.0": + version "9.6.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.6.0.tgz#5b0cb058cc13d9c92d4e561d3538807fa5127c95" + integrity sha512-D9B0/3vNg44ZeWbYMpBoXqNP4j6eQD5vNwIlGAuFRRzK/WtT/jvDQW3Bi9kkf3PMDMlM7Yi+73VLUsn5bJcl8A== "@eslint/object-schema@^2.1.4": version "2.1.4" @@ -1443,10 +1443,10 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.16.1": - version "1.16.1" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.16.1.tgz#73db3c48b975eeb06d0006481bde4f5f2d17d1cd" - integrity sha512-es2g3dq6Nb07iFxGk5GuHN20RwBZOsuDQN7izWIisUcv9r+d2C5jQxqmgkdebXgReWfiyUabcki6Fg77mSNrig== +"@remix-run/router@1.17.0": + version "1.17.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.17.0.tgz#fbb0add487478ef42247d5942e7a5d8a2e20095f" + integrity sha512-2D6XaHEVvkCn682XBnipbJjgZUU7xjLtA4dGJRBVUKpEaDYOZMENZoZjAOSb7qirxt5RupjzZxz4fK2FO+EFPw== "@rollup/rollup-android-arm-eabi@4.18.0": version "4.18.0" @@ -1684,7 +1684,7 @@ acorn-jsx@^5.3.2: resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== -acorn@^8.11.3: +acorn@^8.12.0: version "8.12.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.0.tgz#1627bfa2e058148036133b8d9b51a700663c294c" integrity sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw== @@ -2793,15 +2793,15 @@ eslint-visitor-keys@^4.0.0: integrity sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw== eslint@^9.0.0: - version "9.5.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.5.0.tgz#11856034b94a9e1a02cfcc7e96a9f0956963cd2f" - integrity sha512-+NAOZFrW/jFTS3dASCGBxX1pkFD0/fsO+hfAkJ4TyYKwgsXZbqzrw+seCYFCcPCYXvnD67tAnglU7GQTz6kcVw== + version "9.6.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.6.0.tgz#9f54373afa15e1ba356656a8d96233182027fb49" + integrity sha512-ElQkdLMEEqQNM9Njff+2Y4q2afHk7JpkPvrd7Xh7xefwgQynqPxwf55J7di9+MEibWUGdNjFF9ITG9Pck5M84w== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" - "@eslint/config-array" "^0.16.0" + "@eslint/config-array" "^0.17.0" "@eslint/eslintrc" "^3.1.0" - "@eslint/js" "9.5.0" + "@eslint/js" "9.6.0" "@humanwhocodes/module-importer" "^1.0.1" "@humanwhocodes/retry" "^0.3.0" "@nodelib/fs.walk" "^1.2.8" @@ -2812,7 +2812,7 @@ eslint@^9.0.0: escape-string-regexp "^4.0.0" eslint-scope "^8.0.1" eslint-visitor-keys "^4.0.0" - espree "^10.0.1" + espree "^10.1.0" esquery "^1.5.0" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -2832,12 +2832,12 @@ eslint@^9.0.0: strip-ansi "^6.0.1" text-table "^0.2.0" -espree@^10.0.1: - version "10.0.1" - resolved "https://registry.yarnpkg.com/espree/-/espree-10.0.1.tgz#600e60404157412751ba4a6f3a2ee1a42433139f" - integrity sha512-MWkrWZbJsL2UwnjxTX3gG8FneachS/Mwg7tdGXce011sJd5b0JG54vat5KHnfSBODZ3Wvzd2WnjxyzsRoVv+ww== +espree@^10.0.1, espree@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/espree/-/espree-10.1.0.tgz#8788dae611574c0f070691f522e4116c5a11fc56" + integrity sha512-M1M6CpiE6ffoigIOWYO9UDP8TMUw9kqb21tf+08IgDYjCsOvCuDt4jQcZmoYxx+w7zlKw9/N0KXfto+I8/FrXA== dependencies: - acorn "^8.11.3" + acorn "^8.12.0" acorn-jsx "^5.3.2" eslint-visitor-keys "^4.0.0" @@ -4382,7 +4382,7 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: +minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== @@ -4868,19 +4868,19 @@ react-refresh@^0.14.2: integrity sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA== react-router-dom@^6: - version "6.23.1" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.23.1.tgz#30cbf266669693e9492aa4fc0dde2541ab02322f" - integrity sha512-utP+K+aSTtEdbWpC+4gxhdlPFwuEfDKq8ZrPFU65bbRJY+l706qjR7yaidBpo3MSeA/fzwbXWbKBI6ftOnP3OQ== + version "6.24.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.24.0.tgz#ec49dc38c49bb9bd25b310a8ae849268d3085e1d" + integrity sha512-960sKuau6/yEwS8e+NVEidYQb1hNjAYM327gjEyXlc6r3Skf2vtwuJ2l7lssdegD2YjoKG5l8MsVyeTDlVeY8g== dependencies: - "@remix-run/router" "1.16.1" - react-router "6.23.1" + "@remix-run/router" "1.17.0" + react-router "6.24.0" -react-router@6.23.1: - version "6.23.1" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.23.1.tgz#d08cbdbd9d6aedc13eea6e94bc6d9b29cb1c4be9" - integrity sha512-fzcOaRF69uvqbbM7OhvQyBTFDVrrGlsFdS3AL+1KfIBtGETibHzi3FkoTRyiDJnWNc2VxrfvR+657ROHjaNjqQ== +react-router@6.24.0: + version "6.24.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.24.0.tgz#aa46648f26b6525e07f908ad3e1ad2e68d131155" + integrity sha512-sQrgJ5bXk7vbcC4BxQxeNa5UmboFm35we1AFK0VvQaz9g0LzxEIuLOhHIoZ8rnu9BO21ishGeL9no1WB76W/eg== dependencies: - "@remix-run/router" "1.16.1" + "@remix-run/router" "1.17.0" react-textarea-autosize@8.3.4: version "8.3.4" @@ -5670,9 +5670,9 @@ v8-to-istanbul@^9.0.1: convert-source-map "^1.6.0" vite@^5.0.5: - version "5.3.1" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.1.tgz#bb2ca6b5fd7483249d3e86b25026e27ba8a663e6" - integrity sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ== + version "5.3.2" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.2.tgz#2f0a8531c71060467ed3e0a205a203f269b6d9c8" + integrity sha512-6lA7OBHBlXUxiJxbO5aAY2fsHHzDr1q7DvXYnyZycRs2Dz+dXBWuhpWHvmljTRTpQC2uvGmUFFkSHF2vGo90MA== dependencies: esbuild "^0.21.3" postcss "^8.4.38" diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java index 265a99e2f72..927161812c4 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java @@ -56,7 +56,7 @@ public class ClusterControllerClusterConfigurer extends AbstractComponent { ZookeepersConfig zookeepersConfig) { Distribution distribution = new Distribution(distributionConfig); FleetControllerOptions.Builder builder = new FleetControllerOptions.Builder(fleetcontrollerConfig.cluster_name(), distribution.getNodes()); - builder.setStorageDistribution(distribution); + builder.setDistributionConfig(distributionConfig); configure(builder, fleetcontrollerConfig); configure(builder, slobroksConfig); configure(builder, zookeepersConfig); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java index de56272d0e8..7ac2121a496 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vespa.config.content.StorDistributionConfig; import java.util.List; import java.util.Map; @@ -34,6 +35,7 @@ public class ClusterStateBundle { private final AnnotatedClusterState baselineState; private final Map<String, AnnotatedClusterState> derivedBucketSpaceStates; + private final DistributionConfigBundle distributionConfig; private final FeedBlock feedBlock; private final boolean deferredActivation; @@ -113,6 +115,7 @@ public class ClusterStateBundle { public static class Builder { private final AnnotatedClusterState baselineState; private Map<String, AnnotatedClusterState> explicitDerivedStates; + private DistributionConfigBundle distributionConfig; private ClusterStateDeriver stateDeriver; private Set<String> bucketSpaces; private boolean deferredActivation = false; @@ -149,6 +152,16 @@ public class ClusterStateBundle { return this; } + public Builder distributionConfig(StorDistributionConfig config) { + this.distributionConfig = DistributionConfigBundle.of(config); + return this; + } + + public Builder distributionConfig(DistributionConfigBundle config) { + this.distributionConfig = config; + return this; + } + public Builder deferredActivation(boolean deferred) { this.deferredActivation = deferred; return this; @@ -161,27 +174,29 @@ public class ClusterStateBundle { public ClusterStateBundle deriveAndBuild() { if ((stateDeriver == null || bucketSpaces == null || bucketSpaces.isEmpty()) && explicitDerivedStates == null) { - return ClusterStateBundle.ofBaselineOnly(baselineState, feedBlock, deferredActivation); + return ClusterStateBundle.ofBaselineOnly(baselineState, distributionConfig, feedBlock, deferredActivation); } Map<String, AnnotatedClusterState> derived; derived = Objects.requireNonNullElseGet(explicitDerivedStates, () -> bucketSpaces.stream() .collect(Collectors.toUnmodifiableMap( Function.identity(), s -> stateDeriver.derivedFrom(baselineState, s)))); - return new ClusterStateBundle(baselineState, derived, feedBlock, deferredActivation); + return new ClusterStateBundle(baselineState, derived, distributionConfig, feedBlock, deferredActivation); } } private ClusterStateBundle(AnnotatedClusterState baselineState, Map<String, AnnotatedClusterState> derivedBucketSpaceStates) { - this(baselineState, derivedBucketSpaceStates, null, false); + this(baselineState, derivedBucketSpaceStates, null, null, false); } private ClusterStateBundle(AnnotatedClusterState baselineState, Map<String, AnnotatedClusterState> derivedBucketSpaceStates, + DistributionConfigBundle distributionConfig, FeedBlock feedBlock, boolean deferredActivation) { this.baselineState = baselineState; this.derivedBucketSpaceStates = Map.copyOf(derivedBucketSpaceStates); + this.distributionConfig = distributionConfig; this.feedBlock = feedBlock; this.deferredActivation = deferredActivation; } @@ -198,13 +213,28 @@ public class ClusterStateBundle { Map<String, AnnotatedClusterState> derivedBucketSpaceStates, FeedBlock feedBlock, boolean deferredActivation) { - return new ClusterStateBundle(baselineState, derivedBucketSpaceStates, feedBlock, deferredActivation); + return new ClusterStateBundle(baselineState, derivedBucketSpaceStates, null, feedBlock, deferredActivation); + } + + public static ClusterStateBundle of(AnnotatedClusterState baselineState, + Map<String, AnnotatedClusterState> derivedBucketSpaceStates, + FeedBlock feedBlock, + DistributionConfigBundle distributionConfig, + boolean deferredActivation) { + return new ClusterStateBundle(baselineState, derivedBucketSpaceStates, distributionConfig, feedBlock, deferredActivation); + } + + public static ClusterStateBundle ofBaselineOnly(AnnotatedClusterState baselineState, + FeedBlock feedBlock, + boolean deferredActivation) { + return new ClusterStateBundle(baselineState, Map.of(), null, feedBlock, deferredActivation); } public static ClusterStateBundle ofBaselineOnly(AnnotatedClusterState baselineState, + DistributionConfigBundle distributionConfig, FeedBlock feedBlock, boolean deferredActivation) { - return new ClusterStateBundle(baselineState, Map.of(), feedBlock, deferredActivation); + return new ClusterStateBundle(baselineState, Map.of(), distributionConfig, feedBlock, deferredActivation); } public static ClusterStateBundle ofBaselineOnly(AnnotatedClusterState baselineState) { @@ -235,7 +265,7 @@ public class ClusterStateBundle { Map<String, AnnotatedClusterState> clonedDerived = derivedBucketSpaceStates.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().cloneWithClusterState( mapper.apply(e.getValue().getClusterState().clone())))); - return new ClusterStateBundle(clonedBaseline, clonedDerived, feedBlock, deferredActivation); + return new ClusterStateBundle(clonedBaseline, clonedDerived, distributionConfig, feedBlock, deferredActivation); } public ClusterStateBundle clonedWithVersionSet(int version) { @@ -255,6 +285,11 @@ public class ClusterStateBundle { if (clusterFeedIsBlocked() && !feedBlock.similarTo(other.feedBlock)) { return false; } + // Distribution configs must match exactly for bundles to be similar. + // It may be the case that they are both null, in which case they are also considered equal. + if (!Objects.equals(distributionConfig, other.distributionConfig)) { + return false; + } // FIXME we currently treat mismatching bucket space sets as unchanged to avoid breaking some tests return derivedBucketSpaceStates.entrySet().stream() .allMatch(entry -> other.derivedBucketSpaceStates.getOrDefault(entry.getKey(), entry.getValue()) @@ -265,6 +300,10 @@ public class ClusterStateBundle { return baselineState.getClusterState().getVersion(); } + public Optional<DistributionConfigBundle> distributionConfig() { + return Optional.ofNullable(distributionConfig); + } + public Optional<FeedBlock> getFeedBlock() { return Optional.ofNullable(feedBlock); } @@ -282,17 +321,20 @@ public class ClusterStateBundle { String feedBlockedStr = clusterFeedIsBlocked() ? String.format(", feed blocked: '%s'", feedBlock.description) : ""; + String distributionConfigStr = (distributionConfig != null) + ? ", distribution config: %s".formatted(distributionConfig.highLevelDescription()) + : ""; if (derivedBucketSpaceStates.isEmpty()) { - return String.format("ClusterStateBundle('%s'%s%s)", baselineState, + return String.format("ClusterStateBundle('%s'%s%s%s)", baselineState, deferredActivation ? " (deferred activation)" : "", - feedBlockedStr); + feedBlockedStr, distributionConfigStr); } Map<String, AnnotatedClusterState> orderedStates = new TreeMap<>(derivedBucketSpaceStates); - return String.format("ClusterStateBundle('%s', %s%s%s)", baselineState, orderedStates.entrySet().stream() + return String.format("ClusterStateBundle('%s', %s%s%s%s)", baselineState, orderedStates.entrySet().stream() .map(e -> String.format("%s '%s'", e.getKey(), e.getValue())) .collect(Collectors.joining(", ")), deferredActivation ? " (deferred activation)" : "", - feedBlockedStr); + feedBlockedStr, distributionConfigStr); } @Override @@ -303,11 +345,12 @@ public class ClusterStateBundle { return (deferredActivation == that.deferredActivation && Objects.equals(baselineState, that.baselineState) && Objects.equals(derivedBucketSpaceStates, that.derivedBucketSpaceStates) && + Objects.equals(distributionConfig, that.distributionConfig) && Objects.equals(feedBlock, that.feedBlock)); } @Override public int hashCode() { - return Objects.hash(baselineState, derivedBucketSpaceStates, feedBlock, deferredActivation); + return Objects.hash(baselineState, derivedBucketSpaceStates, distributionConfig, feedBlock, deferredActivation); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionConfigBundle.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionConfigBundle.java new file mode 100644 index 00000000000..ab9aca2eeaa --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionConfigBundle.java @@ -0,0 +1,95 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigInstanceSerializer; +import com.yahoo.slime.Slime; +import com.yahoo.vdslib.distribution.Distribution; +import com.yahoo.vdslib.distribution.Group; +import com.yahoo.vdslib.distribution.GroupVisitor; +import com.yahoo.vespa.config.content.StorDistributionConfig; + +import java.util.Objects; + +/** + * Immutable encapsulation of a content cluster distribution configuration, + * including the original config object as well as its processed Distribution + * and Slime config representations. + */ +public class DistributionConfigBundle { + + // Note: All the encapsulated distribution representations are for the _same_ config + // (enforcing this invariant is also why this is not a record type). + private final StorDistributionConfig config; + // Config instances use reflection for most iterating operations, so cache the underlying + // string representation to avoid this for such cases. + private final String canonicalStringRepr; + private final Slime precomputedSlimeRepr; + private final Distribution distribution; + private final int groupsTotal; + private final int nodesTotal; + + public DistributionConfigBundle(StorDistributionConfig config) { + Objects.requireNonNull(config); + this.config = config; + this.canonicalStringRepr = config.toString(); + precomputedSlimeRepr = new Slime(); + ConfigInstance.serialize(config, new ConfigInstanceSerializer(precomputedSlimeRepr, precomputedSlimeRepr.setObject())); + this.distribution = new Distribution(config); + + var gv = new GroupVisitor() { + int groupsTotal = 0; + int nodesTotal = 0; + + @Override + public boolean visitGroup(Group g) { + if (g.isLeafGroup()) { + groupsTotal++; + nodesTotal += g.getNodes().size(); + } + return true; + } + }; + this.distribution.visitGroups(gv); + this.groupsTotal = gv.groupsTotal; + this.nodesTotal = gv.nodesTotal; + } + + public static DistributionConfigBundle of(StorDistributionConfig config) { + return new DistributionConfigBundle(config); + } + + public StorDistributionConfig config() { return config; } + public Slime precomputedSlimeRepr() { return precomputedSlimeRepr; } + public Distribution distribution() { return distribution; } + public int totalLeafGroupCount() { return groupsTotal; } + public int totalNodeCount() { return nodesTotal; } + public int redundancy() { return config.redundancy(); } + public int searchableCopies() { return config.ready_copies(); } + + public String highLevelDescription() { + return "%d nodes; %d groups; redundancy %d; searchable-copies %d".formatted( + totalNodeCount(), totalLeafGroupCount(), redundancy(), searchableCopies()); + } + + // Note: since all fields are deterministically derived from the original config, + // we use the canonical string representation for equals, hashCode and toString. + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DistributionConfigBundle that = (DistributionConfigBundle) o; + return Objects.equals(canonicalStringRepr, that.canonicalStringRepr); + } + + @Override + public int hashCode() { + return canonicalStringRepr.hashCode(); + } + + @Override + public String toString() { + return canonicalStringRepr; + } + +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionDiff.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionDiff.java new file mode 100644 index 00000000000..2cc06229076 --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionDiff.java @@ -0,0 +1,90 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +/** + * Immutable representation of the high-level difference between two distribution configs. + * + * <code>toString()</code> gives a human-readable summary of the diff. + * + * See <code>DistributionDiffCalculatorTest</code> for examples of diff output. + */ +public class DistributionDiff { + + public record NodeDiff(List<Integer> added, List<Integer> removed) {} + + public record IntegerDiff(int before, int after) { + + boolean differs() { return before != after; } + + static IntegerDiff of(int before, int after) { + return new IntegerDiff(before, after); + } + + } + + // (Ordered) mapping from canonical group name to a node diff for that particular group. + private final TreeMap<String, NodeDiff> groupDiffs; + private final IntegerDiff groupCountDiff; + private final IntegerDiff redundancyDiff; + private final IntegerDiff searchableCopiesDiff; + + public DistributionDiff(TreeMap<String, NodeDiff> groupDiffs, + IntegerDiff groupCountDiff, + IntegerDiff redundancyDiff, + IntegerDiff searchableCopiesDiff) { + this.groupCountDiff = groupCountDiff; + this.groupDiffs = groupDiffs; + this.redundancyDiff = redundancyDiff; + this.searchableCopiesDiff = searchableCopiesDiff; + } + + @Override + public String toString() { + var diffs = new ArrayList<String>(); + addIfDiffers(redundancyDiff, "redundancy", diffs); + addIfDiffers(searchableCopiesDiff, "searchable-copies", diffs); + addIfDiffers(groupCountDiff, "groups", diffs); + for (var kv : groupDiffs.entrySet()) { + addGroupDiffStringIfDiffers(kv, diffs); + } + return String.join(", ", diffs); + } + + private static void addIfDiffers(IntegerDiff diff, String name, List<String> diffs) { + if (diff.differs()) { + diffs.add("%s: %d -> %d".formatted(name, diff.before, diff.after)); + } + } + + private static void addGroupDiffStringIfDiffers(Map.Entry<String, NodeDiff> kv, List<String> diffs) { + var v = kv.getValue(); + if (v.added.isEmpty() && v.removed.isEmpty()) { + return; + } + var sb = new StringBuilder(); + sb.append(kv.getKey()).append(": "); + if (!v.added.isEmpty()) { + appendIntsAsSet(sb, "added", v.added); + } + if (!v.removed.isEmpty()) { + if (!v.added.isEmpty()) { + sb.append(' '); + } + appendIntsAsSet(sb, "removed", v.removed); + } + diffs.add(sb.toString()); + } + + private static void appendIntsAsSet(StringBuilder sb, String setName, List<Integer> ints) { + sb.append(setName).append(" {"); + sb.append(ints.stream().map(i -> Integer.toString(i)).collect(Collectors.joining(", "))); + sb.append('}'); + } + +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionDiffCalculator.java new file mode 100644 index 00000000000..3c5b7a8aa4f --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributionDiffCalculator.java @@ -0,0 +1,74 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import java.util.HashSet; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.clustercontroller.core.DistributionDiff.NodeDiff; +import static com.yahoo.vespa.clustercontroller.core.DistributionDiff.IntegerDiff; + +/** + * Utility for computing a "minimal" diff between two arbitrary distribution configs. + */ +public class DistributionDiffCalculator { + + private record GroupNodes(Set<Integer> nodes) { + + NodeDiff sortedDiff(GroupNodes other) { + var added = other.nodes.stream().filter(n -> !nodes.contains(n)).sorted().toList(); + var removed = nodes.stream().filter(n -> !other.nodes.contains(n)).sorted().toList(); + return new NodeDiff(added, removed); + } + + } + + public static DistributionDiff computeDiff(DistributionConfigBundle oldCfg, DistributionConfigBundle newCfg) { + return new DistributionDiff( + computeGroupDiff(oldCfg, newCfg), + IntegerDiff.of(oldCfg.totalLeafGroupCount(), newCfg.totalLeafGroupCount()), + IntegerDiff.of(oldCfg.redundancy(), newCfg.redundancy()), + IntegerDiff.of(oldCfg.searchableCopies(), newCfg.searchableCopies())); + } + + // Returns an ordered mapping from canonical group name to the number of added/removed nodes for + // that group. "Canonical" here means a name that uniquely identifies a group regardless of its + // placement in the topology, i.e. the name is a function of the group's path from the root. + // Adding a new group is represented as if all its nodes were added in one go. + // Conversely, removing an entire group is represented as if all its nodes were removed. + private static TreeMap<String, NodeDiff> computeGroupDiff(DistributionConfigBundle oldCfg, DistributionConfigBundle newCfg) { + var oldGroupNodes = enumerateGroupNodes(oldCfg); + var newGroupNodes = enumerateGroupNodes(newCfg); + + var unionLeafGroupNames = new HashSet<>(oldGroupNodes.keySet()); + unionLeafGroupNames.addAll(newGroupNodes.keySet()); + + var emptySentinel = new GroupNodes(Set.of()); + var diff = new TreeMap<String, NodeDiff>(); + for (String leafName : unionLeafGroupNames) { + var oldG = oldGroupNodes.getOrDefault(leafName, emptySentinel); + var newG = newGroupNodes.getOrDefault(leafName, emptySentinel); + diff.put(leafName, oldG.sortedDiff(newG)); + } + return diff; + } + + // Returns an ordered mapping from canonical group name to the set of distinct node distribution + // keys contained within that group. Note: we assume that no config is ever received that contains + // the same node in different groups; such a config is inherently invalid. + private static TreeMap<String, GroupNodes> enumerateGroupNodes(DistributionConfigBundle cfg) { + var groups = new TreeMap<String, GroupNodes>(); + // Config is already in flattened form for groups, so use it directly. + for (var g : cfg.config().group()) { + // Use group "index" as name, as it encodes the group hierarchy as a dotted string path. + // "invalid" is the fixed index string value of the root group. If the root group contains + // any nodes at all, there will not be any nested groups by definition, as only leaf groups + // may contain nodes. + String name = "invalid".equals(g.index()) ? "root group" : "group %s".formatted(g.index()); + groups.put(name, new GroupNodes(g.nodes().stream().map(n -> n.index()).collect(Collectors.toSet()))); + } + return groups; + } + +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java index d57f32ee3cb..83475fcfe14 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java @@ -11,6 +11,7 @@ import com.yahoo.vdslib.state.State; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -65,6 +66,8 @@ public class EventDiffCalculator { final AnnotatedClusterState toState; final ClusterStateBundle.FeedBlock feedBlockFrom; final ClusterStateBundle.FeedBlock feedBlockTo; + final DistributionConfigBundle distributionConfigFrom; + final DistributionConfigBundle distributionConfigTo; final long currentTime; final long maxMaintenanceGracePeriodTimeMs; @@ -74,6 +77,8 @@ public class EventDiffCalculator { AnnotatedClusterState toState, ClusterStateBundle.FeedBlock feedBlockFrom, ClusterStateBundle.FeedBlock feedBlockTo, + DistributionConfigBundle distributionConfigFrom, + DistributionConfigBundle distributionConfigTo, long currentTime, long maxMaintenanceGracePeriodTimeMs) { this.cluster = cluster; @@ -82,6 +87,8 @@ public class EventDiffCalculator { this.toState = toState; this.feedBlockFrom = feedBlockFrom; this.feedBlockTo = feedBlockTo; + this.distributionConfigFrom = distributionConfigFrom; + this.distributionConfigTo = distributionConfigTo; this.currentTime = currentTime; this.maxMaintenanceGracePeriodTimeMs = maxMaintenanceGracePeriodTimeMs; } @@ -103,13 +110,53 @@ public class EventDiffCalculator { params.toState.getBaselineAnnotatedState(), params.fromState.getFeedBlockOrNull(), params.toState.getFeedBlockOrNull(), + params.fromState.distributionConfig().orElse(null), + params.toState.distributionConfig().orElse(null), params.currentTime, params.maxMaintenanceGracePeriodTimeMs); } private static void emitWholeClusterDiffEvent(final PerStateParams params, final List<Event> events) { - final ClusterState fromState = params.fromState.getClusterState(); - final ClusterState toState = params.toState.getClusterState(); + emitClusterAvailabilityDiffEvent(params, events); + emitClusterFeedBlockDiffEvent(params, events); + emitDistributionConfigDiffEvent(params, events); + } + + private static void emitDistributionConfigDiffEvent(PerStateParams params, List<Event> events) { + if (distributionConfigHasChanged(params)) { + if (params.distributionConfigFrom == null) { + // distributionConfigTo must be non-null + events.add(createClusterEvent( + "Cluster controller is now the authoritative source for distribution config. " + + "Active config: %s".formatted(params.distributionConfigTo.highLevelDescription()), params)); + } else if (params.distributionConfigTo == null) { + events.add(createClusterEvent( + "Cluster controller is no longer the authoritative source for distribution config", params)); + } else { + // Distribution config was present in both old and new; emit a human-readable diff. + String configDiff = DistributionDiffCalculator.computeDiff(params.distributionConfigFrom, params.distributionConfigTo).toString(); + if (!configDiff.isEmpty()) { + events.add(createClusterEvent("Distribution config changed: %s".formatted(configDiff), params)); + } + } + } + } + + private static void emitClusterFeedBlockDiffEvent(PerStateParams params, List<Event> events) { + // TODO should we emit any events when description changes? + if (feedBlockStateHasChanged(params)) { + if (params.feedBlockTo != null) { + events.add(createClusterEvent(String.format("Cluster feed blocked due to resource exhaustion: %s", + params.feedBlockTo.getDescription()), params)); + } else { + events.add(createClusterEvent("Cluster feed no longer blocked", params)); + } + } + } + + private static void emitClusterAvailabilityDiffEvent(PerStateParams params, List<Event> events) { + ClusterState fromState = params.fromState.getClusterState(); + ClusterState toState = params.toState.getClusterState(); if (clusterHasTransitionedToUpState(fromState, toState)) { events.add(createClusterEvent("Enough nodes available for system to become up", params)); @@ -126,15 +173,10 @@ public class EventDiffCalculator { events.add(createClusterEvent("Cluster is down", params)); } } - // TODO should we emit any events when description changes? - if (feedBlockStateHasChanged(params)) { - if (params.feedBlockTo != null) { - events.add(createClusterEvent(String.format("Cluster feed blocked due to resource exhaustion: %s", - params.feedBlockTo.getDescription()), params)); - } else { - events.add(createClusterEvent("Cluster feed no longer blocked", params)); - } - } + } + + private static boolean distributionConfigHasChanged(PerStateParams params) { + return ! Objects.equals(params.distributionConfigFrom, params.distributionConfigTo); } private static boolean feedBlockStateHasChanged(PerStateParams params) { @@ -275,6 +317,8 @@ public class EventDiffCalculator { toDerivedState, null, // Not used in per-space event derivation null, // Ditto + null, // etc. + null, // etc. params.currentTime, params.maxMaintenanceGracePeriodTimeMs); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 5cffa4957c6..6731a8c3861 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -850,6 +850,10 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta return timeNowMs >= firstAllowedStateBroadcast || cluster.allStatesReported(); } + private DistributionConfigBundle distributionConfigIfEnabledOrNull() { + return options.includeDistributionConfigInClusterStateBundles() ? options.distributionConfig() : null; + } + private boolean recomputeClusterStateIfRequired() { boolean stateWasChanged = false; if (mustRecomputeCandidateClusterState()) { @@ -860,6 +864,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta final ClusterStateBundle candidateBundle = ClusterStateBundle.builder(candidate) .bucketSpaces(configuredBucketSpaces) .stateDeriver(createBucketSpaceStateDeriver()) + .distributionConfig(distributionConfigIfEnabledOrNull()) .deferredActivation(options.enableTwoPhaseClusterStateActivation()) .feedBlock(createResourceExhaustionCalculator() .inferContentClusterFeedBlockOrNull(cluster)) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index 44b2c8833db..9aee020a1c7 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -7,6 +7,7 @@ import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vespa.clustercontroller.core.database.DatabaseFactory; import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; +import com.yahoo.vespa.config.content.StorDistributionConfig; import java.time.Duration; import java.util.Collection; @@ -108,7 +109,7 @@ public class FleetControllerOptions { /** Maximum time a node can be missing from slobrok before it is tagged down. */ private final int maxSlobrokDisconnectGracePeriod; - private final Distribution storageDistribution; + private final DistributionConfigBundle distributionConfigBundle; // TODO: Get rid of this by always getting nodes by distribution.getNodes() private final Set<ConfiguredNode> nodes; @@ -133,6 +134,8 @@ public class FleetControllerOptions { private final Function<FleetControllerContext, DatabaseFactory> dbFactoryFn; + private final boolean includeDistributionConfigInClusterStateBundles; + // TODO less impressive length...! private FleetControllerOptions(String clusterName, int fleetControllerIndex, @@ -164,7 +167,7 @@ public class FleetControllerOptions { int minTimeBetweenNewSystemStates, boolean showLocalSystemStatesInEventLog, int maxSlobrokDisconnectGracePeriod, - Distribution storageDistribution, + DistributionConfigBundle distributionConfigBundle, Set<ConfiguredNode> nodes, Duration maxDeferredTaskVersionWaitTime, boolean clusterHasGlobalDocumentTypes, @@ -175,7 +178,8 @@ public class FleetControllerOptions { Map<String, Double> clusterFeedBlockLimit, double clusterFeedBlockNoiseLevel, int maxNumberOfGroupsAllowedToBeDown, - Function<FleetControllerContext, DatabaseFactory> dbFactoryFn) { + Function<FleetControllerContext, DatabaseFactory> dbFactoryFn, + boolean includeDistributionConfigInClusterStateBundles) { this.clusterName = clusterName; this.fleetControllerIndex = fleetControllerIndex; this.fleetControllerCount = fleetControllerCount; @@ -206,7 +210,7 @@ public class FleetControllerOptions { this.minTimeBetweenNewSystemStates = minTimeBetweenNewSystemStates; this.showLocalSystemStatesInEventLog = showLocalSystemStatesInEventLog; this.maxSlobrokDisconnectGracePeriod = maxSlobrokDisconnectGracePeriod; - this.storageDistribution = storageDistribution; + this.distributionConfigBundle = distributionConfigBundle; this.nodes = nodes; this.maxDeferredTaskVersionWaitTime = maxDeferredTaskVersionWaitTime; this.clusterHasGlobalDocumentTypes = clusterHasGlobalDocumentTypes; @@ -218,6 +222,7 @@ public class FleetControllerOptions { this.clusterFeedBlockNoiseLevel = clusterFeedBlockNoiseLevel; this.maxNumberOfGroupsAllowedToBeDown = maxNumberOfGroupsAllowedToBeDown; this.dbFactoryFn = dbFactoryFn; + this.includeDistributionConfigInClusterStateBundles = includeDistributionConfigInClusterStateBundles; } public Duration getMaxDeferredTaskVersionWaitTime() { @@ -349,7 +354,11 @@ public class FleetControllerOptions { } public Distribution storageDistribution() { - return storageDistribution; + return distributionConfigBundle.distribution(); + } + + public DistributionConfigBundle distributionConfig() { + return distributionConfigBundle; } public Set<ConfiguredNode> nodes() { @@ -392,6 +401,10 @@ public class FleetControllerOptions { public Function<FleetControllerContext, DatabaseFactory> dbFactoryFn() { return dbFactoryFn; } + public boolean includeDistributionConfigInClusterStateBundles() { + return this.includeDistributionConfigInClusterStateBundles; + } + public static class Builder { private String clusterName; @@ -424,7 +437,7 @@ public class FleetControllerOptions { private int minTimeBetweenNewSystemStates = 0; private boolean showLocalSystemStatesInEventLog = true; private int maxSlobrokDisconnectGracePeriod = 1000; - private Distribution storageDistribution; + private DistributionConfigBundle distributionConfigBundle; private Set<ConfiguredNode> nodes; private Duration maxDeferredTaskVersionWaitTime = Duration.ofSeconds(30); private boolean clusterHasGlobalDocumentTypes = false; @@ -436,6 +449,7 @@ public class FleetControllerOptions { private double clusterFeedBlockNoiseLevel = 0.01; private int maxNumberOfGroupsAllowedToBeDown = 1; private Function<FleetControllerContext, DatabaseFactory> dbFactoryFn = ZooKeeperDatabaseFactory::new; + private boolean includeDistributionConfigInClusterStateBundles = false; public Builder(String clusterName, Collection<ConfiguredNode> nodes) { this.clusterName = clusterName; @@ -629,8 +643,8 @@ public class FleetControllerOptions { return this; } - public Builder setStorageDistribution(Distribution storageDistribution) { - this.storageDistribution = storageDistribution; + public Builder setDistributionConfig(StorDistributionConfig config) { + this.distributionConfigBundle = DistributionConfigBundle.of(config); return this; } @@ -693,6 +707,11 @@ public class FleetControllerOptions { return this; } + public Builder setIncludeDistributionConfigInClusterStateBundles(boolean includeConfig) { + this.includeDistributionConfigInClusterStateBundles = includeConfig; + return this; + } + public FleetControllerOptions build() { return new FleetControllerOptions(clusterName, index, @@ -724,7 +743,7 @@ public class FleetControllerOptions { minTimeBetweenNewSystemStates, showLocalSystemStatesInEventLog, maxSlobrokDisconnectGracePeriod, - storageDistribution, + distributionConfigBundle, nodes, maxDeferredTaskVersionWaitTime, clusterHasGlobalDocumentTypes, @@ -735,7 +754,8 @@ public class FleetControllerOptions { clusterFeedBlockLimit, clusterFeedBlockNoiseLevel, maxNumberOfGroupsAllowedToBeDown, - dbFactoryFn); + dbFactoryFn, + includeDistributionConfigInClusterStateBundles); } public static Builder copy(FleetControllerOptions options) { @@ -770,7 +790,7 @@ public class FleetControllerOptions { builder.minTimeBetweenNewSystemStates = options.minTimeBetweenNewSystemStates; builder.showLocalSystemStatesInEventLog = options.showLocalSystemStatesInEventLog; builder.maxSlobrokDisconnectGracePeriod = options.maxSlobrokDisconnectGracePeriod; - builder.storageDistribution = options.storageDistribution; + builder.distributionConfigBundle = options.distributionConfigBundle; builder.nodes = Set.copyOf(options.nodes); builder.maxDeferredTaskVersionWaitTime = options.maxDeferredTaskVersionWaitTime; builder.clusterHasGlobalDocumentTypes = options.clusterHasGlobalDocumentTypes; @@ -782,6 +802,7 @@ public class FleetControllerOptions { builder.clusterFeedBlockNoiseLevel = options.clusterFeedBlockNoiseLevel; builder.maxNumberOfGroupsAllowedToBeDown = options.maxNumberOfGroupsAllowedToBeDown; builder.dbFactoryFn = options.dbFactoryFn; + builder.includeDistributionConfigInClusterStateBundles = options.includeDistributionConfigInClusterStateBundles; return builder; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodec.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodec.java index 3a6b9a635fa..7b52a8e7773 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodec.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodec.java @@ -8,9 +8,13 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vespa.clustercontroller.core.AnnotatedClusterState; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; +import com.yahoo.vespa.clustercontroller.core.DistributionConfigBundle; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.content.StorDistributionConfig; import java.util.HashMap; import java.util.Map; @@ -28,6 +32,7 @@ import java.util.Map; */ public class SlimeClusterStateBundleCodec implements ClusterStateBundleCodec, EnvelopedClusterStateBundleCodec { + // TODO zstd instead. Compression type already encoded on wire private static final Compressor compressor = new Compressor(CompressionType.LZ4, 3, 0.90, 1024); @Override @@ -50,6 +55,10 @@ public class SlimeClusterStateBundleCodec implements ClusterStateBundleCodec, En feedBlock.setString("description", stateBundle.getFeedBlock().get().getDescription()); } + stateBundle.distributionConfig().ifPresent(cfg -> { + SlimeUtils.copyObject(cfg.precomputedSlimeRepr().get(), root.setObject("distribution-config")); + }); + Compressor.Compression compression = BinaryFormat.encode_and_compress(slime, compressor); return EncodedClusterStateBundle.fromCompressionBuffer(compression); } @@ -75,8 +84,21 @@ public class SlimeClusterStateBundleCodec implements ClusterStateBundleCodec, En feedBlock = ClusterStateBundle.FeedBlock.blockedWithDescription(fb.field("description").asString()); } + DistributionConfigBundle distributionConfig = null; + Inspector dc = root.field("distribution-config"); + if (dc.valid()) { + // ConfigPayload works on full Slime objects and not Inspectors, so we have to copy out + // the subtree prior to decode. This is not code used in a hot path, so this should be fine. + Slime cfgSlime = new Slime(); + SlimeUtils.copyObject(dc, cfgSlime.setObject()); + // TODO toInstance transitively invokes reflection which _mutates_ field access attributes, + // toggling `accessible` on and off. Sounds like an inherent latent race condition... + distributionConfig = DistributionConfigBundle.of( + new ConfigPayload(cfgSlime).toInstance(StorDistributionConfig.class, null)); + } + return ClusterStateBundle.of(AnnotatedClusterState.withoutAnnotations(baseline), derivedStates, - feedBlock, deferredActivation); + feedBlock, distributionConfig, deferredActivation); } // Technically the Slime enveloping could be its own class that is bundle codec independent, but diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index 550ef835a95..b3ec6f8ef7e 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -61,7 +61,7 @@ public class ClusterFeedBlockTest extends FleetControllerTest { private static FleetControllerOptions.Builder createOptions(Map<String, Double> feedBlockLimits, double clusterFeedBlockNoiseLevel) { return defaultOptions() - .setStorageDistribution(DistributionBuilder.forFlatCluster(NODE_COUNT)) + .setDistributionConfig(DistributionBuilder.configForFlatCluster(NODE_COUNT)) .setNodes(new HashSet<>(DistributionBuilder.buildConfiguredNodes(NODE_COUNT))) .setClusterFeedBlockEnabled(true) .setClusterFeedBlockLimit(feedBlockLimits) diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java index 760967a68dd..405eeb9d9a8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java @@ -6,6 +6,7 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.config.content.StorDistributionConfig; import org.junit.jupiter.api.Test; import java.util.Set; @@ -60,6 +61,12 @@ public class ClusterStateBundleTest { .deriveAndBuild(); } + private static ClusterStateBundle createTestBundleWithDistributionConfig(StorDistributionConfig config) { + return createTestBundleBuilder(false) + .distributionConfig(config) + .deriveAndBuild(); + } + private static ClusterStateBundle createTestBundle() { return createTestBundle(true); } @@ -130,6 +137,20 @@ public class ClusterStateBundleTest { } @Test + void similarity_test_considers_distribution_config() { + var bundle5Nodes = createTestBundleWithDistributionConfig(DistributionBuilder.configForFlatCluster(5)); + var bundle5Nodes2 = createTestBundleWithDistributionConfig(DistributionBuilder.configForFlatCluster(5)); + var bundle6Nodes = createTestBundleWithDistributionConfig(DistributionBuilder.configForFlatCluster(6)); + var bundle2x3Grouped = createTestBundleWithDistributionConfig(DistributionBuilder.configForHierarchicCluster( + DistributionBuilder.withGroups(2).eachWithNodeCount(3))); + assertTrue(bundle5Nodes.similarTo(bundle5Nodes)); + assertTrue(bundle5Nodes.similarTo(bundle5Nodes2)); + assertTrue(bundle5Nodes2.similarTo(bundle5Nodes)); + assertFalse(bundle5Nodes.similarTo(bundle6Nodes)); + assertFalse(bundle6Nodes.similarTo(bundle2x3Grouped)); + } + + @Test void feed_block_state_is_available() { var nonBlockingBundle = createTestBundle(false); var blockingBundle = createTestBundleWithFeedBlock("foo"); @@ -171,6 +192,17 @@ public class ClusterStateBundleTest { } @Test + void toString_with_distribution_config_includes_high_level_information() { + var bundle = createTestBundleWithDistributionConfig(DistributionBuilder.configForHierarchicCluster( + DistributionBuilder.withGroups(2).eachWithNodeCount(3))); + assertThat(bundle.toString(), equalTo("ClusterStateBundle('distributor:2 storage:2', " + + "default 'distributor:2 storage:2', " + + "global 'distributor:2 storage:2', " + + "narnia 'distributor:2 .0.s:d storage:2', " + + "distribution config: 6 nodes; 2 groups; redundancy 2; searchable-copies 0)")); + } + + @Test void toString_without_derived_states_specifies_deferred_activation_iff_set() { var bundle = ClusterStateBundle.ofBaselineOnly(annotatedStateOf("distributor:2 storage:2"), null, true); assertThat(bundle.toString(), equalTo("ClusterStateBundle('distributor:2 storage:2' (deferred activation))")); @@ -248,4 +280,11 @@ public class ClusterStateBundleTest { assertEquals(bundle, derived); } + @Test + void cloning_preserves_distribution_config() { + var bundle = createTestBundleWithDistributionConfig(DistributionBuilder.configForFlatCluster(5)); + var derived = bundle.cloneWithMapper(Function.identity()); + assertEquals(bundle, derived); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBuilder.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBuilder.java index 12db7288d4a..44aeba6b7cf 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBuilder.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBuilder.java @@ -12,42 +12,42 @@ import java.util.stream.IntStream; public class DistributionBuilder { // TODO support nested groups - static class GroupBuilder { + public static class GroupBuilder { final int groupCount; List<Integer> groupsWithNodeCount; - GroupBuilder(int groupCount) { + public GroupBuilder(int groupCount) { this.groupCount = groupCount; } - GroupBuilder(int... nodeCounts) { + public GroupBuilder(int... nodeCounts) { this.groupCount = nodeCounts.length; this.groupsWithNodeCount = IntStream.of(nodeCounts).boxed() .toList(); } - GroupBuilder eachWithNodeCount(int nodeCount) { + public GroupBuilder eachWithNodeCount(int nodeCount) { groupsWithNodeCount = IntStream.range(0, groupCount) .map(i -> nodeCount).boxed() .toList(); return this; } - int totalNodeCount() { + public int totalNodeCount() { return groupsWithNodeCount.stream().reduce(0, Integer::sum); } - String groupDistributionSpec() { + public String groupDistributionSpec() { return IntStream.range(0, groupCount).mapToObj(i -> "1") .collect(Collectors.joining("|")) + "|*"; } } - static GroupBuilder withGroups(int groups) { + public static GroupBuilder withGroups(int groups) { return new GroupBuilder(groups); } - static GroupBuilder withGroupNodes(int... nodeCounts) { + public static GroupBuilder withGroupNodes(int... nodeCounts) { return new GroupBuilder(nodeCounts); } @@ -57,6 +57,10 @@ public class DistributionBuilder { .toList(); } + static List<ConfiguredNode> buildConfiguredNodes(Collection<Integer> nodes) { + return nodes.stream().map(i -> new ConfiguredNode(i, false)).toList(); + } + private static StorDistributionConfig.Group.Nodes.Builder configuredNode(ConfiguredNode node) { StorDistributionConfig.Group.Nodes.Builder builder = new StorDistributionConfig.Group.Nodes.Builder(); builder.index(node.index()); @@ -64,25 +68,41 @@ public class DistributionBuilder { } private static StorDistributionConfig.Group.Builder configuredGroup( - String name, int index, Collection<ConfiguredNode> nodes) { + String name, String index, Collection<ConfiguredNode> nodes) { StorDistributionConfig.Group.Builder builder = new StorDistributionConfig.Group.Builder(); builder.name(name); - builder.index(Integer.toString(index)); + builder.index(index); nodes.forEach(n -> builder.nodes(configuredNode(n))); return builder; } - public static Distribution forFlatCluster(int nodeCount) { - Collection<ConfiguredNode> nodes = buildConfiguredNodes(nodeCount); + private static StorDistributionConfig.Group.Builder configuredGroup( + String name, int index, Collection<ConfiguredNode> nodes) { + return configuredGroup(name, Integer.toString(index), nodes); + } + public static StorDistributionConfig configForFlatCluster(int redundancy, int searchableCopies, Collection<Integer> nodes) { StorDistributionConfig.Builder configBuilder = new StorDistributionConfig.Builder(); - configBuilder.redundancy(2); - configBuilder.group(configuredGroup("bar", 0, nodes)); + configBuilder.redundancy(redundancy); + configBuilder.ready_copies(searchableCopies); + configBuilder.group(configuredGroup("invalid", "invalid", buildConfiguredNodes(nodes))); - return new Distribution(new StorDistributionConfig(configBuilder)); + return new StorDistributionConfig(configBuilder); } - static Distribution forHierarchicCluster(GroupBuilder root) { + public static StorDistributionConfig configForFlatCluster(int redundancy, int searchableCopies, int nodeCount) { + return configForFlatCluster(redundancy, searchableCopies, IntStream.range(0, nodeCount).boxed().toList()); + } + + public static StorDistributionConfig configForFlatCluster(int nodeCount) { + return configForFlatCluster(2, 0, nodeCount); + } + + public static Distribution forFlatCluster(int nodeCount) { + return new Distribution(configForFlatCluster(nodeCount)); + } + + public static StorDistributionConfig configForHierarchicCluster(GroupBuilder root) { List<ConfiguredNode> nodes = buildConfiguredNodes(root.totalNodeCount()); StorDistributionConfig.Builder configBuilder = new StorDistributionConfig.Builder(); @@ -103,6 +123,10 @@ public class DistributionBuilder { offset += nodeCount; } - return new Distribution(new StorDistributionConfig(configBuilder)); + return new StorDistributionConfig(configBuilder); + } + + public static Distribution forHierarchicCluster(GroupBuilder root) { + return new Distribution(configForHierarchicCluster(root)); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionDiffCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionDiffCalculatorTest.java new file mode 100644 index 00000000000..b9efda78757 --- /dev/null +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionDiffCalculatorTest.java @@ -0,0 +1,149 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import com.yahoo.vdslib.distribution.ConfiguredNode; +import com.yahoo.vdslib.distribution.Distribution; +import com.yahoo.vespa.config.content.StorDistributionConfig; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DistributionDiffCalculatorTest { + + private static DistributionConfigBundle flat(int redundancy, int searchableCopies, int nodes) { + return DistributionConfigBundle.of(DistributionBuilder.configForFlatCluster(redundancy, searchableCopies, nodes)); + } + + private static DistributionConfigBundle flat(int redundancy, int searchableCopies, Collection<Integer> nodes) { + return DistributionConfigBundle.of(DistributionBuilder.configForFlatCluster(redundancy, searchableCopies, nodes)); + } + + private static DistributionConfigBundle grouped(int... nodesInGroups) { + var gb = new DistributionBuilder.GroupBuilder(nodesInGroups); + return DistributionConfigBundle.of(DistributionBuilder.configForHierarchicCluster(gb)); + } + + private static String diff(DistributionConfigBundle oldCfg, DistributionConfigBundle newCfg) { + return DistributionDiffCalculator.computeDiff(oldCfg, newCfg).toString(); + } + + @Test + void no_diff_returns_empty_string() { + assertEquals("", diff(flat(1, 1, 3), flat(1, 1, 3))); + assertEquals("", diff(grouped(1, 2, 3), grouped(1, 2, 3))); + } + + @Test + void changed_redundancy_or_searchable_copies_is_reflected() { + assertEquals("redundancy: 1 -> 2", diff(flat(1, 1, 3), flat(2, 1, 3))); + assertEquals("redundancy: 2 -> 1", diff(flat(2, 1, 3), flat(1, 1, 3))); + assertEquals("searchable-copies: 1 -> 2", diff(flat(1, 1, 3), flat(1, 2, 3))); + assertEquals("searchable-copies: 2 -> 1", diff(flat(1, 2, 3), flat(1, 1, 3))); + assertEquals("redundancy: 4 -> 5, searchable-copies: 1 -> 2", diff(flat(4, 1, 7), flat(5, 2, 7))); + } + + @Test + void flat_group_topology_renders_changed_node_sets() { + assertEquals("root group: added {1}", diff(flat(1, 1, Set.of(0)), flat(1, 1, Set.of(0, 1)))); + assertEquals("root group: added {2, 3}", diff(flat(1, 1, Set.of(1)), flat(1, 1, Set.of(1, 2, 3)))); + assertEquals("root group: removed {3}", diff(flat(1, 1, Set.of(2, 3)), flat(1, 1, Set.of(2)))); + assertEquals("root group: added {1} removed {0}", diff(flat(1, 1, Set.of(0)), flat(1, 1, Set.of(1)))); + } + + @Test + void grouped_topology_renders_diff_sets_across_groups() { + // The test group builder assigns linearly increasing node indexes, so changing the + // count of nodes in one group affects the indices of the nodes in subsequent groups. + // It also starts group name indexes at 1, not 0. + // group 1 {0} --> group 1 {0}, group 2 {1} + assertEquals("groups: 1 -> 2, group 2: added {1}", diff(grouped(1), grouped(1, 1))); + // group 1 {0} --> group 1 {0}, group 2 {1, 2, 3} + assertEquals("groups: 1 -> 2, group 2: added {1, 2, 3}", diff(grouped(1), grouped(1, 3))); + assertEquals("groups: 1 -> 3, group 2: added {1}, group 3: added {2}", diff(grouped(1), grouped(1, 1, 1))); + assertEquals("groups: 3 -> 1, group 2: removed {1}, group 3: removed {2}", diff(grouped(1, 1, 1), grouped(1))); + // Partial overlap + assertEquals("groups: 2 -> 4, group 1: removed {1}, group 2: added {1} " + + "removed {2, 3}, group 3: added {2}, group 4: added {3}", diff(grouped(2, 2), grouped(1, 1, 1, 1))); + + // Flat <--> grouped + assertEquals("groups: 1 -> 2, group 1: added {0}, group 2: added {1}, root group: removed {0, 1}", + diff(flat(2, 0, Set.of(0, 1)), grouped(1, 1))); + assertEquals("groups: 2 -> 1, group 1: removed {0}, group 2: removed {1}, root group: added {0, 1}", + diff(grouped(1, 1), flat(2, 0, Set.of(0, 1)))); + } + + private static StorDistributionConfig.Group.Nodes.Builder configuredNode(int nodeIndex) { + StorDistributionConfig.Group.Nodes.Builder builder = new StorDistributionConfig.Group.Nodes.Builder(); + builder.index(nodeIndex); + return builder; + } + + private static StorDistributionConfig.Group.Builder groupBuilder(String name, String index, String partitions, Set<Integer> nodes) { + var builder = new StorDistributionConfig.Group.Builder(); + builder.name(name); + builder.index(index); + builder.partitions(partitions); + nodes.forEach(n -> builder.nodes(configuredNode(n))); + return builder; + } + + private static StorDistributionConfig.Group.Builder groupBuilder(String name, String index, String partitions) { + return groupBuilder(name, index, partitions, Set.of()); + } + + private static StorDistributionConfig nestedGroupConfig2x2(Set<Integer> g1Nodes, Set<Integer> g2Nodes, + Set<Integer> g3Nodes, Set<Integer> g4Nodes) { + var configBuilder = new StorDistributionConfig.Builder(); + configBuilder.redundancy(2); + + // Example group structure for g1={0, 1, 2}, g2={3, 4, 5}, g3={6, 7, 8}, g4={9, 10, 11} + // root + // |-- group0 + // | |-- group0_0 + // | | |-- node 0 + // | | |-- node 1 + // | | `-- node 2 + // | `-- group0_1 + // | |-- node 3 + // | |-- node 4 + // | `-- node 5 + // |-- group1 + // | |-- group1_0 + // | | |-- node 6 + // | | |-- node 7 + // | | `-- node 8 + // | `-- group1_1 + // | |-- node 9 + // | |-- node 10 + // | `-- node 11 + configBuilder.group(groupBuilder("invalid", "invalid", "*|*")); + configBuilder.group(groupBuilder("group0", "0", "1|*")); + configBuilder.group(groupBuilder("group0_0", "0.0", "", g1Nodes)); + configBuilder.group(groupBuilder("group0_1", "0.1", "", g2Nodes)); + configBuilder.group(groupBuilder("group1", "1", "1|*")); + configBuilder.group(groupBuilder("group1_0", "1.0", "", g3Nodes)); + configBuilder.group(groupBuilder("group1_1", "1.1", "", g4Nodes)); + + return new StorDistributionConfig(configBuilder); + } + + @Test + void nested_grouped_topology_renders_with_canonical_group_names() { + var cfg1 = DistributionConfigBundle.of(nestedGroupConfig2x2(Set.of(0, 1, 2), Set.of(3, 4, 5), + Set.of(6, 7, 8), Set.of(9, 10, 11))); + // Add 1 node to 0.1 and remove 1 node from 1.1 + var cfg2 = DistributionConfigBundle.of(nestedGroupConfig2x2(Set.of(0, 1, 2), Set.of(3, 4, 5, 12), + Set.of(6, 7, 8), Set.of(9, 11))); + + assertEquals("group 0.1: added {12}, group 1.1: removed {10}", diff(cfg1, cfg2)); + // Inverse + assertEquals("group 0.1: removed {12}, group 1.1: added {10}", diff(cfg2, cfg1)); + } + +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java index 2a995996a26..5577540eaba 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java @@ -35,6 +35,8 @@ public class EventDiffCalculatorTest { Map<String, AnnotatedClusterState.Builder> derivedAfter = new HashMap<>(); ClusterStateBundle.FeedBlock feedBlockBefore = null; ClusterStateBundle.FeedBlock feedBlockAfter = null; + DistributionConfigBundle distributionConfigBefore = null; + DistributionConfigBundle distributionConfigAfter = null; long currentTimeMs = 0; long maxMaintenanceGracePeriodTimeMs = 10_000; @@ -98,6 +100,14 @@ public class EventDiffCalculatorTest { this.feedBlockAfter = feedBlock; return this; } + EventFixture distributionConfigBefore(DistributionConfigBundle configBefore) { + this.distributionConfigBefore = configBefore; + return this; + } + EventFixture distributionConfigAfter(DistributionConfigBundle configAfter) { + this.distributionConfigAfter = configAfter; + return this; + } private static AnnotatedClusterState.Builder getBuilder(Map<String, AnnotatedClusterState.Builder> derivedStates, String bucketSpace) { return derivedStates.computeIfAbsent(bucketSpace, key -> new AnnotatedClusterState.Builder()); } @@ -106,8 +116,10 @@ public class EventDiffCalculatorTest { return EventDiffCalculator.computeEventDiff( EventDiffCalculator.params() .cluster(clusterFixture.cluster()) - .fromState(ClusterStateBundle.of(baselineBefore.build(), toDerivedStates(derivedBefore), feedBlockBefore, false)) - .toState(ClusterStateBundle.of(baselineAfter.build(), toDerivedStates(derivedAfter), feedBlockAfter, false)) + .fromState(ClusterStateBundle.of(baselineBefore.build(), toDerivedStates(derivedBefore), + feedBlockBefore, distributionConfigBefore, false)) + .toState(ClusterStateBundle.of(baselineAfter.build(), toDerivedStates(derivedAfter), + feedBlockAfter, distributionConfigAfter, false)) .currentTimeMs(currentTimeMs) .maxMaintenanceGracePeriodTimeMs(maxMaintenanceGracePeriodTimeMs)); } @@ -550,4 +562,63 @@ public class EventDiffCalculatorTest { nodeEventForBaseline()))); } + private static DistributionConfigBundle flatClusterDistribution(int nodes) { + return DistributionConfigBundle.of(DistributionBuilder.configForFlatCluster(nodes)); + } + + @Test + void enabling_distribution_config_in_state_bundle_emits_cluster_level_event() { + var fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .clusterStateAfter("distributor:3 storage:3") + .distributionConfigBefore(null) + .distributionConfigAfter(flatClusterDistribution(3)); + + var events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(1)); + assertThat(events, hasItem(clusterEventWithDescription( + "Cluster controller is now the authoritative source for distribution config. " + + "Active config: 3 nodes; 1 groups; redundancy 2; searchable-copies 0"))); + } + + @Test + void disabling_distribution_config_in_state_bundle_emits_cluster_level_event() { + var fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .clusterStateAfter("distributor:3 storage:3") + .distributionConfigBefore(flatClusterDistribution(3)) + .distributionConfigAfter(null); + + var events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(1)); + assertThat(events, hasItem(clusterEventWithDescription( + "Cluster controller is no longer the authoritative source for distribution config"))); + } + + @Test + void changed_distribution_config_emits_event_with_config_diff() { + var fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .clusterStateAfter("distributor:3 storage:3") + .distributionConfigBefore(flatClusterDistribution(3)) + .distributionConfigAfter(flatClusterDistribution(5)); + + var events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(1)); + assertThat(events, hasItem(clusterEventWithDescription( + "Distribution config changed: root group: added {3, 4}"))); + } + + @Test + void unchanged_distribution_config_does_not_emit_event() { + var fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .clusterStateAfter("distributor:3 storage:3") + .distributionConfigBefore(flatClusterDistribution(3)) + .distributionConfigAfter(flatClusterDistribution(3)); + + var events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(0)); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index 688d82e5ebb..442304e286f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -85,7 +85,7 @@ public abstract class FleetControllerTest implements Waiter { protected static FleetControllerOptions.Builder defaultOptions(Collection<ConfiguredNode> nodes) { var builder = new FleetControllerOptions.Builder("mycluster", nodes); builder.enableTwoPhaseClusterStateActivation(true); // Enable by default, tests can explicitly disable. - builder.setStorageDistribution(DistributionBuilder.forFlatCluster(builder.nodes().size())); + builder.setDistributionConfig(DistributionBuilder.configForFlatCluster(builder.nodes().size())); builder.setZooKeeperServerAddress("localhost:2181"); return builder; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java index 477fed5e170..6d08bb0a449 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java @@ -16,7 +16,7 @@ public class GroupAutoTakedownLiveConfigTest extends FleetControllerTest { private static FleetControllerOptions.Builder createOptions(DistributionBuilder.GroupBuilder groupBuilder, double minNodeRatio) { return defaultOptions() - .setStorageDistribution(DistributionBuilder.forHierarchicCluster(groupBuilder)) + .setDistributionConfig(DistributionBuilder.configForHierarchicCluster(groupBuilder)) .setNodes(new HashSet<>(DistributionBuilder.buildConfiguredNodes(groupBuilder.totalNodeCount()))) .setMinNodeRatioPerGroup(minNodeRatio) .setMaxTransitionTime(NodeType.DISTRIBUTOR, 0) @@ -37,7 +37,7 @@ public class GroupAutoTakedownLiveConfigTest extends FleetControllerTest { FleetControllerOptions.Builder builder = FleetControllerOptions.Builder.copy(options) .setNodes(new HashSet<>(DistributionBuilder.buildConfiguredNodes(groupBuilder.totalNodeCount()))) - .setStorageDistribution(DistributionBuilder.forHierarchicCluster(groupBuilder)); + .setDistributionConfig(DistributionBuilder.configForHierarchicCluster(groupBuilder)); updateConfigLive(builder.build()); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java index c713cd842f5..327c634dae6 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java @@ -10,7 +10,6 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.vdslib.distribution.ConfiguredNode; -import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; @@ -439,7 +438,7 @@ public class RpcServerTest extends FleetControllerTest { @Test void testSetNodeStateOutOfRange() throws Exception { FleetControllerOptions.Builder options = defaultOptions(); - options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(2, 10))); + options.setDistributionConfig(DistributionBuilder.configForFlatCluster(10)); setUpFleetController(timer, options); setUpVdsNodes(timer); waitForStableSystem(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index 9e9206526e5..e8084e1fd40 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -8,6 +8,7 @@ import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter; +import com.yahoo.vespa.config.content.StorDistributionConfig; import org.junit.jupiter.api.Test; import java.time.Duration; @@ -18,6 +19,7 @@ import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -1249,7 +1251,56 @@ public class StateChangeTest extends FleetControllerTest { Event: storage.2: Altered node state in cluster state from 'U' to 'I, i 0.100 (read)' Event: storage.2: Altered min distribution bit count from 16 to 17 """); + } + + class DistributionConfigInBundleFixture { + StorDistributionConfig bootstrapConfig; + FleetControllerOptions.Builder options; + + DistributionConfigInBundleFixture(boolean enableConfigInBundle) throws Exception { + bootstrapConfig = DistributionBuilder.configForFlatCluster(7); + options = defaultOptions() + .setIncludeDistributionConfigInClusterStateBundles(enableConfigInBundle) + .setDistributionConfig(bootstrapConfig); + initialize(options); + ctrl.tick(); + } + + void updateWithNewConfig(StorDistributionConfig newConfig) throws Exception { + ctrl.updateOptions(options.setDistributionConfig(newConfig).build()); + ctrl.tick(); // Propagate options internally + ctrl.tick(); // New options actually take effect + } + } + + @Test + void cluster_state_bundles_do_not_contain_distribution_config_if_feature_is_disabled() throws Exception { + var f = new DistributionConfigInBundleFixture(false); + var bundleCfg = ctrl.getClusterStateBundle().distributionConfig(); + assertFalse(bundleCfg.isPresent()); + } + @Test + void cluster_state_bundles_contain_distribution_config_if_feature_is_enabled() throws Exception { + var f = new DistributionConfigInBundleFixture(true); + var bundleCfg = ctrl.getClusterStateBundle().distributionConfig(); + assertTrue(bundleCfg.isPresent()); + assertEquals(f.bootstrapConfig, bundleCfg.get().config()); + } + + @Test + void distribution_config_change_triggers_new_versioned_state_bundle_with_updated_config() throws Exception { + var f = new DistributionConfigInBundleFixture(true); + int oldVersion = ctrl.getClusterStateBundle().getVersion(); + + var newDistrCfg = DistributionBuilder.configForFlatCluster(8); + f.updateWithNewConfig(newDistrCfg); + + int newVersion = ctrl.getClusterStateBundle().getVersion(); + assertThat(newVersion, greaterThan(oldVersion)); + var bundleCfg = ctrl.getClusterStateBundle().distributionConfig(); + assertTrue(bundleCfg.isPresent()); + assertEquals(newDistrCfg, bundleCfg.get().config()); } private static abstract class MockTask extends RemoteClusterControllerTask { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodecTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodecTest.java index b9fbd2e2ee1..902129e9e4c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodecTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodecTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.clustercontroller.core.rpc; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundleUtil; +import com.yahoo.vespa.clustercontroller.core.DistributionBuilder; import com.yahoo.vespa.clustercontroller.core.StateMapping; import org.junit.jupiter.api.Test; @@ -96,4 +97,14 @@ public class SlimeClusterStateBundleCodecTest { assertThat(roundtripEncode(stateBundle), equalTo(stateBundle)); } + @Test + void can_roundtrip_encode_bundle_with_distribution_config() { + var stateBundle = ClusterStateBundleUtil.makeBundleBuilder("distributor:2 storage:2") + .distributionConfig(DistributionBuilder.configForHierarchicCluster( + DistributionBuilder.withGroups(2).eachWithNodeCount(3))) + .deriveAndBuild(); + + assertThat(roundtripEncode(stateBundle), equalTo(stateBundle)); + } + } diff --git a/cmake/vespaTargets.cmake b/cmake/vespaTargets.cmake index 00c691db6e8..9a3a01454a2 100644 --- a/cmake/vespaTargets.cmake +++ b/cmake/vespaTargets.cmake @@ -3,5 +3,5 @@ # This file is included from vespaConfig.cmake and contains a list of # targets that are imported from vespa. -add_library(searchlib SHARED IMPORTED) -set_target_properties(searchlib PROPERTIES IMPORTED_LOCATION ${VESPA_HOME}/lib64/libsearchlib.so INTERFACE_INCLUDE_DIRECTORIES ${VESPA_INCLUDE_DIR}) +add_library(vespa_searchlib SHARED IMPORTED) +set_target_properties(vespa_searchlib PROPERTIES IMPORTED_LOCATION ${VESPA_HOME}/lib64/libvespa_searchlib.so INTERFACE_INCLUDE_DIRECTORIES ${VESPA_INCLUDE_DIR}) diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java index c04874f2b16..6ce08250649 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java @@ -288,7 +288,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { /** Returns whether this instance deploys to the given zone, either implicitly or explicitly */ public boolean deploysTo(Environment environment, RegionName region) { - return zones().stream().anyMatch(zone -> zone.concerns(environment, Optional.of(region))); + return zones().stream().anyMatch(zone -> zone.concerns(environment, Optional.ofNullable(region))); } /** Returns the zone endpoint specified for the given region, or empty. */ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index fdaa7d57074..2ab7c128046 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -223,11 +223,16 @@ public class DeploymentSpec { * 5. None of the above apply, and the default of a publicly visible endpoint is used. */ public ZoneEndpoint zoneEndpoint(InstanceName instance, ZoneId zone, ClusterSpec.Id cluster) { - // TODO: look up endpoints from <dev> tag, or so, if we're to support non-prod settings. if ( zone.environment().isTest() && instances().stream() .anyMatch(spec -> spec.zoneEndpoints().getOrDefault(cluster, Map.of()).values().stream() .anyMatch(endpoint -> ! endpoint.isPublicEndpoint()))) return ZoneEndpoint.privateEndpoint; + + if (zone.environment().isManuallyDeployed()) + return instance(instance).filter(spec -> spec.deploysTo(zone.environment(), zone.region())) + .map(spec -> spec.zoneEndpoints().getOrDefault(cluster, Map.of()).get(null)) + .orElse(ZoneEndpoint.defaultEndpoint); + if (zone.environment() != Environment.prod) return ZoneEndpoint.defaultEndpoint; return instance(instance).flatMap(spec -> spec.zoneEndpoint(zone, cluster)) .orElse(ZoneEndpoint.defaultEndpoint); diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index a3df216eea7..d950970f0bb 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -1232,6 +1232,19 @@ public class DeploymentSpecTest { <allow with='gcp-service-connect' project='nine' /> </endpoint> </endpoints> + <dev /> + </instance> + <instance id='other'> + <dev /> + <perf /> + <endpoints> + <endpoint container-id='pew' type='private' /> + </endpoints> + </instance> + <instance id='last'> + <endpoints> + <endpoint container-id='pew' type='private' /> + </endpoints> </instance> </deployment>"""); @@ -1249,6 +1262,8 @@ public class DeploymentSpecTest { var zone = from(prod, RegionName.from("us-east")); var testZone = from(test, RegionName.from("us-east")); + var devZone = from(dev, RegionName.from("us-east")); + var perfZone = from(perf, RegionName.from("us-east")); assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.from("custom"), zone, ClusterSpec.Id.from("bax"))); assertEquals(ZoneEndpoint.defaultEndpoint, @@ -1260,9 +1275,29 @@ public class DeploymentSpecTest { assertEquals(ZoneEndpoint.privateEndpoint, spec.zoneEndpoint(InstanceName.from("default"), testZone, ClusterSpec.Id.from("froz"))); + assertEquals(ZoneEndpoint.defaultEndpoint, + spec.zoneEndpoint(InstanceName.from("default"), devZone, ClusterSpec.Id.from("bax"))); + assertEquals(ZoneEndpoint.privateEndpoint, + spec.zoneEndpoint(InstanceName.from("default"), devZone, ClusterSpec.Id.from("froz"))); + + assertEquals(ZoneEndpoint.defaultEndpoint, + spec.zoneEndpoint(InstanceName.from("default"), perfZone, ClusterSpec.Id.from("bax"))); + assertEquals(ZoneEndpoint.defaultEndpoint, + spec.zoneEndpoint(InstanceName.from("default"), perfZone, ClusterSpec.Id.from("froz"))); + assertEquals(new ZoneEndpoint(false, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "barn"), new AllowedUrn(AccessType.gcpServiceConnect, "nine"))), spec.zoneEndpoint(InstanceName.from("default"), zone, ClusterSpec.Id.from("froz"))); + + assertEquals(new ZoneEndpoint(true, true, List.of()), + spec.zoneEndpoint(InstanceName.from("other"), devZone, ClusterSpec.Id.from("pew"))); + assertEquals(new ZoneEndpoint(true, true, List.of()), + spec.zoneEndpoint(InstanceName.from("other"), perfZone, ClusterSpec.Id.from("pew"))); + + assertEquals(ZoneEndpoint.defaultEndpoint, + spec.zoneEndpoint(InstanceName.from("last"), devZone, ClusterSpec.Id.from("pew"))); + assertEquals(ZoneEndpoint.defaultEndpoint, + spec.zoneEndpoint(InstanceName.from("last"), perfZone, ClusterSpec.Id.from("pew"))); } @Test diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 133195c7039..e797c92ee52 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -87,6 +87,7 @@ public class ConfigserverCluster extends TreeConfigProducer builder.dynamicReconfiguration(options.hostedVespa().orElse(false)); builder.snapshotMethod(options.zooKeeperSnapshotMethod()); + builder.juteMaxBuffer(options.zookeeperJuteMaxBuffer()); } @Override @@ -197,6 +198,7 @@ public class ConfigserverCluster extends TreeConfigProducer builder.server(curatorBuilder); } builder.zookeeperLocalhostAffinity(true); + builder.juteMaxBuffer(options.zookeeperJuteMaxBuffer()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index ec0fa973d6c..5b5914b3012 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -38,5 +38,6 @@ public interface CloudConfigOptions { Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); String zooKeeperSnapshotMethod(); + Integer zookeeperJuteMaxBuffer(); // in bytes } diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java index 6252fe62a1d..47d3c64d351 100644 --- a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java @@ -28,7 +28,6 @@ import static org.junit.jupiter.api.Assertions.*; public class SchemaImporterTestCase extends AbstractSchemaTestCase { @Test - @SuppressWarnings("deprecation") void testSimpleImporting() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder sb = new ApplicationBuilder(rankProfileRegistry, new QueryProfileRegistry()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index 899e92d1111..6d8a8b1b9a0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; -import java.util.Objects; import java.util.Optional; /** @@ -18,7 +17,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); - private String zooKeeperSnapshotMethod = "gz"; + private static final String zooKeeperSnapshotMethod = "gz"; @Override public Optional<Integer> rpcPort() { @@ -111,6 +110,9 @@ public class TestOptions implements CloudConfigOptions { @Override public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } + @Override + public Integer zookeeperJuteMaxBuffer() { return 12345; } + public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -136,10 +138,4 @@ public class TestOptions implements CloudConfigOptions { return this; } - public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { - Objects.requireNonNull(snapshotMethod); - this.zooKeeperSnapshotMethod = snapshotMethod; - return this; - } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index 672bba83e87..7a9c6aaa99c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -101,18 +101,18 @@ public class SchemaInfoTestCase { void testFieldWithAttributeExporting() { var schemaInfoTester = new SchemaInfoTester(); var schema = schemaInfoTester.createSchema("test"); - var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); + var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.INT)); var attribute = field.addAttribute(new Attribute(schema.getName(), field.getName(), "f1Attribute", field.getDataType())); attribute.getAliases().add("a1"); attribute.getAliases().add("a2"); assertEquals(""" schema[0].name "test" schema[0].field[0].name "f1" - schema[0].field[0].type "string" + schema[0].field[0].type "int" schema[0].field[0].attribute false schema[0].field[0].index false schema[0].field[1].name "f1Attribute" - schema[0].field[1].type "string" + schema[0].field[1].type "int" schema[0].field[1].alias[0] "a1" schema[0].field[1].alias[1] "a2" schema[0].field[1].attribute true diff --git a/config/src/tests/functiontest/functiontest.cpp b/config/src/tests/functiontest/functiontest.cpp index bb687ee92eb..5557fe6d12f 100644 --- a/config/src/tests/functiontest/functiontest.cpp +++ b/config/src/tests/functiontest/functiontest.cpp @@ -24,10 +24,10 @@ checkVariableAccess(const FunctionTestConfig & config) EXPECT_EQUAL(true, config.boolWithDef); EXPECT_EQUAL(5, config.intVal); EXPECT_EQUAL(-14, config.intWithDef); - EXPECT_EQUAL(12345678901LL, config.longVal); - EXPECT_EQUAL(-9876543210LL, config.longWithDef); + EXPECT_EQUAL(12345678901L, config.longVal); + EXPECT_EQUAL(-9876543210L, config.longWithDef); EXPECT_APPROX(41.23, config.doubleVal, 0.000001); - EXPECT_APPROX(-12, config.doubleWithDef, 0.000001); + EXPECT_APPROX(-12.0, config.doubleWithDef, 0.000001); EXPECT_EQUAL("foo", config.stringVal); EXPECT_EQUAL("bar", config.stringwithdef); EXPECT_EQUAL("FOOBAR", FunctionTestConfig::getEnumValName(config.enumVal)); @@ -188,8 +188,8 @@ TEST_F("testDefaultValues", TestFixture("defaultvalues")) { EXPECT_EQUAL(false, f._config->boolWithDef); EXPECT_EQUAL(5, f._config->intVal); EXPECT_EQUAL(-545, f._config->intWithDef); - EXPECT_EQUAL(1234567890123LL, f._config->longVal); - EXPECT_EQUAL(-50000000000LL, f._config->longWithDef); + EXPECT_EQUAL(1234567890123L, f._config->longVal); + EXPECT_EQUAL(-50000000000L, f._config->longWithDef); EXPECT_APPROX(41.23, f._config->doubleVal, 0.000001); EXPECT_APPROX(-6.43, f._config->doubleWithDef, 0.000001); EXPECT_EQUAL("foo", f._config->stringVal); @@ -265,10 +265,10 @@ TEST_F("testRandomOrder", TestFixture("randomorder")) { EXPECT_EQUAL(true, f._config->boolWithDef); EXPECT_EQUAL(5, f._config->intVal); EXPECT_EQUAL(-14, f._config->intWithDef); - EXPECT_EQUAL(666000666000LL, f._config->longVal); - EXPECT_EQUAL(-333000333000LL, f._config->longWithDef); + EXPECT_EQUAL(666000666000L, f._config->longVal); + EXPECT_EQUAL(-333000333000L, f._config->longWithDef); EXPECT_APPROX(41.23, f._config->doubleVal, 0.000001); - EXPECT_APPROX(-12, f._config->doubleWithDef, 0.000001); + EXPECT_APPROX(-12.0, f._config->doubleWithDef, 0.000001); EXPECT_EQUAL("foo", f._config->stringVal); EXPECT_EQUAL("bar", f._config->stringwithdef); EXPECT_EQUAL("FOOBAR", FunctionTestConfig::getEnumValName(f._config->enumVal)); diff --git a/configdefinitions/src/vespa/curator.def b/configdefinitions/src/vespa/curator.def index 7b19a2c9543..992a4973c21 100644 --- a/configdefinitions/src/vespa/curator.def +++ b/configdefinitions/src/vespa/curator.def @@ -10,3 +10,8 @@ zookeeperLocalhostAffinity bool default=false # session timeout, the high default is used by config servers zookeeperSessionTimeoutSeconds int default=120 + +# Jute maxbuffer. Used by zookeeper to determine max buffer when serializing/desesrializing +# Value used in server must correspond to this one (so if decreasing it one must be sure +# that no node has store more than this many bytes) +juteMaxBuffer int default=104857600 diff --git a/configdefinitions/src/vespa/fleetcontroller.def b/configdefinitions/src/vespa/fleetcontroller.def index 39dbd9f9e12..f4850531e7e 100644 --- a/configdefinitions/src/vespa/fleetcontroller.def +++ b/configdefinitions/src/vespa/fleetcontroller.def @@ -204,3 +204,10 @@ cluster_feed_block_noise_level double default=0.0 # For apps that have several groups this controls how many groups are allowed to # be down simultaneously in this cluster. max_number_of_groups_allowed_to_be_down int default=-1 + +## Iff true, cluster state bundles sent from the cluster controller to distributors +## and content nodes will include the current distribution config. The CC-provided +## distribution config takes precedence over the node-local config. When enabled, +## a given versioned state corresponds directly to a particular bucket ownership +## and replica placement mapping for the ideal state algorithm. +include_distribution_config_in_cluster_state_bundle bool default=false diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index d1cea49ee6f..5c6134aadbf 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -24,8 +24,8 @@ autopurge.snapRetainCount int default=15 # Vespa home is prepended if the file is relative myidFile string default="var/zookeeper/myid" -# Change from default of 1 Mb in zookeeper to 50 Mb -juteMaxBuffer int default=52428800 +# Change from default of 1 Mb in zookeeper to 100 Mb +juteMaxBuffer int default=104857600 myid int restart server[].id int diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index cfc1fea0fc5..0ba68cd71d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -283,7 +283,7 @@ public class ConfigConvergenceChecker extends AbstractComponent { private static RequestConfig createRequestConfig(Duration timeout) { return RequestConfig.custom() - .setConnectionRequestTimeout(Timeout.ofSeconds(1)) + .setConnectionRequestTimeout(Timeout.ofSeconds(10)) .setResponseTimeout(Timeout.ofMilliseconds(timeout.toMillis())) .build(); } @@ -296,7 +296,7 @@ public class ConfigConvergenceChecker extends AbstractComponent { .setMaxConnPerRoute(10) .setDefaultConnectionConfig(ConnectionConfig.custom() .setTimeToLive(TimeValue.ofMilliseconds(1)) - .setConnectTimeout(Timeout.ofSeconds(1)) + .setConnectTimeout(Timeout.ofSeconds(10)) // Times out at 1s over wireguard with 500+ services. .build()) .setTlsStrategy(tlsStrategy) .build()) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 6c4a3d22659..75d94434303 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.config.server.http.v2; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; -import com.yahoo.component.annotation.Inject; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ServiceInfo; @@ -217,7 +217,7 @@ public class ApplicationHandler extends HttpHandler { private HttpResponse logs(ApplicationId applicationId, HttpRequest request) { HttpURL requestURL = HttpURL.from(request.getUri()); Optional<DomainName> hostname = Optional.ofNullable(requestURL.query().lastEntries().get("hostname")).map(DomainName::of); - return applicationRepository.getLogs(applicationId, hostname, requestURL.query().remove("hostname")); + return applicationRepository.getLogs(applicationId, hostname, requestURL.query()); } private HttpResponse searchNodeMetrics(ApplicationId applicationId) { diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index ef00a1e3087..a7e555a94aa 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -23,7 +23,9 @@ }, "com.yahoo.component.chain.ChainedComponent" : { "superClass" : "com.yahoo.component.AbstractComponent", - "interfaces" : [ ], + "interfaces" : [ + "com.yahoo.component.chain.model.Chainable" + ], "attributes" : [ "public", "abstract" @@ -32,9 +34,7 @@ "public void <init>(com.yahoo.component.ComponentId)", "protected void <init>()", "public void initDependencies(com.yahoo.component.chain.dependencies.Dependencies)", - "public com.yahoo.component.chain.dependencies.Dependencies getDependencies()", - "protected com.yahoo.component.chain.dependencies.Dependencies getDefaultAnnotatedDependencies()", - "protected com.yahoo.component.chain.dependencies.Dependencies getAnnotatedDependencies(java.lang.Class, java.lang.Class, java.lang.Class)" + "public com.yahoo.component.chain.dependencies.Dependencies getDependencies()" ], "fields" : [ ] }, @@ -2370,7 +2370,9 @@ }, "com.yahoo.jdisc.http.filter.RequestFilterBase" : { "superClass" : "java.lang.Object", - "interfaces" : [ ], + "interfaces" : [ + "com.yahoo.component.chain.model.Chainable" + ], "attributes" : [ "public", "interface", @@ -2414,7 +2416,9 @@ }, "com.yahoo.jdisc.http.filter.ResponseFilterBase" : { "superClass" : "java.lang.Object", - "interfaces" : [ ], + "interfaces" : [ + "com.yahoo.component.chain.model.Chainable" + ], "attributes" : [ "public", "interface", @@ -2426,7 +2430,8 @@ "com.yahoo.jdisc.http.filter.SecurityRequestFilter" : { "superClass" : "java.lang.Object", "interfaces" : [ - "com.yahoo.jdisc.http.filter.RequestFilterBase" + "com.yahoo.jdisc.http.filter.RequestFilterBase", + "com.yahoo.component.chain.model.Chainable" ], "attributes" : [ "public", @@ -2459,7 +2464,8 @@ "com.yahoo.jdisc.http.filter.SecurityResponseFilter" : { "superClass" : "java.lang.Object", "interfaces" : [ - "com.yahoo.jdisc.http.filter.ResponseFilterBase" + "com.yahoo.jdisc.http.filter.ResponseFilterBase", + "com.yahoo.component.chain.model.Chainable" ], "attributes" : [ "public", diff --git a/container-core/src/main/java/com/yahoo/component/chain/ChainedComponent.java b/container-core/src/main/java/com/yahoo/component/chain/ChainedComponent.java index 2c2ee5226a6..109c82fe1c7 100644 --- a/container-core/src/main/java/com/yahoo/component/chain/ChainedComponent.java +++ b/container-core/src/main/java/com/yahoo/component/chain/ChainedComponent.java @@ -7,6 +7,7 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Dependencies; import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.component.chain.model.Chainable; import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; @@ -19,80 +20,32 @@ import java.util.List; * * @author Tony Vaagenes */ -public abstract class ChainedComponent extends AbstractComponent { +public abstract class ChainedComponent extends AbstractComponent implements Chainable { - /** The immutable set of dependencies of this. NOTE: the default is only for unit testing. */ - private Dependencies dependencies = getDefaultAnnotatedDependencies(); + /** + * The immutable set of dependencies of this. NOTE: the default is only for unit testing. + */ + private Dependencies dependencies = getAnnotatedDependencies(); public ChainedComponent(ComponentId id) { super(id); } - protected ChainedComponent() {} + protected ChainedComponent() { } /** * Called by the container to assign the full set of dependencies to this class (configured and declared). * This is called once before this is started. - * @param dependencies The configured dependencies, that this method will merge with annotated dependencies. + * + * @param dependencies The configured dependencies, that this method will merge with annotated dependencies. */ public void initDependencies(Dependencies dependencies) { - this.dependencies = dependencies.union(getDefaultAnnotatedDependencies()); - } - - /** Returns the configured and declared dependencies of this chainedcomponent */ - public Dependencies getDependencies() { return dependencies; } - - /** This method is here only for legacy reasons, do not override. */ - protected Dependencies getDefaultAnnotatedDependencies() { - Dependencies dependencies = getAnnotatedDependencies(com.yahoo.yolean.chain.Provides.class, com.yahoo.yolean.chain.Before.class, com.yahoo.yolean.chain.After.class); - Dependencies legacyDependencies = getAnnotatedDependencies(Provides.class, Before.class, After.class); - - return dependencies.union(legacyDependencies); + this.dependencies = dependencies.union(getAnnotatedDependencies()); } /** - * @param providesClass The annotation class representing 'provides'. - * @param beforeClass The annotation class representing 'before'. - * @param afterClass The annotation class representing 'after'. - * @return a new {@link Dependencies} created from the annotations given in this component's class. + * Returns the configured and declared dependencies of this chainedcomponent */ - protected Dependencies getAnnotatedDependencies(Class<? extends Annotation> providesClass, - Class<? extends Annotation> beforeClass, - Class<? extends Annotation> afterClass) { - return new Dependencies( - allOf(getSymbols(this, providesClass), this.getClass().getSimpleName(), this.getClass().getName()), - getSymbols(this, beforeClass), - getSymbols(this, afterClass)); - } - - // TODO: move to vespajlib. - private static List<String> allOf(List<String> symbols, String... otherSymbols) { - List<String> result = new ArrayList<>(symbols); - result.addAll(List.of(otherSymbols)); - return result; - } - - - private static List<String> getSymbols(ChainedComponent component, Class<? extends Annotation> annotationClass) { - List<String> result = new ArrayList<>(); - - result.addAll(annotationSymbols(component, annotationClass)); - return result; - } - - private static Collection<String> annotationSymbols(ChainedComponent component, Class<? extends Annotation> annotationClass) { - - try { - Annotation annotation = component.getClass().getAnnotation(annotationClass); - if (annotation != null) { - Object values = annotationClass.getMethod("value").invoke(annotation); - return List.of((String[])values); - } - return List.of(); - - } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } + public Dependencies getDependencies() { return dependencies; } -} +}
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java b/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java index c0bc01be27c..b80746f8b56 100644 --- a/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java +++ b/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java @@ -148,14 +148,7 @@ public class ChainBuilder<T extends ChainedComponent> { } private NameProvider getNameProvider(String name) { - NameProvider nameProvider = nameProviders.get(name); - if (nameProvider != null) - return nameProvider; - else { - nameProvider = new ComponentNameProvider(name); - nameProviders.put(name, nameProvider); - return nameProvider; - } + return nameProviders.computeIfAbsent(name, ComponentNameProvider::new); } private OrderedReadyNodes getReadyNodes() { diff --git a/container-core/src/main/java/com/yahoo/component/chain/model/Chainable.java b/container-core/src/main/java/com/yahoo/component/chain/model/Chainable.java new file mode 100644 index 00000000000..67ed40f8e77 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/component/chain/model/Chainable.java @@ -0,0 +1,42 @@ +package com.yahoo.component.chain.model; + +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.dependencies.Provides; + +import java.lang.annotation.Annotation; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Components which can be chained together, and where dependency information is provided through annotations. + * + * @author jonmv + */ +public interface Chainable { + + default Dependencies getAnnotatedDependencies() { + Set<String> provides = new LinkedHashSet<>(); + Set<String> before = new LinkedHashSet<>(); + Set<String> after = new LinkedHashSet<>(); + + for (Annotation annotation : getClass().getAnnotations()) { + if (annotation instanceof Provides p) provides.addAll(List.of(p.value())); + if (annotation instanceof com.yahoo.yolean.chain.Provides p) provides.addAll(List.of(p.value())); + + if (annotation instanceof Before b) before.addAll(List.of(b.value())); + if (annotation instanceof com.yahoo.yolean.chain.Before b) before.addAll(List.of(b.value())); + + if (annotation instanceof After a) after.addAll(List.of(a.value())); + if (annotation instanceof com.yahoo.yolean.chain.After a) after.addAll(List.of(a.value())); + } + + provides.add(getClass().getSimpleName()); + provides.add(getClass().getName()); + + return new Dependencies(provides, before, after); + } + +}
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java index 10f8400fa37..7d9aea8fd63 100644 --- a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java +++ b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java @@ -7,6 +7,8 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.Chain; import com.yahoo.component.chain.ChainedComponent; import com.yahoo.component.chain.ChainsConfigurer; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.Chainable; import com.yahoo.component.chain.model.ChainsModel; import com.yahoo.component.chain.model.ChainsModelBuilder; import com.yahoo.component.provider.ComponentRegistry; @@ -65,16 +67,18 @@ public class FilterChainRepository extends AbstractComponent { } } + @SafeVarargs private static void addAllChains(ComponentRegistry<Object> destination, ChainsConfig chainsConfig, - ComponentRegistry<?>... filters) { + ComponentRegistry<? extends Chainable>... filters) { ChainRegistry<FilterWrapper> chainRegistry = buildChainRegistry(chainsConfig, filters); chainRegistry.allComponents() .forEach(chain -> destination.register(chain.getId(), toJDiscChain(chain))); } + @SafeVarargs private static ChainRegistry<FilterWrapper> buildChainRegistry(ChainsConfig chainsConfig, - ComponentRegistry<?>... filters) { + ComponentRegistry<? extends Chainable>... filters) { ChainRegistry<FilterWrapper> chainRegistry = new ChainRegistry<>(); ChainsModel chainsModel = ChainsModelBuilder.buildFromConfig(chainsConfig); ChainsConfigurer.prepareChainRegistry(chainRegistry, chainsModel, allFiltersWrapped(filters)); @@ -146,9 +150,10 @@ public class FilterChainRepository extends AbstractComponent { } } - private static ComponentRegistry<FilterWrapper> allFiltersWrapped(ComponentRegistry<?>... registries) { + @SafeVarargs + private static ComponentRegistry<FilterWrapper> allFiltersWrapped(ComponentRegistry<? extends Chainable>... registries) { ComponentRegistry<FilterWrapper> wrappedFilters = new ComponentRegistry<>(); - for (ComponentRegistry<?> registry : registries) { + for (ComponentRegistry<? extends Chainable> registry : registries) { registry.allComponentsById() .forEach((id, filter) -> wrappedFilters.register(id, new FilterWrapper(id, filter))); } @@ -176,16 +181,21 @@ public class FilterChainRepository extends AbstractComponent { } private static class FilterWrapper extends ChainedComponent { - public final Object filter; - public final Class<?> filterType; + public final Chainable filter; + public final Class<? extends Chainable> filterType; - public FilterWrapper(ComponentId id, Object filter) { + public FilterWrapper(ComponentId id, Chainable filter) { super(id); this.filter = filter; this.filterType = getFilterType(filter); } - private static Class<?> getFilterType(Object filter) { + @Override + public Dependencies getAnnotatedDependencies() { + return filter == null ? super.getAnnotatedDependencies() : filter.getAnnotatedDependencies(); + } + + private static Class<? extends Chainable> getFilterType(Object filter) { if (filter instanceof RequestFilter) return RequestFilter.class; else if (filter instanceof ResponseFilter) diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/RequestFilterBase.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/RequestFilterBase.java index 49fe5a3af4e..2a38682b080 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/RequestFilterBase.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/RequestFilterBase.java @@ -1,9 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter; +import com.yahoo.component.chain.model.Chainable; + /** * @author gjoranv * @since 2.4 */ -public interface RequestFilterBase { +public interface RequestFilterBase extends Chainable { } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/ResponseFilter.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/ResponseFilter.java index 64d54d305d4..e219132a147 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/ResponseFilter.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/ResponseFilter.java @@ -6,7 +6,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.SharedResource; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public interface ResponseFilter extends SharedResource, ResponseFilterBase { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/ResponseFilterBase.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/ResponseFilterBase.java index 6cf00672808..acbf6592046 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/ResponseFilterBase.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/ResponseFilterBase.java @@ -1,9 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter; +import com.yahoo.component.chain.model.Chainable; + /** * @author gjoranv * @since 2.4 */ -public interface ResponseFilterBase { +public interface ResponseFilterBase extends Chainable { } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityRequestFilter.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityRequestFilter.java index cc6633f633a..dd2a1719a3a 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityRequestFilter.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityRequestFilter.java @@ -1,12 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter; +import com.yahoo.component.chain.model.Chainable; import com.yahoo.jdisc.handler.ResponseHandler; /** * @author Simon Thoresen Hult */ -public interface SecurityRequestFilter extends RequestFilterBase { +public interface SecurityRequestFilter extends RequestFilterBase, Chainable { void filter(DiscFilterRequest request, ResponseHandler handler); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityResponseFilter.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityResponseFilter.java index 05ca9c04d26..4abceed6d16 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityResponseFilter.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityResponseFilter.java @@ -1,7 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter; -public interface SecurityResponseFilter extends ResponseFilterBase { +import com.yahoo.component.chain.model.Chainable; + +public interface SecurityResponseFilter extends ResponseFilterBase, Chainable { void filter(DiscFilterResponse response, RequestView request); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java index b2afd73bc80..4e8d9d00a09 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java @@ -38,8 +38,8 @@ public class DataplaneProxyCredentials extends AbstractComponent { @Inject public DataplaneProxyCredentials() { this( - Paths.get(Defaults.getDefaults().underVespaHome("tmp/proxy_cert.pem")), - Paths.get(Defaults.getDefaults().underVespaHome("tmp/proxy_key.pem")) + Paths.get(Defaults.getDefaults().underVespaHome("secure/proxy_cert.pem")), + Paths.get(Defaults.getDefaults().underVespaHome("secure/proxy_key.pem")) ); } diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index 440df4f9be9..9624f773493 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -334,7 +334,7 @@ public final class CompoundName { /** * Creates a CompoundName from a string, possibly reusing from cache. * Prefer over constructing on the fly. - **/ + */ public static CompoundName from(String name) { CompoundName found = cache.get(name); if (found != null) return found; diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java index d69d058dd91..6ae257c97ad 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java @@ -109,6 +109,6 @@ public class PrometheusV1HandlerTest { void consumer_is_propagated_to_metrics_proxy_api() { String response = testDriver.sendRequest(VALUES_URI + consumerQuery(CUSTOM_CONSUMER)).readAll(); - assertTrue(response.contains(REPLACED_CPU_METRIC)); + assertTrue(response.contains(REPLACED_CPU_METRIC), response); } } diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index a13485fb6bb..b6f2a578469 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -57,7 +57,7 @@ getconfig() { qrstartcfg="`cat ${config_dir}/qr-start.cfg`" ;; *) - qrstartcfg="`$VESPA_HOME/bin/vespa-get-config -l -w 10 -n search.config.qr-start -i ${VESPA_CONFIG_ID}`" + qrstartcfg="`$VESPA_HOME/bin/vespa-get-config -l -w 30 -n search.config.qr-start -i ${VESPA_CONFIG_ID}`" ;; esac cmds=`echo "$qrstartcfg" | sed -n 's/^\([^. ]*\)[.]/\1_/;s/ /=/p'` diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java index 7acbaa73fe4..44a7578fd9f 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java @@ -2,15 +2,29 @@ package com.yahoo.container.jdisc; import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.ChainedComponent; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.core.ChainsConfig; +import com.yahoo.container.core.ChainsConfig.Chains; +import com.yahoo.container.core.ChainsConfig.Components; +import com.yahoo.container.core.ChainsConfig.Components.Builder; import com.yahoo.container.http.filter.FilterChainRepository; +import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.RequestFilter; import com.yahoo.jdisc.http.filter.ResponseFilter; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain; import com.yahoo.jdisc.http.server.jetty.FilterBindings; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -50,6 +64,71 @@ public class FilterBindingsProviderTest { } @Test + void requireThatChainOrderIsCorrect() { + + List<ChainedComponent> filteredBy = new ArrayList<>(); + + class DummyFilter extends ChainedComponent implements SecurityRequestFilter { + @Override public void filter(DiscFilterRequest request, ResponseHandler handler) { filteredBy.add(this); } + @Override public String toString() { return getClass().getAnnotation(Provides.class).value()[0]; } + } + + @Provides("foo") class Filter1 extends DummyFilter { } + @Provides("bar") @After("foo") class Filter2 extends DummyFilter { } + @Provides("baz") @Before("foo") class Filter3 extends DummyFilter { } + + ComponentRegistry<SecurityRequestFilter> filters = new ComponentRegistry<>(); + + SecurityRequestFilter foo = new Filter1(); + SecurityRequestFilter bar = new Filter2(); + SecurityRequestFilter baz = new Filter3(); + + filters.register(ComponentId.fromString("foo"), foo); + filters.register(ComponentId.fromString("bar"), bar); + filters.register(ComponentId.fromString("baz"), baz); + + ChainsConfig.Builder oneChain = new ChainsConfig.Builder(); + oneChain.components(new Components.Builder().id("foo")) + .components(new Components.Builder().id("bar")) + .components(new Components.Builder().id("baz")) + .chains(new Chains.Builder().id("chain") + .components("bar") + .components("foo") + .components("baz")); + + ((SecurityRequestFilterChain) new FilterChainRepository(oneChain.build(), + new ComponentRegistry<>(), + new ComponentRegistry<>(), + filters, + new ComponentRegistry<>()) + .getFilter(ComponentSpecification.fromString("chain"))).filter((DiscFilterRequest) null, null); + + assertEquals(List.of(baz, foo, bar), filteredBy); + + filteredBy.clear(); + ChainsConfig.Builder childChain = new ChainsConfig.Builder(); + childChain.components(new Builder().id("foo")) + .components(new Builder().id("bar")) + .components(new Builder().id("baz")) + .chains(new Chains.Builder().id("parent") + .components("bar") + .components("baz")) + .chains(new Chains.Builder().id("child") + .components("foo") + .inherits("parent")); + + ((SecurityRequestFilterChain) new FilterChainRepository(childChain.build(), + new ComponentRegistry<>(), + new ComponentRegistry<>(), + filters, + new ComponentRegistry<>()) + .getFilter(ComponentSpecification.fromString("child"))).filter((DiscFilterRequest) null, null); + + assertEquals(List.of(baz, foo, bar), filteredBy); + + } + + @Test void requireThatCorrectlyConfiguredFiltersAreIncluded() { final String requestFilter1Id = "requestFilter1"; final String requestFilter2Id = "requestFilter2"; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java index 8325101b4ad..ad2197cb39b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java @@ -363,12 +363,13 @@ public abstract class CompositeItem extends Item { @Override public void remove() { - owner.removing(current); wrapped.remove(); + owner.removing(current); } @Override public void set(Item newItem) { + if (newItem == current) return; owner.removing(current); owner.adding(newItem); current = newItem; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 612a6ca5618..778894e61b3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -20,7 +20,7 @@ import java.util.Objects; public final class WeakAndItem extends NonReducibleCompositeItem { /** The default N used if none is specified: 100 */ - public static final int defaultN = 100; //TODO Make private + public static final int defaultN = 100; // TODO Vespa 9: Make private private int n; private String index; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java index 64b9b91936b..fbcd8935ebe 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java @@ -77,9 +77,8 @@ public class Match { @Override public boolean equals(Object o) { - if (! (o instanceof Match)) return false; + if (! (o instanceof Match other)) return false; - Match other = (Match)o; if (other.position != position) return false; if ( ! other.item.equals(item)) return false; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java index 284e98d4ceb..7994fe2fa41 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java @@ -32,7 +32,8 @@ public class ReplacingProductionRule extends ProductionRule { itemCount--; Match match = e.getNonreferencedMatch(i); - match.getItem().getParent().removeItem(match.getPosition()); + if (match.getItem().getParent() != null) + match.getItem().getParent().removeItem(match.getPosition()); } } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index 77f27d3d411..26181a7f0d7 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -19,7 +19,16 @@ class SchemaInfoConfigurer { } static Schema toSchema(SchemaInfoConfig.Schema schemaInfoConfig) { - Schema.Builder builder = new Schema.Builder(schemaInfoConfig.name()); + Schema.Builder schemaBuilder = new Schema.Builder(schemaInfoConfig.name()); + + for (var fieldConfig : schemaInfoConfig.field()) { + Field.Builder fieldBuilder = new Field.Builder(fieldConfig.name(), fieldConfig.type()); + fieldBuilder.setAttribute(fieldConfig.attribute()); + fieldBuilder.setIndex(fieldConfig.index()); + for (var alias : fieldConfig.alias()) + fieldBuilder.addAlias(alias); + schemaBuilder.add(fieldBuilder.build()); + } for (var profileConfig : schemaInfoConfig.rankprofile()) { RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()) @@ -28,7 +37,7 @@ class SchemaInfoConfigurer { .setUseSignificanceModel(profileConfig.significance().useModel()); for (var inputConfig : profileConfig.input()) profileBuilder.addInput(inputConfig.name(), RankProfile.InputType.fromSpec(inputConfig.type())); - builder.add(profileBuilder.build()); + schemaBuilder.add(profileBuilder.build()); } for (var summaryConfig : schemaInfoConfig.summaryclass()) { @@ -38,10 +47,10 @@ class SchemaInfoConfigurer { summaryBuilder.setDynamic(true); summaryBuilder.add(new DocumentSummary.Field(field.name(), field.type())); } - builder.add(summaryBuilder.build()); + schemaBuilder.add(summaryBuilder.build()); } - return builder.build(); + return schemaBuilder.build(); } static List<Cluster> toClusters(QrSearchersConfig config) { diff --git a/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java index 924951fe430..aad892a501a 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java @@ -27,6 +27,7 @@ import com.yahoo.search.searchchain.Execution; @Beta @After(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND) public class OpportunisticWeakAndSearcher extends Searcher { + private static final CompoundName OPPORTUNISTIC_AND = CompoundName.from("weakAnd.opportunistic.and"); private static final CompoundName OPPORTUNISTIC_FACTOR = CompoundName.from("weakAnd.opportunistic.factor"); diff --git a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java index 9e6e2f785fd..8ba8f747019 100644 --- a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java @@ -14,6 +14,7 @@ import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.query.ranking.Significance; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.schema.Schema; @@ -22,6 +23,7 @@ import com.yahoo.search.searchchain.Execution; import java.util.HashSet; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -69,6 +71,7 @@ public class SignificanceSearcher extends Searcher { // This will result in a failure later (in a "backend searcher") anyway. Optional.ofNullable(schema.rankProfiles().get(rankProfileName)) .map(RankProfile::useSignificanceModel).orElse(false))); + log.log(Level.FINE, () -> "Significance setup per schema: " + perSchemaSetup); var uniqueSetups = new HashSet<>(perSchemaSetup.values()); // Fail if the significance setup for the selected schemas are conflicting @@ -93,30 +96,76 @@ public class SignificanceSearcher extends Searcher { } private Result calculateAndSetSignificance(Query query, Execution execution) { - Language language = query.getModel().getParsingLanguage(); - Optional<SignificanceModel> model = significanceModelRegistry.getModel(language); + try { + var significanceModel = getSignificanceModelFromQueryLanguage(query); + log.log(Level.FINE, () -> "Got model for language %s: %s" + .formatted(query.getModel().getParsingLanguage(), significanceModel.getId())); - if (model.isEmpty()) return execution.search(query); + setIDF(query.getModel().getQueryTree().getRoot(), significanceModel); - setIDF(query.getModel().getQueryTree().getRoot(), model.get()); + return execution.search(query); + } catch (IllegalArgumentException e) { + var result = new Result(query); + result.hits().addError( + ErrorMessage.createIllegalQuery(e.getMessage())); + return result; + } + } + + private SignificanceModel getSignificanceModelFromQueryLanguage(Query query) throws IllegalArgumentException { + Language explicitLanguage = query.getModel().getLanguage(); + Language implicitLanguage = query.getModel().getParsingLanguage(); + + if (explicitLanguage == null && implicitLanguage == null) { + throw new IllegalArgumentException("No language found in query"); + } + + if (explicitLanguage != null) { + if (explicitLanguage == Language.UNKNOWN) { + return handleFallBackToUnknownLanguage(); + } + var model = significanceModelRegistry.getModel(explicitLanguage); + if (model.isEmpty()) { + throw new IllegalArgumentException("No significance model available for set language " + explicitLanguage); + } + return model.get(); + } - return execution.search(query); + if (implicitLanguage == Language.UNKNOWN) { + return handleFallBackToUnknownLanguage(); + } + var model = significanceModelRegistry.getModel(implicitLanguage); + if (model.isEmpty()) { + throw new IllegalArgumentException("No significance model available for implicit language " + implicitLanguage); + } + return model.get(); + } + + private SignificanceModel handleFallBackToUnknownLanguage() throws IllegalArgumentException { + var unknownModel = significanceModelRegistry.getModel(Language.UNKNOWN); + var englishModel = significanceModelRegistry.getModel(Language.ENGLISH); + + if (unknownModel.isEmpty() && englishModel.isEmpty()) { + throw new IllegalArgumentException("No significance model available for unknown or english language"); + } + + return unknownModel.orElseGet(englishModel::get); } private void setIDF(Item root, SignificanceModel significanceModel) { if (root == null || root instanceof NullItem) return; - if (root instanceof WordItem) { - - var documentFrequency = significanceModel.documentFrequency(((WordItem) root).getWord()); + if (root instanceof WordItem wi) { + var word = wi.getWord(); + var documentFrequency = significanceModel.documentFrequency(word); long N = documentFrequency.corpusSize(); long nq_i = documentFrequency.frequency(); double idf = calculateIDF(N, nq_i); - - ((WordItem) root).setSignificance(idf); - } else if (root instanceof CompositeItem) { - for (int i = 0; i < ((CompositeItem) root).getItemCount(); i++) { - setIDF(((CompositeItem) root).getItem(i), significanceModel); + log.log(Level.FINE, () -> "Setting IDF for " + word + " to " + idf); + wi.setSignificance(idf); + } else if (root instanceof CompositeItem ci) { + for (int i = 0; i < ci.getItemCount(); i++) { + setIDF(ci.getItem(i), significanceModel); } } } diff --git a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java index a8e988057d4..21691f3de90 100755 --- a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java +++ b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java @@ -14,7 +14,6 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.EventStream; import com.yahoo.search.searchchain.Execution; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.File; @@ -139,7 +138,6 @@ public class LLMSearcherTest { } @Test - @Disabled public void testAsyncGeneration() { var executor = Executors.newFixedThreadPool(2); var sb = new StringBuilder(); @@ -156,17 +154,13 @@ public class LLMSearcherTest { assertEquals(1, result.getHitCount()); assertTrue(result.hits().get(0) instanceof EventStream); EventStream eventStream = (EventStream) result.hits().get(0); - var incoming = eventStream.incoming(); - incoming.addNewDataListener(() -> { - incoming.drain().forEach(event -> sb.append(event.toString())); - }, executor); incoming.completedFuture().join(); assertTrue(incoming.isComplete()); - // Ensure incoming has been fully drained to avoid race condition in this test - incoming.drain().forEach(event -> sb.append(event.toString())); + incoming.drain().forEach(event -> eventStream.add(event)); + eventStream.asList().forEach(event -> sb.append(event.toString())); } finally { executor.shutdownNow(); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryLanguageTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryLanguageTestCase.java index a05124a42b1..8fd296045df 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryLanguageTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryLanguageTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; +import com.yahoo.search.query.Model; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -100,6 +101,10 @@ public class QueryLanguageTestCase { private void assertLanguage(Language expectedLanguage, String languageParameter) { Query query = new Query("?query=test&language=" + languageParameter); + Query query2 = new Query("?query=test"); + Model model = query.getModel(); + Model model2 = query2.getModel(); + assertEquals(expectedLanguage, query.getModel().getParsingLanguage()); /* diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java index 2ba399cf42d..820de5a6cb7 100644 --- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java @@ -62,6 +62,8 @@ public class SchemaInfoTester { .addInput("query(myTensor4)", InputType.fromSpec("tensor<float>(x[5])")); schemas.add(new Schema.Builder("a") .add(common.build()) + .add(new Field.Builder("field1", "string").setIndex(true).addAlias("alias1").addAlias("alias2").build()) + .add(new Field.Builder("field2", "int").setAttribute(true).build()) .add(new RankProfile.Builder("inconsistent") .addInput("query(myTensor1)", InputType.fromSpec("tensor(a{},b{})")) .build()) @@ -102,6 +104,12 @@ public class SchemaInfoTester { var schemaA = new SchemaInfoConfig.Schema.Builder(); schemaA.name("a"); + schemaA.field(new SchemaInfoConfig.Schema.Field.Builder().name("field1").type("string") + .index(true).attribute(false) + .alias("alias1").alias("alias2")); + schemaA.field(new SchemaInfoConfig.Schema.Field.Builder().name("field2").type("int") + .index(false).attribute(true)); + schemaA.rankprofile(rankProfileCommon); var rankProfileInconsistentA = new SchemaInfoConfig.Schema.Rankprofile.Builder(); rankProfileInconsistentA.name("inconsistent"); diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java index be68c87efb3..29e3c002c21 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -4,6 +4,12 @@ package com.yahoo.search.significance.test; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.detect.Detection; +import com.yahoo.language.detect.Detector; +import com.yahoo.language.detect.Hint; +import com.yahoo.language.opennlp.OpenNlpLinguistics; +import com.yahoo.language.process.*; import com.yahoo.language.significance.SignificanceModel; import com.yahoo.language.significance.SignificanceModelRegistry; import com.yahoo.language.significance.impl.DefaultSignificanceModelRegistry; @@ -20,6 +26,7 @@ import com.yahoo.search.significance.SignificanceSearcher; import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; @@ -27,6 +34,7 @@ import java.util.List; import static com.yahoo.test.JunitCompat.assertEquals; +import static java.nio.charset.StandardCharsets.UTF_8; /** * Tests significance term in the search chain. @@ -51,12 +59,90 @@ public class SignificanceSearcherTest { searcher = new SignificanceSearcher(significanceModelRegistry, new SchemaInfo(List.of(schema.build()), List.of())); } + private static class MockLinguistics implements Linguistics { + + private final MockDetector mockDetector; + MockLinguistics(Language language) { + this.mockDetector = new MockDetector(language); + } + + @Override + public Stemmer getStemmer() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Tokenizer getTokenizer() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Normalizer getNormalizer() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Transformer getTransformer() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Segmenter getSegmenter() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Detector getDetector() { + return this.mockDetector; + } + + @Override + public GramSplitter getGramSplitter() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public CharacterClasses getCharacterClasses() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public boolean equals(Linguistics other) { + return false; + } + } + + private static class MockDetector implements Detector { + + private Language detectionLanguage; + MockDetector(Language detectionLanguage) { + this.detectionLanguage = detectionLanguage; + } + + @Override + public Detection detect(byte[] input, int offset, int length, Hint hint) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Detection detect(ByteBuffer input, Hint hint) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Detection detect(String input, Hint hint) { + return new Detection(detectionLanguage, UTF_8.name(), false); + } + } + private Execution createExecution(SignificanceSearcher searcher) { return new Execution(new Chain<>(searcher), Execution.Context.createContextStub()); } - private Execution createExecution() { - return new Execution(new Chain<>(), Execution.Context.createContextStub()); + private Execution createExecution(SignificanceSearcher searcher, Language language) { + var context = Execution.Context.createContextStub(); + context.setLinguistics(new MockLinguistics(language)); + return new Execution(new Chain<>(searcher), context); } @Test @@ -191,35 +277,6 @@ public class SignificanceSearcherTest { } - @Test - void testSignificanceValueOnEmptyQuery() { - Query q = new Query(); - q.getModel().setLanguage(Language.NORWEGIAN_BOKMAL); - AndItem root = new AndItem(); - WordItem tmp; - tmp = new WordItem("Hei", true); - root.addItem(tmp); - tmp = new WordItem("Verden", true); - root.addItem(tmp); - - - q.getModel().getQueryTree().setRoot(root); - Result r = createExecution(searcher).search(q); - root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); - - WordItem w0 = (WordItem) root.getItem(0); - WordItem w1 = (WordItem) root.getItem(1); - - Result r0 = createExecution().search(q); - root = (AndItem) r0.getQuery().getModel().getQueryTree().getRoot(); - - WordItem w0_0 = (WordItem) root.getItem(0); - WordItem w0_1 = (WordItem) root.getItem(1); - - assertEquals(w0_0.getSignificance(), w0.getSignificance()); - assertEquals(w0_1.getSignificance(), w1.getSignificance()); - - } @Test public void failsOnConflictingSignificanceConfiguration() { @@ -252,4 +309,53 @@ public class SignificanceSearcherTest { "(https://docs.vespa.ai/en/reference/schema-reference.html#significance).", errorMessage.getDetailedMessage()); } + + @Test + public void testSignificanceSearcherWithExplictitAndImplictSetLanguages() { + Query q = new Query(); + q.getModel().setLanguage(Language.UNKNOWN); + q.getRanking().setProfile("significance-ranking"); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("hello", true); + root.addItem(tmp); + + q.getModel().getQueryTree().setRoot(root); + + SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get(); + var helloFrequency = model.documentFrequency("hello"); + var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency()); + Result r = createExecution(searcher).search(q); + + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + assertEquals(helloSignificanceValue, w0.getSignificance()); + + + Query q2 = new Query(); + q2.getModel().setLanguage(Language.FRENCH); + q2.getRanking().setProfile("significance-ranking"); + AndItem root2 = new AndItem(); + WordItem tmp2; + tmp2 = new WordItem("hello", true); + root2.addItem(tmp2); + + q2.getModel().getQueryTree().setRoot(root2); + Result r2 = createExecution(searcher).search(q2); + + assertEquals(1, r2.hits().getErrorHit().errors().size()); + + + Query q3 = new Query(); + q3.getRanking().setProfile("significance-ranking"); + WordItem root3 = new WordItem("Я с детства хотел завести собаку, но родители мне не разрешали.", true); + + q3.getModel().getQueryTree().setRoot(root3); + Execution execution = createExecution(searcher, Language.RUSSIAN); + Result r3 = execution.search(q3); + + assertEquals(1, r3.hits().getErrorHit().errors().size()); + + + } } diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index f172b09b7fe..f1437950201 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -68,7 +68,7 @@ <assertj.vespa.version>3.26.0</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.748</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.752</aws-sdk.vespa.version> <athenz.vespa.version>1.11.60</athenz.vespa.version> <!-- Athenz END --> @@ -119,7 +119,7 @@ <junit4.vespa.version>4.13.2</junit4.vespa.version> <kherud.llama.vespa.version>3.2.1</kherud.llama.vespa.version> <luben.zstd.vespa.version>1.5.6-3</luben.zstd.vespa.version> - <lucene.vespa.version>9.11.0</lucene.vespa.version> + <lucene.vespa.version>9.11.1</lucene.vespa.version> <maven-archiver.vespa.version>3.6.2</maven-archiver.vespa.version> <maven-wagon.vespa.version>3.5.3</maven-wagon.vespa.version> <maven-xml-impl.vespa.version>4.0.0-beta-3</maven-xml-impl.vespa.version> @@ -147,7 +147,7 @@ <surefire.vespa.version>3.3.0</surefire.vespa.version> <velocity.vespa.version>2.3</velocity.vespa.version> <velocity.tools.vespa.version>3.1</velocity.tools.vespa.version> - <wiremock.vespa.version>3.7.0</wiremock.vespa.version> + <wiremock.vespa.version>3.8.0</wiremock.vespa.version> <woodstox.vespa.version>7.0.0</woodstox.vespa.version> <stax2-api.vespa.version>4.2.2</stax2-api.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> diff --git a/dist/release-vespa-rpm.sh b/dist/release-vespa-rpm.sh index d426ed26de3..ffa4c8e3e2e 100755 --- a/dist/release-vespa-rpm.sh +++ b/dist/release-vespa-rpm.sh @@ -8,11 +8,6 @@ if [[ $# -ne 2 ]]; then exit 1 fi -if [[ -z $COPR_WEBHOOK ]]; then - echo "This script requires the COPR_WEBHOOK environment variable to be set." - exit 1 -fi - readonly VERSION=$1 readonly GITREF=$2 readonly RELEASE_TAG="v$VERSION" diff --git a/dist/vespa.spec b/dist/vespa.spec index 9e617fb8acd..86f9679c27e 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -508,6 +508,7 @@ fi %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/search %{_prefix}/man %{_prefix}/sbin +%dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/secure %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/crash %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db diff --git a/document/src/tests/annotation/annotation_test.cpp b/document/src/tests/annotation/annotation_test.cpp index c5635b9f4f9..34bc67d0238 100644 --- a/document/src/tests/annotation/annotation_test.cpp +++ b/document/src/tests/annotation/annotation_test.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for annotation. -#include <stdlib.h> #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/document/annotation/annotation.h> #include <vespa/document/annotation/span.h> @@ -18,7 +17,7 @@ #include <vespa/document/fieldvalue/doublefieldvalue.h> #include <vespa/document/fieldvalue/structfieldvalue.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <memory> +#include <vespa/vespalib/testkit/test_master.hpp> using std::unique_ptr; using namespace document; diff --git a/document/src/tests/datatype/datatype_test.cpp b/document/src/tests/datatype/datatype_test.cpp index 1362afe8dbc..9ed86378883 100644 --- a/document/src/tests/datatype/datatype_test.cpp +++ b/document/src/tests/datatype/datatype_test.cpp @@ -7,8 +7,9 @@ #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/eval/eval/value_type.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace document; diff --git a/document/src/tests/datatype/referencedatatype_test.cpp b/document/src/tests/datatype/referencedatatype_test.cpp index 54489feb2f2..a2d564364ba 100644 --- a/document/src/tests/datatype/referencedatatype_test.cpp +++ b/document/src/tests/datatype/referencedatatype_test.cpp @@ -3,10 +3,10 @@ #include <vespa/document/base/field.h> #include <vespa/document/datatype/referencedatatype.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> -#include <ostream> #include <sstream> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace document; diff --git a/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp b/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp index edb7f8e4f45..4db2ea38d7e 100644 --- a/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp +++ b/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp @@ -5,6 +5,7 @@ #include <vespa/document/repo/document_type_repo_factory.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using vespalib::string; using namespace document::config_builder; diff --git a/document/src/tests/fieldvalue/referencefieldvalue_test.cpp b/document/src/tests/fieldvalue/referencefieldvalue_test.cpp index 57b313b2237..98f9a2dbfbb 100644 --- a/document/src/tests/fieldvalue/referencefieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/referencefieldvalue_test.cpp @@ -4,9 +4,10 @@ #include <vespa/document/datatype/referencedatatype.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <ostream> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace document; diff --git a/document/src/tests/predicate/predicate_test.cpp b/document/src/tests/predicate/predicate_test.cpp index 327d05743a0..ccd4048c1a7 100644 --- a/document/src/tests/predicate/predicate_test.cpp +++ b/document/src/tests/predicate/predicate_test.cpp @@ -195,8 +195,8 @@ TEST("require that feature range can be open") { EXPECT_EQUAL(feature_name, set.getKey()); EXPECT_FALSE(set.hasMin()); EXPECT_FALSE(set.hasMax()); - EXPECT_EQUAL(LLONG_MIN, set.getMin()); - EXPECT_EQUAL(LLONG_MAX, set.getMax()); + EXPECT_EQUAL(LONG_MIN, set.getMin()); + EXPECT_EQUAL(LONG_MAX, set.getMax()); } PredicateNode::UP getPredicateNode() { diff --git a/document/src/tests/repo/doctype_config_test.cpp b/document/src/tests/repo/doctype_config_test.cpp index 0705b53c2df..c3c0d375a95 100644 --- a/document/src/tests/repo/doctype_config_test.cpp +++ b/document/src/tests/repo/doctype_config_test.cpp @@ -14,9 +14,10 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <set> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("doctype_config_test"); diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index b2d79209c12..a26b65f73f7 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -14,10 +14,11 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <set> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using config::AsciiConfigWriter; using std::set; diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 65a77b1f872..ae711d81bd2 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -43,10 +43,11 @@ #include <vespa/eval/eval/test/value_compare.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/exceptions.h> #include <vespa/vespalib/util/compressionconfig.h> #include <filesystem> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using vespalib::File; using vespalib::Slime; diff --git a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp index 4cc4e5e6215..a84b3e0f93b 100644 --- a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp +++ b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp @@ -1,9 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for tensor_fieldvalue. -#include <vespa/log/log.h> -LOG_SETUP("fieldvalue_test"); - #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> @@ -12,6 +9,10 @@ LOG_SETUP("fieldvalue_test"); #include <vespa/eval/eval/value.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> + +#include <vespa/log/log.h> +LOG_SETUP("fieldvalue_test"); using namespace document; using vespalib::eval::SimpleValue; diff --git a/documentapi/src/tests/priority/priority.cpp b/documentapi/src/tests/priority/priority.cpp index d0290495035..f8c55864289 100644 --- a/documentapi/src/tests/priority/priority.cpp +++ b/documentapi/src/tests/priority/priority.cpp @@ -1,9 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/documentapi/messagebus/priority.h> #include <fstream> #include <algorithm> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace documentapi; diff --git a/documentapi/src/vespa/documentapi/messagebus/routable_factories_8.cpp b/documentapi/src/vespa/documentapi/messagebus/routable_factories_8.cpp index 9ef54932f68..5a351676089 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routable_factories_8.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routable_factories_8.cpp @@ -669,7 +669,7 @@ std::shared_ptr<IRoutableFactory> RoutableFactories80::visitor_info_message_fact return make_codec<VisitorInfoMessage, protobuf::VisitorInfoRequest>( [](const VisitorInfoMessage& src, protobuf::VisitorInfoRequest& dest) { set_bucket_id_vector(*dest.mutable_finished_buckets(), src.getFinishedBuckets()); - dest.set_error_message(src.getErrorMessage()); + dest.set_error_message(static_cast<std::string_view>(src.getErrorMessage())); }, [](const protobuf::VisitorInfoRequest& src) { auto msg = std::make_unique<VisitorInfoMessage>(); @@ -878,7 +878,7 @@ std::shared_ptr<IRoutableFactory> RoutableFactories80::stat_bucket_message_facto std::shared_ptr<IRoutableFactory> RoutableFactories80::stat_bucket_reply_factory() { return make_codec<StatBucketReply, protobuf::StatBucketResponse>( [](const StatBucketReply& src, protobuf::StatBucketResponse& dest) { - dest.set_results(src.getResults()); + dest.set_results(static_cast<std::string_view>(src.getResults())); }, [](const protobuf::StatBucketResponse& src) { auto reply = std::make_unique<StatBucketReply>(); @@ -895,7 +895,7 @@ std::shared_ptr<IRoutableFactory> RoutableFactories80::stat_bucket_reply_factory std::shared_ptr<IRoutableFactory> RoutableFactories80::wrong_distribution_reply_factory() { return make_codec<WrongDistributionReply, protobuf::WrongDistributionResponse>( [](const WrongDistributionReply& src, protobuf::WrongDistributionResponse& dest) { - dest.mutable_cluster_state()->set_state_string(src.getSystemState()); + dest.mutable_cluster_state()->set_state_string(static_cast<std::string_view>(src.getSystemState())); }, [](const protobuf::WrongDistributionResponse& src) { auto reply = std::make_unique<WrongDistributionReply>(); diff --git a/eval/src/tests/eval/aggr/aggr_test.cpp b/eval/src/tests/eval/aggr/aggr_test.cpp index f4eb6dc6ddc..0a849dc43ec 100644 --- a/eval/src/tests/eval/aggr/aggr_test.cpp +++ b/eval/src/tests/eval/aggr/aggr_test.cpp @@ -197,17 +197,17 @@ float aggr_merge(const std::vector<float> &a, const std::vector<float> &b) { TEST("require that aggregator merge works") { float my_nan = std::numeric_limits<float>::quiet_NaN(); - EXPECT_EQUAL(aggr_merge<Avg>({1,2},{3,4}), 2.5); - EXPECT_EQUAL(aggr_merge<Count>({1,2},{3,4}), 4.0); - EXPECT_EQUAL(aggr_merge<Prod>({1,2},{3,4}), 24.0); - EXPECT_EQUAL(aggr_merge<Sum>({1,2},{3,4}), 10.0); - EXPECT_EQUAL(aggr_merge<Max>({1,2},{3,4}), 4.0); - EXPECT_EQUAL(aggr_merge<Median>({1,2},{3,4}), 2.5); - EXPECT_EQUAL(aggr_merge<Median>({1,2},{3,4,5}), 3); - EXPECT_EQUAL(aggr_merge<Median>({0,1,2},{3,4}), 2); + EXPECT_EQUAL(aggr_merge<Avg>({1,2},{3,4}), 2.5f); + EXPECT_EQUAL(aggr_merge<Count>({1,2},{3,4}), 4.0f); + EXPECT_EQUAL(aggr_merge<Prod>({1,2},{3,4}), 24.0f); + EXPECT_EQUAL(aggr_merge<Sum>({1,2},{3,4}), 10.0f); + EXPECT_EQUAL(aggr_merge<Max>({1,2},{3,4}), 4.0f); + EXPECT_EQUAL(aggr_merge<Median>({1,2},{3,4}), 2.5f); + EXPECT_EQUAL(aggr_merge<Median>({1,2},{3,4,5}), 3.0f); + EXPECT_EQUAL(aggr_merge<Median>({0,1,2},{3,4}), 2.0f); EXPECT_TRUE(std::isnan(aggr_merge<Median>({1,2,my_nan,3},{4,5}))); EXPECT_TRUE(std::isnan(aggr_merge<Median>({1,2,3},{4,my_nan,5}))); - EXPECT_EQUAL(aggr_merge<Min>({1,2},{3,4}), 1.0); + EXPECT_EQUAL(aggr_merge<Min>({1,2},{3,4}), 1.0f); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/eval/param_usage/param_usage_test.cpp b/eval/src/tests/eval/param_usage/param_usage_test.cpp index 3ba76c726b5..a2f4cfc30e4 100644 --- a/eval/src/tests/eval/param_usage/param_usage_test.cpp +++ b/eval/src/tests/eval/param_usage/param_usage_test.cpp @@ -1,8 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/param_usage.h> #include <vespa/eval/eval/llvm/compiled_function.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/test/insertion_operators.h> using vespalib::approx_equal; diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index f472a5fb6a6..9663969ae80 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/bfloat16.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <ostream> using vespalib::BFloat16; diff --git a/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp b/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp index 1a81c35ba5f..06c59fbb321 100644 --- a/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp +++ b/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp @@ -1,12 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/instruction/join_with_number_function.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/unwind_message.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib; using namespace vespalib::eval; diff --git a/eval/src/tests/instruction/mixed_simple_join_function/mixed_simple_join_function_test.cpp b/eval/src/tests/instruction/mixed_simple_join_function/mixed_simple_join_function_test.cpp index cf99dd4e437..ab7854b0e27 100644 --- a/eval/src/tests/instruction/mixed_simple_join_function/mixed_simple_join_function_test.cpp +++ b/eval/src/tests/instruction/mixed_simple_join_function/mixed_simple_join_function_test.cpp @@ -1,12 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/instruction/mixed_simple_join_function.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/util/stringfmt.h> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 67f1cf25510..4097e0034d0 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -48,6 +48,23 @@ public class Flags { private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); + public static final UnboundBooleanFlag USE_VESPA_ATHENZ = defineFeatureFlag( + "use-vespa-athenz", false, + List.of("hakonhall"), "2024-06-25", "2024-10-25", + "Whether to talk to Vespa Athenz instead of Yahoo Athenz in public systems. " + + "node-type is config in config server, controller in controller, and the appropriate " + + "host node type in host-admin.", + "Takes immediate effect wherever possible.", + NODE_TYPE); + + public static final UnboundBooleanFlag USE_SYSTEM_DOMAIN_FOR_AWS_CREDS = defineFeatureFlag( + "use-system-domain-for-aws-creds", false, + List.of("freva"), "2024-06-27", "2024-09-01", + "Whether to use system athenz domain (instead of sys.auth) when assuming role for AWS" + + "temp credentials against Vespa Athenz", + "Takes immediate effect wherever possible.", + NODE_TYPE); + public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, List.of("baldersheim"), "2020-12-02", "2024-12-31", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 7d1c6bcecdb..a79c90877c1 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -469,6 +469,13 @@ public class PermanentFlags { list -> list.stream().allMatch(s -> s.matches("^[a-zA-Z/.0-9-]+:(0(\\.\\d+)?|1(\\.0+)?):\\d+(B|kB|MB|GB)?$")), INSTANCE_ID); + public static final UnboundIntFlag ZOOKEEPER_JUTE_MAX_BUFFER = defineIntFlag( + "zookeeper-jute-max-buffer", 104857600, + "Jute maxbuffer. Used by zookeeper to determine max buffer when serializing/deserializing." + + "Values used in server and client must correspond (so if decreasing this one must be sure" + + "that no node has stored more bytes than this)", + "Takes effect on next reboot of config server"); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp index 6466848f216..809cdce2bdd 100644 --- a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp +++ b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/transport_thread.h> #include <vespa/fnet/frt/supervisor.h> @@ -11,6 +10,8 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/time.h> #include <thread> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib; using vespalib::make_string_short::fmt; diff --git a/fnet/src/tests/frt/rpc/invoke.cpp b/fnet/src/tests/frt/rpc/invoke.cpp index 7fb17468db4..314c1aac3d1 100644 --- a/fnet/src/tests/frt/rpc/invoke.cpp +++ b/fnet/src/tests/frt/rpc/invoke.cpp @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/tls/capability_env_config.h> #include <vespa/vespalib/net/tls/statistics.h> @@ -14,6 +13,8 @@ #include <mutex> #include <condition_variable> #include <string_view> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using vespalib::SocketSpec; using vespalib::BenchmarkTimer; diff --git a/fnet/src/tests/transport_debugger/transport_debugger_test.cpp b/fnet/src/tests/transport_debugger/transport_debugger_test.cpp index 859c6e5bb3d..e5e53ff3f0c 100644 --- a/fnet/src/tests/transport_debugger/transport_debugger_test.cpp +++ b/fnet/src/tests/transport_debugger/transport_debugger_test.cpp @@ -12,6 +12,7 @@ #include <vespa/fnet/frt/target.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> #include <vespa/vespalib/test/make_tls_options_for_testing.h> +#include <vespa/vespalib/testkit/test_master.hpp> vespalib::CryptoEngine::SP tls_crypto = std::make_shared<vespalib::TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()); diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java index 90e5eeb99ba..7080dce432e 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java @@ -2,6 +2,7 @@ package com.yahoo.jdisc.http.filter.security.base; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.component.chain.ChainedComponent; import com.yahoo.json.Jackson; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -25,7 +26,7 @@ import java.util.logging.Logger; * * @author bjorncs */ -public abstract class JsonSecurityRequestFilterBase extends AbstractComponent implements SecurityRequestFilter { +public abstract class JsonSecurityRequestFilterBase extends ChainedComponent implements SecurityRequestFilter { private static final Logger log = Logger.getLogger(JsonSecurityRequestFilterBase.class.getName()); diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java index 6f3a108e9e1..45c6054f748 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java @@ -18,6 +18,9 @@ import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.ArrayList; +import java.util.Objects; +import java.util.logging.Logger; /** * Default implementation of {@link SignificanceModelRegistry}. @@ -27,6 +30,8 @@ import java.util.Optional; */ public class DefaultSignificanceModelRegistry implements SignificanceModelRegistry { + private static final Logger log = Logger.getLogger(DefaultSignificanceModelRegistry.class.getName()); + private final Map<Language, SignificanceModel> models; @Inject @@ -45,6 +50,7 @@ public class DefaultSignificanceModelRegistry implements SignificanceModelRegist } public void addModel(Path path) { + log.fine(() -> "Loading model from " + path); ObjectMapper objectMapper = new ObjectMapper(); try { InputStream in = path.toString().endsWith(".zst") ? @@ -53,9 +59,16 @@ public class DefaultSignificanceModelRegistry implements SignificanceModelRegist SignificanceModelFile file = objectMapper.readValue(in, SignificanceModelFile.class); for (var pair : file.languages().entrySet()) { - this.models.put( - Language.fromLanguageTag(pair.getKey()), - new DefaultSignificanceModel(pair.getValue(), file.id())); + + var languagesStr = pair.getKey(); + log.fine(() -> "Found model for languages '%s'".formatted(languagesStr)); + String[] languageTags = languagesStr.split(","); + + for (var languageTag : languageTags) { + var language = Language.fromLanguageTag(languageTag); + log.fine(() -> "Adding model for language %s with id %s".formatted(language, file.id())); + this.models.put(language, new DefaultSignificanceModel(pair.getValue(), file.id())); + } } } catch (IOException e) { throw new UncheckedIOException("Failed to load model from " + path, e); diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java index 94030108671..fd9b206b334 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java @@ -3,11 +3,9 @@ package com.yahoo.language.significance.impl; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.HashMap; -import java.util.List; /** * diff --git a/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java b/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java index a5be567717e..ead0f2c15c3 100644 --- a/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java +++ b/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java @@ -35,7 +35,7 @@ public class DefaultSignificanceModelRegistryTest { var englishModel = optionalEnglishModel.get(); var norwegianModel = optionalNorwegianModel.get(); - assertTrue( defaultSignificanceModelRegistry.getModel(Language.FRENCH).isEmpty()); + assertTrue( defaultSignificanceModelRegistry.getModel(Language.GERMAN).isEmpty()); assertNotNull(englishModel); assertNotNull(norwegianModel); @@ -55,6 +55,37 @@ public class DefaultSignificanceModelRegistryTest { } @Test + public void testDefaultSignificanceModelRegistryWithUnknownLanguage() { + List<Path> models = new ArrayList<>(); + + models.add(Path.of("src/test/models/docv2.json")); + + DefaultSignificanceModelRegistry defaultSignificanceModelRegistry = new DefaultSignificanceModelRegistry(models); + + assertTrue(defaultSignificanceModelRegistry.getModel(Language.ENGLISH).isPresent()); + assertTrue(defaultSignificanceModelRegistry.getModel(Language.FRENCH).isPresent()); + assertTrue(defaultSignificanceModelRegistry.getModel(Language.UNKNOWN).isPresent()); + + var frenchModel = defaultSignificanceModelRegistry.getModel(Language.FRENCH).get(); + var unknownModel = defaultSignificanceModelRegistry.getModel(Language.UNKNOWN).get(); + + assertEquals("test::2", frenchModel.getId()); + assertEquals("test::2", unknownModel.getId()); + + assertEquals(1, frenchModel.documentFrequency("non-existent-word").frequency()); + assertEquals(20, frenchModel.documentFrequency("non-existent-word").corpusSize()); + + assertEquals(1, unknownModel.documentFrequency("non-existent-word").frequency()); + assertEquals(20, unknownModel.documentFrequency("non-existent-word").corpusSize()); + + assertEquals(6, frenchModel.documentFrequency("bonjour").frequency()); + assertEquals(20, frenchModel.documentFrequency("bonjour").corpusSize()); + assertEquals(6, unknownModel.documentFrequency("bonjour").frequency()); + assertEquals(20, unknownModel.documentFrequency("bonjour").corpusSize()); + + } + + @Test public void testDefaultSignificanceModelRegistryWithZSTDecompressing() { List<Path> models = new ArrayList<>(); diff --git a/linguistics/src/test/models/docv2.json b/linguistics/src/test/models/docv2.json index c00d02fb744..1f9d3779f26 100644 --- a/linguistics/src/test/models/docv2.json +++ b/linguistics/src/test/models/docv2.json @@ -26,6 +26,19 @@ "ja": 12, "nei": 3 } + }, + "fr,un": { + "description" : "french / unknown model", + "document-count" : 20, + "document-frequencies" : { + "usa" : 2, + "bonjour": 6, + "monde": 5, + "test": 2, + "france": 11, + "oui": 12, + "no": 3 + } } } } diff --git a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp index 835ac26d5c1..f050a993e0a 100644 --- a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp +++ b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp @@ -2,12 +2,13 @@ // Unit tests for dummypersistence. #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/documentid.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/config-stor-distribution.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace storage::spi; using namespace storage; diff --git a/screwdriver.yaml b/screwdriver.yaml index edc3cfee3c1..0adca9bd03d 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -247,6 +247,7 @@ jobs: screwdriver.cd/buildPeriodically: H 4,10,16,22 * * 1,2,3,4 secrets: + - BUILDKITE_TRIGGER_TOKEN - COPR_WEBHOOK - OSSRH_USER - OSSRH_TOKEN @@ -276,20 +277,12 @@ jobs: return 1 fi meta set vespa.version $VESPA_VERSION - - install-dependencies: | - dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - dnf install -y docker-ce docker-ce-cli containerd.io python3 python3-pip - docker system info - - release-java-artifacts: | - screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF - - release-rpms: | - screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF - - release-container-image: | - screwdriver/release-container-image-docker.sh $VESPA_VERSION - - release-ann-benchmark: | - screwdriver/release-ann-benchmark.sh $VESPA_VERSION - - update-sample-apps: | - screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION + - trig-buildkite: | + BUILDKITE_JSON=$(screwdriver/trig-buildkite-pipeline.sh vespa-engine-vespa-release master $VESPA_VERSION $VESPA_REF) + - wait-for-buildkite: | + BUILDKITE_URL=$(jq -re '.url' <<< "$BUILDKITE_JSON") + BUILDKITE_WEB_URL=$(jq -re '.web_url' <<< "$BUILDKITE_JSON") + screwdriver/wait-for-buildkite-build.sh $BUILDKITE_URL $BUILDKITE_WEB_URL 1800 - update-released-time: | screwdriver/factory-command.sh update-released-time $VESPA_VERSION diff --git a/screwdriver/release-container-image-crane.sh b/screwdriver/release-container-image-crane.sh new file mode 100755 index 00000000000..9d39cffdcab --- /dev/null +++ b/screwdriver/release-container-image-crane.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 <Vespa version>" + exit 1 +fi + +readonly VESPA_VERSION=$1 +readonly VESPA_MAJOR=$(echo $VESPA_VERSION | cut -d. -f1) + +if [[ -z "$DOCKER_HUB_DEPLOY_TOKEN" ]]; then + echo "Environment variable DOCKER_HUB_DEPLOY_TOKEN must be set, but is empty." + exit 1 +fi +if [[ -z "$GHCR_DEPLOY_TOKEN" ]]; then + echo "Environment variable GHCR_DEPLOY_TOKEN must be set, but is empty." + exit 1 +fi + +crane auth login -u aressem -p $GHCR_DEPLOY_TOKEN ghcr.io +SRC_IMAGE=ghcr.io/vespa-engine/vespa-preview:$VESPA_VERSION +SRC_IMAGE_DIGEST=$(crane digest $SRC_IMAGE) +cosign verify \ + --certificate-identity https://buildkite.com/vespaai/vespa-engine-vespa \ + --certificate-oidc-issuer https://agent.buildkite.com \ + $SRC_IMAGE@$SRC_IMAGE_DIGEST + +# Copy to Docker Hub +if curl -fsSL https://hub.docker.com/v2/repositories/vespaengine/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then + echo "Container image docker.io/vespaengine/$IMAGE_NAME:$VESPA_VERSION already exists." +else + DST_IMAGE=docker.io/vespaengine/vespa:$VESPA_VERSION + crane auth login -u aressem -p $DOCKER_HUB_DEPLOY_TOKEN docker.io + crane cp $SRC_IMAGE@$SRC_IMAGE_DIGEST $DST_IMAGE + crane tag $DST_IMAGE $VESPA_MAJOR + crane tag $DST_IMAGE latest +fi + +# Copy to GitHub Container Registry +JWT=$(curl -sSL -u aressem:$GHCR_DEPLOY_TOKEN "https://ghcr.io/token?service=ghcr.io&scope=repository:vespa-engine/vespa:pull" | jq -re '.token') +IMAGE_TAGS=$(curl -sSL -H "Authorization: Bearer $JWT" https://ghcr.io/v2/vespa-engine/vespa/tags/list | jq -re '.tags[]') +if grep $VESPA_VERSION <<< "$IMAGE_TAGS" &> /dev/null; then + echo "Container image ghcr.io/vespa-engine/vespa:$VESPA_VERSION already exists." +else + DST_IMAGE=ghcr.io/vespa-engine/vespa:$VESPA_VERSION + crane cp $SRC_IMAGE@$SRC_IMAGE_DIGEST $DST_IMAGE + crane tag $DST_IMAGE $VESPA_MAJOR + crane tag $DST_IMAGE latest +fi diff --git a/screwdriver/release-java-artifacts.sh b/screwdriver/release-java-artifacts.sh index e0107b36cf5..b835ec8cf98 100755 --- a/screwdriver/release-java-artifacts.sh +++ b/screwdriver/release-java-artifacts.sh @@ -3,18 +3,17 @@ set -euo pipefail -if [[ $# -ne 2 ]]; then - echo "Usage: $0 <Vespa version> <Git reference>" +if [[ $# -ne 1 ]]; then + echo "Usage: $0 <Vespa release version>" exit 1 fi -if [[ -z $OSSRH_USER ]] || [[ -z $OSSRH_TOKEN ]] || [[ -z $GPG_KEYNAME ]] || [[ -z $GPG_PASSPHRASE ]] || [[ -z $GPG_ENCPHRASE ]]; then - echo -e "The follwing env variables must be set:\n OSSRH_USER\n OSSRH_TOKEN\n GPG_KEYNAME\n GPG_PASSPHRASE\n GPG_ENCPHRASE" +if [[ -z $OSSRH_USER ]] || [[ -z $OSSRH_TOKEN ]] || [[ -z $GPG_KEYNAME ]] || [[ -z $GPG_PASSPHRASE_TOKEN ]] || [[ -z $GPG_ENCPHRASE_TOKEN ]]; then + echo -e "The follwing env variables must be set:\n OSSRH_USER\n OSSRH_TOKEN\n GPG_KEYNAME\n GPG_PASSPHRASE_TOKEN\n GPG_ENCPHRASE_TOKEN" exit 1 fi readonly VESPA_RELEASE="$1" -readonly VESPA_REF="$2" QUERY_VERSION_HTTP_CODE=$(curl --write-out %{http_code} --silent --location --output /dev/null https://oss.sonatype.org/content/repositories/releases/com/yahoo/vespa/parent/${VESPA_RELEASE}/parent-${VESPA_RELEASE}.pom) if [[ "200" == $QUERY_VERSION_HTTP_CODE ]]; then @@ -22,63 +21,93 @@ if [[ "200" == $QUERY_VERSION_HTTP_CODE ]]; then exit 0 fi -export JAVA_HOME=$(dirname $(dirname $(readlink -f /usr/bin/java))) +export SOURCE_DIR=$(pwd) -VESPA_DIR=vespa-clean -git clone https://github.com/vespa-engine/vespa.git $VESPA_DIR +mkdir -p $SOURCE_DIR/screwdriver/deploy +# gpg-agent in RHEL 8 runs out of memory if we use Maven and sign in parallel. Add option to overcome this. +echo "auto-expand-secmem" >> $SOURCE_DIR/screwdriver/deploy/gpg-agent.conf +openssl aes-256-cbc -md md5 -pass pass:$GPG_ENCPHRASE_TOKEN -in $SOURCE_DIR/screwdriver/pubring.gpg.enc -out $SOURCE_DIR/screwdriver/deploy/pubring.gpg -d +openssl aes-256-cbc -md md5 -pass pass:$GPG_ENCPHRASE_TOKEN -in $SOURCE_DIR/screwdriver/secring.gpg.enc -out $SOURCE_DIR/screwdriver/deploy/secring.gpg -d +chmod 700 $SOURCE_DIR/screwdriver/deploy +chmod 600 $SOURCE_DIR/screwdriver/deploy/* +trap "rm -rf $SOURCE_DIR/screwdriver/deploy" EXIT + +# Number of parallel uploads +NUM_PROC=10 + +export MVN=${MVN:-mvn} +export MVN_OPTS=${MVN_OPTS:-} +export MAVEN_GPG_PASSPHRASE=$GPG_PASSPHRASE_TOKEN + +export TMP_STAGING=$(mktemp -d) +mkdir -p $TMP_STAGING +trap "rm -rf $TMP_STAGING" EXIT + +sign_module() { + ECHO="" + + P=$1 + V=$(basename $P) + A=$(basename $(dirname $P)) + G=$(dirname $(dirname $P) | sed 's,/,.,g') + POM="$P/$A-$V.pom" + JAR="$P/$A-$V.jar" + if [[ -f $JAR ]]; then + FILE_OPTS="-Dfile=$JAR -Dpackaging=jar" + else + FILE_OPTS="-Dfile=$POM -Dpackaging=pom" + fi + + AFILES=() + ATYPES=() + ACLASSIFIERS=() + for EX in $(find $P -name "$A-$V-*.jar"); do + AFILES+=($EX) + EXB=$(basename $EX) + EXT=${EXB##*.} + ATYPES+=($EXT) + EXF=${EXB%.*} + EXC=$(echo $EXF|sed "s,$A-$V-,,") + ACLASSIFIERS+=($EXC) + done + + if (( ${#AFILES[@]} > 0 )); then + EXTRA_FILES_OPTS="-Dfiles=$(echo ${AFILES[@]} | sed 's/\ /,/g') -Dtypes=$(echo ${ATYPES[@]} | sed 's/\ /,/g') -Dclassifiers=$(echo ${ACLASSIFIERS[@]} | sed 's/\ /,/g')" + fi + + $ECHO $MVN --settings=$SOURCE_DIR/screwdriver/settings-publish.xml \ + $MVN_OPTS gpg:sign-and-deploy-file \ + -Durl=file://$TMP_STAGING \ + -DrepositoryId=maven-central \ + -DgroupId=$G \ + -DartifactId=$A \ + -Dversion=$V \ + -DpomFile=$POM \ + -DgeneratePom=false \ + $FILE_OPTS \ + $EXTRA_FILES_OPTS + +} +export -f sign_module + +aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/amd64/maven-repo.tar . +aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/amd64/maven-repo.tar.pem . +aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/amd64/maven-repo.tar.sig . +cosign verify-blob --certificate-identity https://buildkite.com/vespaai/vespa-engine-vespa --certificate-oidc-issuer https://agent.buildkite.com --signature maven-repo.tar.sig --certificate maven-repo.tar.pem maven-repo.tar +rm -rf maven-repo +tar xvf maven-repo.tar +REPO_ROOT=$(pwd)/maven-repo + +cd $REPO_ROOT +find . -name "$VESPA_RELEASE" -type d | sed 's,^./,,' | xargs -n 1 -P $NUM_PROC -I '{}' bash -c "sign_module {}" + +$MVN --settings=$SOURCE_DIR/screwdriver/settings-publish.xml \ + org.sonatype.plugins:nexus-staging-maven-plugin:1.6.14:deploy-staged-repository \ + -DrepositoryDirectory=$TMP_STAGING \ + -DnexusUrl=https://oss.sonatype.org \ + -DserverId=ossrh \ + -DautoReleaseAfterClose=true \ + -DstagingProfileId=407c0c3e1a197 -cd $VESPA_DIR -git checkout $VESPA_REF -mkdir -p $SD_SOURCE_DIR/screwdriver/deploy -# gpg-agent in RHEL 8 runs out of memory if we use Maven and sign in parallel. Add option to overcome this. -echo "auto-expand-secmem" >> $SD_SOURCE_DIR/screwdriver/deploy/gpg-agent.conf -openssl aes-256-cbc -md md5 -pass pass:$GPG_ENCPHRASE -in $SD_SOURCE_DIR/screwdriver/pubring.gpg.enc -out $SD_SOURCE_DIR/screwdriver/deploy/pubring.gpg -d -openssl aes-256-cbc -md md5 -pass pass:$GPG_ENCPHRASE -in $SD_SOURCE_DIR/screwdriver/secring.gpg.enc -out $SD_SOURCE_DIR/screwdriver/deploy/secring.gpg -d -chmod 700 $SD_SOURCE_DIR/screwdriver/deploy -chmod 600 $SD_SOURCE_DIR/screwdriver/deploy/* - -# Build the Java code with the correct version set -$SD_SOURCE_DIR/screwdriver/replace-vespa-version-in-poms.sh $VESPA_RELEASE . - -# We disable javadoc for all modules not marked as public API -for MODULE in $(comm -2 -3 \ - <(find . -name "*.java" | awk -F/ '{print $2}' | sort -u) - <(find . -name "package-info.java" -exec grep -HnE "@(com.yahoo.api.annotations.)?PublicApi.*" {} \; | awk -F/ '{print $2}' | sort -u)); do - mkdir -p $MODULE/src/main/javadoc - echo "No javadoc available for module" > $MODULE/src/main/javadoc/README -done - -# Workaround new module without java code and no javadoc -mkdir -p container-spifly/src/main/javadoc -echo "No javadoc available for module" > container-spifly/src/main/javadoc/README - -# Workaround for broken nexus-staging-maven-plugin instead of swapping JDK -export MAVEN_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.desktop/java.awt.font=ALL-UNNAMED" -export VESPA_MAVEN_EXTRA_OPTS="--show-version --batch-mode" -export FACTORY_VESPA_VERSION=$VESPA_RELEASE -./bootstrap.sh - -COMMON_MAVEN_OPTS="$VESPA_MAVEN_EXTRA_OPTS --no-snapshot-updates --settings $(pwd)/screwdriver/settings-publish.xml --activate-profiles ossrh-deploy-vespa -DskipTests" -TMPFILE=$(mktemp) -mvn $COMMON_MAVEN_OPTS -pl :dependency-versions -DskipStagingRepositoryClose=true deploy 2>&1 | tee $TMPFILE - -# Find the stage repo name -STG_REPO=$(cat $TMPFILE | grep 'Staging repository at http' | head -1 | awk -F/ '{print $NF}') -rm -f $TMPFILE - -# Deploy plugins -mvn $COMMON_MAVEN_OPTS --file ./maven-plugins/pom.xml -DskipStagingRepositoryClose=true -DstagingRepositoryId=$STG_REPO deploy - -# Deploy the rest of the artifacts -mvn $COMMON_MAVEN_OPTS --threads 8 -DskipStagingRepositoryClose=true -DstagingRepositoryId=$STG_REPO deploy - -# Close with checks -mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.12:rc-close -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO - -# Release if ok -mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.12:rc-release -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO - -# Delete the GPG rings -rm -rf $SD_SOURCE_DIR/screwdriver/deploy diff --git a/screwdriver/release-rpms.sh b/screwdriver/release-rpms.sh index 76ab74e9a5c..b4934db544b 100755 --- a/screwdriver/release-rpms.sh +++ b/screwdriver/release-rpms.sh @@ -12,10 +12,16 @@ fi readonly VESPA_RELEASE="$1" readonly VESPA_REF="$2" -VESPA_RPM_X86_64=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-8-x86_64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) +# Cloudsmith repo +rpm --import 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/gpg.0F3DA3C70D35DA7B.key' +curl -1sLf 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/config.rpm.txt?distro=el&codename=8' > /tmp/vespa-open-source-rpms.repo +dnf config-manager --add-repo '/tmp/vespa-open-source-rpms.repo' +rm -f /tmp/vespa-open-source-rpms.repo + +VESPA_RPM_X86_64=$(dnf repoquery -y --forcearch x86_64 --repoid=vespa-open-source-rpms -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) echo "Latest x86_64 RPM on Copr: $VESPA_RPM_X86_64" -VESPA_RPM_AARCH64=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-8-aarch64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) +VESPA_RPM_AARCH64=$(dnf repoquery -y --forcearch aarch64 --repoid=vespa-open-source-rpms -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) echo "Latest aarch64 RPM on Copr: $VESPA_RPM_AARCH64" if [[ "$VESPA_RELEASE" == "$VESPA_RPM_X86_64" ]] && [[ "$VESPA_RELEASE" == "$VESPA_RPM_AARCH64" ]]; then @@ -27,19 +33,35 @@ echo "Using vespa repository git reference: $VESPA_REF" ssh-add -D set +x -ssh-add <(echo $VESPA_DEPLOY_KEY | base64 -d) +ssh-add <(echo $VESPA_DEPLOY_TOKEN | base64 -d) set -x git clone git@github.com:vespa-engine/vespa cd vespa dist/release-vespa-rpm.sh $VESPA_RELEASE $VESPA_REF -while [[ "$VESPA_RELEASE" != "$VESPA_RPM_X86_64" ]] || [[ "$VESPA_RELEASE" != "$VESPA_RPM_AARCH64" ]] ; do - dnf clean --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-8-x86_64 --repoid=vespa metadata - VESPA_RPM_X86_64=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-8-x86_64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) - echo "RPM x86_64: $VESPA_RPM_X86_64" - dnf clean --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-8-aarch64 --repoid=vespa metadata - VESPA_RPM_AARCH64=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-8-aarch64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) - echo "RPM aarch64: $VESPA_RPM_AARCH64" - sleep 150 -done +upload_rpms() { + local ARCH=$1 + aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar . + aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar.pem . + aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar.sig . + cosign verify-blob \ + --certificate-identity https://buildkite.com/vespaai/vespa-engine-vespa \ + --certificate-oidc-issuer https://agent.buildkite.com \ + --signature rpm-repo.tar.sig \ + --certificate rpm-repo.tar.pem \ + rpm-repo.tar + tar xvf rpm-repo.tar + for rpm in rpms/*.rpm; do + screwdriver/upload-rpm-to-cloudsmith.sh $rpm + done + rm -rf rpms rpm-repo.* +} + +if [[ "$VESPA_RELEASE" != "$VESPA_RPM_X86_64" ]]; then + upload_rpms amd64 +fi + +if [[ "$VESPA_RELEASE" != "$VESPA_RPM_AARCH64" ]]; then + upload_rpms arm64 +fi diff --git a/screwdriver/settings-publish.xml b/screwdriver/settings-publish.xml index 791c19eca60..116b1de83a9 100644 --- a/screwdriver/settings-publish.xml +++ b/screwdriver/settings-publish.xml @@ -14,22 +14,15 @@ <profile> <id>ossrh-deploy-vespa</id> <activation> - <activeByDefault>false</activeByDefault> + <activeByDefault>true</activeByDefault> </activation> <properties> - <snapshotDeploymentRepository>https://oss.sonatype.org/content/repositories/snapshots</snapshotDeploymentRepository> - <releaseDeploymentRepository>https://oss.sonatype.org/service/local/staging/deploy/maven2/</releaseDeploymentRepository> - <altDeploymentRepository>ossrh::default::${snapshotDeploymentRepository}</altDeploymentRepository> - <altSnapshotDeploymentRepository>ossrh::default::${snapshotDeploymentRepository}</altSnapshotDeploymentRepository> - <altReleaseDeploymentRepository>ossrh::default::${releaseDeploymentRepository}</altReleaseDeploymentRepository> <gpg.executable>gpg</gpg.executable> <gpg.keyname>${env.GPG_KEYNAME}</gpg.keyname> - <gpg.passphrase>${env.GPG_PASSPHRASE}</gpg.passphrase> <gpg.defaultKeyring>false</gpg.defaultKeyring> - <gpg.homedir>${env.SD_SOURCE_DIR}/screwdriver/deploy</gpg.homedir> + <gpg.homedir>${env.SOURCE_DIR}/screwdriver/deploy</gpg.homedir> <gpg.publickeyring>pubring.gpg</gpg.publickeyring> <gpg.secretkeyring>secring.gpg</gpg.secretkeyring> - <nexus.staging.maven.plugin.version>1.6.12</nexus.staging.maven.plugin.version> </properties> </profile> </profiles> diff --git a/screwdriver/trig-buildkite-pipeline.sh b/screwdriver/trig-buildkite-pipeline.sh new file mode 100755 index 00000000000..c2c169efe6c --- /dev/null +++ b/screwdriver/trig-buildkite-pipeline.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $# -lt 5 ]]; then + echo "Usage: $0 <Pipeline slug> <Github branch> <Vespa version> <Vespa gitref>" + exit 1 +fi + +PIPELINE=$1 +BRANCH=$2 +VERSION=$3 +GITREF=$4 + +curl -X POST "https://api.buildkite.com/v2/organizations/vespaai/pipelines/$PIPELINE/builds" \ + -H "Content-Type: application/json" -H "Authorization: Bearer $BUILDKITE_TRIGGER_TOKEN" \ + -d "{ + \"commit\": \"HEAD\", + \"branch\": \"$BRANCH\", + \"message\": \"Vespa $VERSION release trigged from Screwdriver :vespa:\", + \"ignore_pipeline_branch_filters\": true, + \"env\": { + \"VESPA_VERSION\": \"$VERSION\", + \"VESPA_REF\": \"$GITREF\" + } + }" diff --git a/screwdriver/upload-rpm-to-cloudsmith.sh b/screwdriver/upload-rpm-to-cloudsmith.sh index 6cd50277b3d..b20bf033373 100755 --- a/screwdriver/upload-rpm-to-cloudsmith.sh +++ b/screwdriver/upload-rpm-to-cloudsmith.sh @@ -8,8 +8,8 @@ if (( $# < 1 )); then exit 1 fi -if [[ -z $CLOUDSMITH_API_CREDS ]]; then - echo "Environment CLOUDSMITH_API_CREDS not set. Exiting." +if [[ -z $CLOUDSMITH_API_TOKEN ]]; then + echo "Environment CLOUDSMITH_API_TOKEN not set. Exiting." exit 1 fi @@ -21,13 +21,13 @@ main() { FID=$(curl -sLf \ --upload-file $RPM \ - -u "$CLOUDSMITH_API_CREDS" \ + -H "X-Api-Key: $CLOUDSMITH_API_TOKEN" \ -H "Content-Sha256: $(sha256sum $RPM | cut -f1 -d' ')" \ https://upload.cloudsmith.io/vespa/open-source-rpms/$RPM | jq -re '.identifier') if [[ -n $FID ]]; then curl -sLf -X POST -H "Content-Type: application/json" \ - -u "$CLOUDSMITH_API_CREDS" \ + -H "X-Api-Key: $CLOUDSMITH_API_TOKEN" \ -d "{\"package_file\": \"$FID\", \"distribution\": \"$OS_DISTRO/$RELEASEVER\"}" \ https://api-prd.cloudsmith.io/v1/packages/vespa/open-source-rpms/upload/rpm/ fi diff --git a/screwdriver/wait-for-buildkite-build.sh b/screwdriver/wait-for-buildkite-build.sh new file mode 100755 index 00000000000..582e5f37692 --- /dev/null +++ b/screwdriver/wait-for-buildkite-build.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $# -lt 3 ]]; then + echo "Usage: $0 <Buildkite build URL> <Buildkite web URL> <Timeout sec>" + exit 1 +fi + +URL=$1 +WEB_URL=$2 +TIMEOUT=$3 + +WAIT_UNTIL=$(( $(date +%s) + $TIMEOUT )) +while [[ $(date +%s) -le $WAIT_UNTIL ]]; do + STATUS=$(curl -sSL -H "Content-Type: application/json" -H "Authorization: Bearer $BUILDKITE_TRIGGER_TOKEN" "$URL" | jq -re '.state') + + echo "Status for $URL is $STATUS" + + case $STATUS in + passed) + break + ;; + failed) + echo "Buildkite build failed. Visit $WEB_URL for information." + exit 1 + ;; + *) + sleep 5 + ;; + esac +done + +if [[ $STATUS != passed ]]; then + echo "Timed out waiting for build at $WEB_URL ." + exit 1 +fi diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 6a5d3915d6a..2e10da97680 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -65,7 +65,6 @@ vespa_define_module( src/tests/proton/attribute/document_field_populator src/tests/proton/attribute/imported_attributes_context src/tests/proton/attribute/imported_attributes_repo - src/tests/proton/bucketdb/bucketdb src/tests/proton/common src/tests/proton/common/alloc_config src/tests/proton/common/hw_info_sampler @@ -74,25 +73,8 @@ vespa_define_module( src/tests/proton/docsummary src/tests/proton/document_iterator src/tests/proton/documentdb - src/tests/proton/documentdb/buckethandler - src/tests/proton/documentdb/clusterstatehandler - src/tests/proton/documentdb/combiningfeedview - src/tests/proton/documentdb/configurer - src/tests/proton/documentdb/document_scan_iterator - src/tests/proton/documentdb/document_subdbs - src/tests/proton/documentdb/documentbucketmover - src/tests/proton/documentdb/documentdbconfig - src/tests/proton/documentdb/documentdbconfigscout - src/tests/proton/documentdb/executor_threading_service src/tests/proton/documentdb/feedhandler - src/tests/proton/documentdb/feedview src/tests/proton/documentdb/fileconfigmanager - src/tests/proton/documentdb/job_tracked_maintenance_job - src/tests/proton/documentdb/lid_space_compaction - src/tests/proton/documentdb/maintenancecontroller - src/tests/proton/documentdb/move_operation_limiter - src/tests/proton/documentdb/storeonlyfeedview - src/tests/proton/documentdb/threading_service_config src/tests/proton/documentmetastore src/tests/proton/documentmetastore/lidreusedelayer src/tests/proton/documentmetastore/lid_allocator @@ -103,7 +85,6 @@ vespa_define_module( src/tests/proton/flushengine/prepare_restart_flush_strategy src/tests/proton/flushengine/shrink_lid_space_flush_target src/tests/proton/index - src/tests/proton/index/index_writer src/tests/proton/initializer src/tests/proton/matchengine src/tests/proton/matching @@ -117,15 +98,12 @@ vespa_define_module( src/tests/proton/matching/request_context src/tests/proton/matching/same_element_builder src/tests/proton/matching/unpacking_iterators_optimizer - src/tests/proton/metrics/metrics_engine src/tests/proton/persistenceconformance src/tests/proton/persistenceengine src/tests/proton/persistenceengine/resource_usage_tracker src/tests/proton/persistenceengine/persistence_handler_map src/tests/proton/proton - src/tests/proton/proton_config_fetcher src/tests/proton/proton_configurer - src/tests/proton/proton_disk_layout src/tests/proton/reference/gid_to_lid_change_handler src/tests/proton/reference/gid_to_lid_change_listener src/tests/proton/reference/gid_to_lid_change_registrator @@ -140,9 +118,7 @@ vespa_define_module( src/tests/proton/server/disk_mem_usage_filter src/tests/proton/server/disk_mem_usage_metrics src/tests/proton/server/disk_mem_usage_sampler - src/tests/proton/server/health_adapter src/tests/proton/server/initialize_threads_calculator - src/tests/proton/server/memory_flush_config_updater src/tests/proton/server/memoryflush src/tests/proton/server/shared_threading_service src/tests/proton/summaryengine diff --git a/searchcore/src/tests/grouping/grouping_test.cpp b/searchcore/src/tests/grouping/grouping_test.cpp index e77bc4e9e22..4bc4ee7755f 100644 --- a/searchcore/src/tests/grouping/grouping_test.cpp +++ b/searchcore/src/tests/grouping/grouping_test.cpp @@ -12,6 +12,7 @@ #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/searchlib/test/mock_attribute_context.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/util/testclock.h> #include <iostream> #include <vespa/vespalib/testkit/test_kit.h> @@ -36,57 +37,64 @@ const uint32_t NUM_DOCS = 1000; struct MyWorld { MockAttributeContext attributeContext; + document::DocumentType documentType; search::AllocatedBitVector bv; - MyWorld() - : attributeContext(), - bv(NUM_DOCS+1) + MyWorld(); + ~MyWorld(); +}; + +MyWorld::~MyWorld() = default; + +MyWorld::MyWorld() + : attributeContext(), + documentType("test"), + bv(NUM_DOCS+1) +{ + bv.setInterval(0, NUM_DOCS); + // attribute context { - bv.setInterval(0, NUM_DOCS); - // attribute context - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("attr0"); - AttributeVector::DocId docid; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i, docid); // value = docid - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); + auto attr = std::make_shared<SingleInt32ExtAttribute>("attr0"); + AttributeVector::DocId docid; + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i, docid); // value = docid } - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("attr1"); - AttributeVector::DocId docid; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i * 2, docid); // value = docid * 2 - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); + assert(docid + 1 == NUM_DOCS); + attributeContext.add(attr); + } + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("attr1"); + AttributeVector::DocId docid; + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i * 2, docid); // value = docid * 2 } - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("attr2"); - AttributeVector::DocId docid; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i * 3, docid); // value = docid * 3 - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); + assert(docid + 1 == NUM_DOCS); + attributeContext.add(attr); + } + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("attr2"); + AttributeVector::DocId docid; + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i * 3, docid); // value = docid * 3 } - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("attr3"); - AttributeVector::DocId docid; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i * 4, docid); // value = docid * 4 - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); + assert(docid + 1 == NUM_DOCS); + attributeContext.add(attr); + } + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("attr3"); + AttributeVector::DocId docid; + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i * 4, docid); // value = docid * 4 } - + assert(docid + 1 == NUM_DOCS); + attributeContext.add(attr); } -}; + +} //----------------------------------------------------------------------------- @@ -291,7 +299,7 @@ TEST_F("testGroupingSession", DoomFixture()) { SessionId id("foo"); // Test initialization phase - GroupingSession session(id, initContext, world.attributeContext); + GroupingSession session(id, initContext, world.attributeContext, &world.documentType); CheckAttributeReferences attrCheck2; EXPECT_EQUAL(2u, initContext.getGroupingList().size()); for (const auto & g : initContext.getGroupingList()) { @@ -356,7 +364,7 @@ TEST_F("testEmptySessionId", DoomFixture()) { SessionId id; // Test initialization phase - GroupingSession session(id, initContext, world.attributeContext); + GroupingSession session(id, initContext, world.attributeContext, &world.documentType); RankedHit hit; hit._docId = 0; GroupingManager &manager(session.getGroupingManager()); @@ -390,9 +398,9 @@ TEST_F("testSessionManager", DoomFixture()) { SessionId id1("foo"); SessionId id2("bar"); SessionId id3("baz"); - auto s1 = std::make_unique<GroupingSession>(id1, initContext, world.attributeContext); - auto s2 = std::make_unique<GroupingSession>(id2, initContext, world.attributeContext); - auto s3 = std::make_unique<GroupingSession>(id3, initContext, world.attributeContext); + auto s1 = std::make_unique<GroupingSession>(id1, initContext, world.attributeContext, &world.documentType); + auto s2 = std::make_unique<GroupingSession>(id2, initContext, world.attributeContext, &world.documentType); + auto s3 = std::make_unique<GroupingSession>(id3, initContext, world.attributeContext, &world.documentType); ASSERT_EQUAL(f1.timeOfDoom, s1->getTimeOfDoom()); mgr.insert(std::move(s1)); @@ -442,13 +450,12 @@ TEST_F("test grouping fork/join", DoomFixture()) { auto g1 = std::make_shared<Grouping>(request); GroupingContext context(world.bv, f1.clock.nowRef(), f1.timeOfDoom); context.addGrouping(g1); - GroupingSession session(SessionId(), context, world.attributeContext); + GroupingSession session(SessionId(), context, world.attributeContext, &world.documentType); session.prepareThreadContextCreation(4); - - GroupingContext::UP ctx0 = session.createThreadContext(0, world.attributeContext); - GroupingContext::UP ctx1 = session.createThreadContext(1, world.attributeContext); - GroupingContext::UP ctx2 = session.createThreadContext(2, world.attributeContext); - GroupingContext::UP ctx3 = session.createThreadContext(3, world.attributeContext); + GroupingContext::UP ctx0 = session.createThreadContext(0, world.attributeContext, &world.documentType); + GroupingContext::UP ctx1 = session.createThreadContext(1, world.attributeContext, &world.documentType); + GroupingContext::UP ctx2 = session.createThreadContext(2, world.attributeContext, &world.documentType); + GroupingContext::UP ctx3 = session.createThreadContext(3, world.attributeContext, &world.documentType); doGrouping(*ctx0, 12, 30.0, 11, 20.0, 10, 10.0); doGrouping(*ctx1, 22, 150.0, 21, 40.0, 20, 25.0); doGrouping(*ctx2, 32, 100.0, 31, 15.0, 30, 5.0); @@ -483,8 +490,8 @@ TEST_F("test session timeout", DoomFixture()) { GroupingContext initContext1(world.bv, f1.clock.nowRef(), steady_time(duration(10))); GroupingContext initContext2(world.bv, f1.clock.nowRef(), steady_time(duration(20))); - auto s1 = std::make_unique<GroupingSession>(id1, initContext1, world.attributeContext); - auto s2 = std::make_unique<GroupingSession>(id2, initContext2, world.attributeContext); + auto s1 = std::make_unique<GroupingSession>(id1, initContext1, world.attributeContext, &world.documentType); + auto s2 = std::make_unique<GroupingSession>(id2, initContext2, world.attributeContext, &world.documentType); mgr.insert(std::move(s1)); mgr.insert(std::move(s2)); mgr.pruneTimedOutSessions(steady_time(5ns)); diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index df27c4d7cc2..5e270bdae99 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -32,13 +32,14 @@ #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/hw_info.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/config-attributes.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("attribute_manager_test"); diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp index a20e08d5be2..cdb0bcdb66a 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -3,8 +3,9 @@ #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchlib/attribute/address_space_components.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("attribute_usage_filter_test"); diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp index 6f84b793608..f93853d2d1d 100644 --- a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp @@ -15,6 +15,7 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchcore/proton/attribute/document_field_extractor.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using document::Field; using document::DataType; diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/.gitignore b/searchcore/src/tests/proton/bucketdb/bucketdb/.gitignore deleted file mode 100644 index 3512e4268a1..00000000000 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_bucketdb_test_app diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt b/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt deleted file mode 100644 index 70ed1efc7f7..00000000000 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_bucketdb_test_app TEST - SOURCES - bucketdb_test.cpp - DEPENDS - searchcore_bucketdb -) -vespa_add_test(NAME searchcore_bucketdb_test_app COMMAND searchcore_bucketdb_test_app) diff --git a/searchcore/src/tests/proton/common/CMakeLists.txt b/searchcore/src/tests/proton/common/CMakeLists.txt index 7eec733214b..315ad8b5ce6 100644 --- a/searchcore/src/tests/proton/common/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/CMakeLists.txt @@ -2,17 +2,22 @@ vespa_add_executable(searchcore_proton_common_vespa_test_app TEST SOURCES vespa_testrunner.cpp - selectpruner_test.cpp - cachedselect_test.cpp - pendinglidtracker_test.cpp attribute_updater_test.cpp - state_reporter_utils_test.cpp + bucketdb_test.cpp + cachedselect_test.cpp document_type_inspector_test.cpp - feedoperation_test.cpp documentdb_job_trackers_test.cpp + feedoperation_test.cpp + index_writer_test.cpp job_load_sampler_test.cpp job_tracked_flush_test.cpp + metrics_engine_test.cpp + pendinglidtracker_test.cpp + proton_config_fetcher_test.cpp + proton_disk_layout_test.cpp + state_reporter_utils_test.cpp statusreport_test.cpp + selectpruner_test.cpp DEPENDS searchcore_proton_metrics searchcore_feedoperation diff --git a/searchcore/src/tests/proton/common/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater_test.cpp index 432386af0e6..b082e1f9907 100644 --- a/searchcore/src/tests/proton/common/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater_test.cpp @@ -39,6 +39,7 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace document; using document::config_builder::Array; diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp b/searchcore/src/tests/proton/common/bucketdb_test.cpp index 213d9c38290..6d9c355f5f5 100644 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp +++ b/searchcore/src/tests/proton/common/bucketdb_test.cpp @@ -7,9 +7,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/testkit/test_kit.h> - -#include <vespa/log/log.h> -LOG_SETUP("bucketdb_test"); +#include <vespa/vespalib/testkit/test_master.hpp> using namespace document; using namespace proton; @@ -20,6 +18,8 @@ using storage::spi::BucketInfo; using storage::spi::Timestamp; using vespalib::Slime; +namespace { + constexpr uint32_t MIN_NUM_BITS = 8u; const GlobalId GID_1("111111111111"); const BucketId BUCKET_1(MIN_NUM_BITS, GID_1.convertToBucketId().getRawId()); @@ -32,7 +32,6 @@ constexpr uint32_t DOCSIZE_2(10000u); using RS = BucketInfo::ReadyState; using SDT = SubDbType; -namespace { constexpr uint32_t bucket_bits = 16; @@ -48,7 +47,6 @@ GlobalId make_gid(uint32_t n, uint32_t i) return id.getGlobalId(); } -} void assertDocCount(uint32_t ready, @@ -140,6 +138,8 @@ struct Fixture } }; +} + TEST_F("require that bucket db tracks doc counts per sub db type", Fixture) { TEST_DO(assertDocCount(0, 0, 0, f.get())); @@ -404,5 +404,3 @@ TEST_F("test BucketDB active document tracking", Fixture) { f._db.unloadBucket(make_bucket_id(5), bs); EXPECT_EQUAL(0u, f._db.getNumActiveDocs()); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/feedoperation_test.cpp b/searchcore/src/tests/proton/common/feedoperation_test.cpp index 48893aa7da3..cae51211dfe 100644 --- a/searchcore/src/tests/proton/common/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/common/feedoperation_test.cpp @@ -25,6 +25,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using document::BucketId; using document::DataType; diff --git a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp index ad74039a5ee..88bc22932d2 100644 --- a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp +++ b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp @@ -49,7 +49,7 @@ TEST_F("Test that hw_info_sampler uses override info", Fixture) Config samplerCfg(0, 75.0, 100.0, sampleLen, sharedDisk, 0, 0); HwInfoSampler sampler(test_dir, samplerCfg); EXPECT_EQUAL(75.0, sampler.diskWriteSpeed()); - EXPECT_NOT_EQUAL(0, time_point_to_long(sampler.sampleTime())); + EXPECT_NOT_EQUAL(0l, time_point_to_long(sampler.sampleTime())); EXPECT_TRUE(sampler.hwInfo().disk().slow()); } @@ -71,7 +71,7 @@ TEST_F("Test that hw_info_sampler can sample disk write speed", Fixture) Config samplerCfg(0, 0.0, 100.0, sampleLen, sharedDisk, 0, 0); HwInfoSampler sampler(test_dir, samplerCfg); ASSERT_NOT_EQUAL(0.0, sampler.diskWriteSpeed()); - ASSERT_NOT_EQUAL(0, time_point_to_long(sampler.sampleTime())); + ASSERT_NOT_EQUAL(0l, time_point_to_long(sampler.sampleTime())); HwInfoSampler sampler2(test_dir, samplerCfg); EXPECT_APPROX(sampler.diskWriteSpeed(), sampler2.diskWriteSpeed(), 0.1); EXPECT_EQUAL(time_point_to_long(sampler.sampleTime()), diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/common/index_writer_test.cpp index c52eb940817..9f39d6ec66d 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/common/index_writer_test.cpp @@ -7,9 +7,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/log/log.h> -LOG_SETUP("index_writer_test"); - using namespace proton; using namespace search; using namespace search::index; @@ -19,6 +16,7 @@ using vespalib::IDestructorCallback; using document::Document; +namespace { std::string toString(const std::vector<SerialNum> &vec) { @@ -106,6 +104,8 @@ struct Fixture } }; +} + TEST_F("require that index writer ignores old operations", Fixture) { f.mim.flushed = 10; @@ -135,8 +135,3 @@ TEST_F("require that old compactLidSpace is not forwarded to index manager", Fix EXPECT_EQUAL(0u, f.mim.wantedLidLimit); EXPECT_EQUAL(0u, f.mim.compactSerial); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp b/searchcore/src/tests/proton/common/metrics_engine_test.cpp index dd622a4cac1..f4fc4c52ab1 100644 --- a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp +++ b/searchcore/src/tests/proton/common/metrics_engine_test.cpp @@ -5,11 +5,10 @@ #include <vespa/searchcore/proton/metrics/metrics_engine.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/log/log.h> -LOG_SETUP("metrics_engine_test"); - using namespace proton; +namespace { + struct DummyMetricSet : public metrics::MetricSet { DummyMetricSet(const vespalib::string &name) : metrics::MetricSet(name, {}, "", nullptr) {} }; @@ -43,6 +42,8 @@ struct AttributeMetricsFixture { } }; +} + TEST_F("require that attribute metrics can be added", AttributeMetricsFixture) { TEST_DO(f.assertRegisteredMetrics(0)); @@ -74,5 +75,3 @@ TEST_F("require that all attribute metrics can be cleaned", AttributeMetricsFixt TEST_DO(f.assertMetricsNotExists("foo")); TEST_DO(f.assertMetricsNotExists("bar")); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp b/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp index f8d0d218670..45b80d0c32e 100644 --- a/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp +++ b/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp @@ -1,7 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/common/pendinglidtracker.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace proton; diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/common/proton_config_fetcher_test.cpp index 1dfc6c481b6..50ed24a0add 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/common/proton_config_fetcher_test.cpp @@ -24,10 +24,11 @@ #include <vespa/fileacquirer/config-filedistributorrpc.h> #include <vespa/vespalib/util/hw_info.h> #include <vespa/vespalib/util/varholder.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config.h> #include <map> #include <thread> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace config; using namespace proton; @@ -48,6 +49,7 @@ using search::GrowStrategy; using vespalib::datastore::CompactionStrategy; using vespalib::HwInfo; +namespace { constexpr int proton_rpc_port = 9010; // Not used for listening in this test struct DoctypeFixture { @@ -228,6 +230,8 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer ProtonConfigOwner::~ProtonConfigOwner() = default; +} + TEST_F("require that bootstrap config manager creats correct key set", BootstrapConfigManager("foo")) { const ConfigKeySet set(f1.createConfigKeySet()); ASSERT_EQUAL(4u, set.size()); @@ -498,5 +502,3 @@ TEST("test HwInfo equality") { EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 2ul, 1ul)); EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(2, false,false), 1ul, 1ul)); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/common/proton_disk_layout_test.cpp index 881c547df1e..5f3a08246c4 100644 --- a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp +++ b/searchcore/src/tests/proton/common/proton_disk_layout_test.cpp @@ -8,10 +8,11 @@ #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/stringfmt.h> #include <filesystem> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> +#include <vespa/vespalib/test/insertion_operators.h> using search::index::DummyFileHeaderContext; using search::transactionlog::client::TransLogClient; @@ -20,10 +21,11 @@ using proton::DocTypeName; using proton::ProtonDiskLayout; using proton::Transport; -static constexpr unsigned int tlsPort = proton::test::port_numbers::proton_disk_layout_tls_port; +namespace { +constexpr unsigned int tlsPort = proton::test::port_numbers::proton_disk_layout_tls_port; -static const vespalib::string baseDir("testdb"); -static const vespalib::string documentsDir(baseDir + "/documents"); +const vespalib::string baseDir("testdb"); +const vespalib::string documentsDir(baseDir + "/documents"); struct FixtureBase { @@ -114,6 +116,8 @@ struct Fixture : public FixtureBase, public DiskLayoutFixture } }; +} + TEST_F("require that empty config is ok", Fixture) { TEST_DO(f.assertDirs({})); TEST_DO(f.assertDomains({})); @@ -180,5 +184,3 @@ TEST_F("require that live document db dir remove works", Fixture) TEST_DO(f.assertDirs({})); TEST_DO(f.assertDomains({})); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/statusreport_test.cpp b/searchcore/src/tests/proton/common/statusreport_test.cpp index 052eb795529..8465f0ec56d 100644 --- a/searchcore/src/tests/proton/common/statusreport_test.cpp +++ b/searchcore/src/tests/proton/common/statusreport_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/common/statusreport.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> namespace proton { diff --git a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp index 361e37278da..28b26428a5e 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp @@ -65,13 +65,14 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/encoding/base64.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/config-summary.h> #include <filesystem> #include <regex> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("docsummary_test"); @@ -590,7 +591,7 @@ TEST("requireThatSummariesTimeout") const auto & root = rep->root(); const auto & field = root["errors"]; EXPECT_TRUE(field.valid()); - EXPECT_EQUAL(field[0]["type"].asString(), "timeout"); + EXPECT_EQUAL(field[0]["type"].asString(), vespalib::Memory("timeout")); auto bufstring = field[0]["message"].asString(); EXPECT_TRUE(std::regex_search(bufstring.data, bufstring.data + bufstring.size, std::regex("Timed out 1 summaries with -[0-9]+us left."))); } diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index 85883324080..b2920ae5202 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -18,8 +18,9 @@ #include <vespa/persistence/spi/result.h> #include <vespa/persistence/spi/test.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <unordered_set> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("document_iterator_test"); diff --git a/searchcore/src/tests/proton/documentdb/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/CMakeLists.txt index b4d53e92ebf..4bf42c1701b 100644 --- a/searchcore/src/tests/proton/documentdb/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/CMakeLists.txt @@ -20,5 +20,45 @@ vespa_add_executable(searchcore_documentdb_test_app TEST searchcore_grouping searchcore_proton_metrics ) -vespa_add_test(NAME searchcore_documentdb_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/documentdb_test.sh - DEPENDS searchcore_documentdb_test_app) +vespa_add_test(NAME searchcore_documentdb_test_app COMMAND searchcore_documentdb_test_app) + +vespa_add_executable(searchcore_proton_documentdb_vespatest_test_app TEST + SOURCES + vespatest_runner.cpp + buckethandler_test.cpp + combiningfeedview_test.cpp + configurer_test.cpp + document_scan_iterator_test.cpp + document_subdbs_test.cpp + documentdbconfig_test.cpp + documentdbconfigscout_test.cpp + feedview_test.cpp + job_tracked_maintenance_job_test.cpp + maintenancecontroller_test.cpp + storeonlyfeedview_test.cpp + DEPENDS + searchcore_server + searchcore_feedoperation + searchcore_test +) +vespa_add_test(NAME searchcore_proton_documentdb_vespatest_test_app COMMAND searchcore_proton_documentdb_vespatest_test_app) + +vespa_add_executable(searchcore_proton_documentdb_gtest_test_app TEST + SOURCES + gtest_runner.cpp + clusterstatehandler_test.cpp + bucketmover_common.cpp + documentbucketmover_test.cpp + documentmover_test.cpp + executor_threading_service_test.cpp + lid_space_common.cpp + lid_space_compaction_test.cpp + lid_space_handler_test.cpp + lid_space_jobtest.cpp + DEPENDS + searchcore_server + searchcore_feedoperation + searchcore_test + GTest::GTest +) +vespa_add_test(NAME searchcore_proton_documentdb_gtest_test_app COMMAND searchcore_proton_documentdb_gtest_test_app) diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/.gitignore b/searchcore/src/tests/proton/documentdb/buckethandler/.gitignore deleted file mode 100644 index c159971ebc7..00000000000 --- a/searchcore/src/tests/proton/documentdb/buckethandler/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_buckethandler_test_app diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt deleted file mode 100644 index e67143c1b5d..00000000000 --- a/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_buckethandler_test_app TEST - SOURCES - buckethandler_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - searchcore_grouping -) -vespa_add_test(NAME searchcore_buckethandler_test_app COMMAND searchcore_buckethandler_test_app) diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp b/searchcore/src/tests/proton/documentdb/buckethandler_test.cpp index e09a0d2ea2b..4cf6b49d090 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/buckethandler_test.cpp @@ -9,9 +9,6 @@ #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/log/log.h> -LOG_SETUP("buckethandler_test"); - using namespace proton; using document::BucketId; using document::GlobalId; @@ -22,6 +19,7 @@ using storage::spi::test::makeSpiBucket; using vespalib::ThreadStackExecutor; using proton::test::BucketStateCalculator; +namespace { const GlobalId GID_1("111111111111"); const BucketId BUCKET_1(8, GID_1.convertToBucketId().getRawId()); const Timestamp TIME_1(1u); @@ -95,41 +93,8 @@ struct Fixture test::BucketIdListResultHandler _bucketList; test::BucketInfoResultHandler _bucketInfo; std::shared_ptr<test::GenericResultHandler> _genResult; - Fixture() - : _builder(), - _bucketDB(std::make_shared<bucketdb::BucketDBOwner>()), - _ready(_bucketDB, SubDbType::READY), - _removed(_bucketDB, SubDbType::REMOVED), - _notReady(_bucketDB, SubDbType::NOTREADY), - _exec(1), - _handler(_exec), - _changedHandler(), - _calc(new BucketStateCalculator()), - _bucketList(), _bucketInfo(), - _genResult(std::make_shared<test::GenericResultHandler>()) - { - // bucket 2 & 3 & 4 & 7 in ready - _ready.insertDocs(_builder.createDocs(2, 1, 3). // 2 docs - createDocs(3, 3, 6). // 3 docs - createDocs(4, 6, 10). // 4 docs - createDocs(7, 10, 11). // 1 doc - getDocs()); - // bucket 2 in removed - _removed.insertDocs(_builder.clearDocs(). - createDocs(2, 16, 20). // 4 docs - getDocs()); - // bucket 4 in not ready - _notReady.insertDocs(_builder.clearDocs(). - createDocs(4, 22, 24). // 2 docs - getDocs()); - _handler.setReadyBucketHandler(_ready._metaStore); - _handler.addBucketStateChangedHandler(&_changedHandler); - _handler.notifyClusterStateChanged(_calc); - } - ~Fixture() - { - _handler.removeBucketStateChangedHandler(&_changedHandler); - } + Fixture() __attribute__((noinline)); + ~Fixture() __attribute__((noinline)); void sync() { _exec.sync(); } void handleGetBucketInfo(const BucketId &bucket) { _handler.handleGetBucketInfo(makeSpiBucket(bucket), _bucketInfo); @@ -147,6 +112,44 @@ struct Fixture } }; +Fixture::Fixture() + : _builder(), + _bucketDB(std::make_shared<bucketdb::BucketDBOwner>()), + _ready(_bucketDB, SubDbType::READY), + _removed(_bucketDB, SubDbType::REMOVED), + _notReady(_bucketDB, SubDbType::NOTREADY), + _exec(1), + _handler(_exec), + _changedHandler(), + _calc(std::make_shared<BucketStateCalculator>()), + _bucketList(), _bucketInfo(), + _genResult(std::make_shared<test::GenericResultHandler>()) +{ + // bucket 2 & 3 & 4 & 7 in ready + _ready.insertDocs(_builder.createDocs(2, 1, 3). // 2 docs + createDocs(3, 3, 6). // 3 docs + createDocs(4, 6, 10). // 4 docs + createDocs(7, 10, 11). // 1 doc + getDocs()); + // bucket 2 in removed + _removed.insertDocs(_builder.clearDocs(). + createDocs(2, 16, 20). // 4 docs + getDocs()); + // bucket 4 in not ready + _notReady.insertDocs(_builder.clearDocs(). + createDocs(4, 22, 24). // 2 docs + getDocs()); + _handler.setReadyBucketHandler(_ready._metaStore); + _handler.addBucketStateChangedHandler(&_changedHandler); + _handler.notifyClusterStateChanged(_calc); +} +Fixture::~Fixture() +{ + _handler.removeBucketStateChangedHandler(&_changedHandler); +} + +} + TEST_F("require that handleListBuckets() returns buckets from all sub dbs", Fixture) { @@ -291,9 +294,3 @@ TEST_F("node going from maintenance to down state deactivates all buckets", Fixt f.handleGetBucketInfo(f._ready.bucket(2)); EXPECT_FALSE(f._bucketInfo.getInfo().isActive()); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} - diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.cpp b/searchcore/src/tests/proton/documentdb/bucketmover_common.cpp index 5c4cfb393a4..fa366d3cc3e 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.cpp +++ b/searchcore/src/tests/proton/documentdb/bucketmover_common.cpp @@ -3,7 +3,7 @@ #include "bucketmover_common.h" #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> -#include <vespa/vespalib/testkit/test_macros.h> +#include <vespa/vespalib/gtest/gtest.h> using vespalib::IDestructorCallback; @@ -70,7 +70,7 @@ MySubDb::insertDocs(const UserDocuments &docs_) { _metaStore.put(testDoc.getGid(), testDoc.getBucket(), testDoc.getTimestamp(), testDoc.getDocSize(), testDoc.getLid(), 0u); _realRetriever->_docs.push_back(testDoc.getDoc()); - ASSERT_EQUAL(testDoc.getLid() + 1, _realRetriever->_docs.size()); + ASSERT_EQ(testDoc.getLid() + 1, _realRetriever->_docs.size()); } } _docs.merge(docs_); @@ -83,16 +83,22 @@ MySubDb::remove(uint32_t subDbId, uint32_t lid) { return _metaStore.remove(lid, 0u); } +template<typename A, typename B> +bool expect_eq(const A & a, const B & b) { + EXPECT_EQ(a, b); + return (a == b); +} + bool assertEqual(const document::BucketId &bucket, const proton::test::Document &doc, uint32_t sourceSubDbId, uint32_t targetSubDbId, const MoveOperation &op) { - if (!EXPECT_EQUAL(bucket, op.getBucketId())) return false; - if (!EXPECT_EQUAL(doc.getTimestamp(), op.getTimestamp())) return false; - if (!EXPECT_EQUAL(doc.getDocId(), op.getDocument()->getId())) return false; - if (!EXPECT_EQUAL(doc.getLid(), op.getSourceDbdId().getLid())) return false; - if (!EXPECT_EQUAL(sourceSubDbId, op.getSourceDbdId().getSubDbId())) return false; - if (!EXPECT_EQUAL(0u, op.getTargetDbdId().getLid())) return false; - if (!EXPECT_EQUAL(targetSubDbId, op.getTargetDbdId().getSubDbId())) return false; + if (!expect_eq(bucket, op.getBucketId())) return false; + if (!expect_eq(doc.getTimestamp(), op.getTimestamp())) return false; + if (!expect_eq(doc.getDocId(), op.getDocument()->getId())) return false; + if (!expect_eq(doc.getLid(), op.getSourceDbdId().getLid())) return false; + if (!expect_eq(sourceSubDbId, op.getSourceDbdId().getSubDbId())) return false; + if (!expect_eq(0u, op.getTargetDbdId().getLid())) return false; + if (!expect_eq(targetSubDbId, op.getTargetDbdId().getSubDbId())) return false; return true; } diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.h b/searchcore/src/tests/proton/documentdb/bucketmover_common.h index e21f084ff6e..e21f084ff6e 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/bucketmover_common.h +++ b/searchcore/src/tests/proton/documentdb/bucketmover_common.h diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/.gitignore b/searchcore/src/tests/proton/documentdb/clusterstatehandler/.gitignore deleted file mode 100644 index bc38893db32..00000000000 --- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_clusterstatehandler_test_app diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt deleted file mode 100644 index b89d4d9301f..00000000000 --- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_clusterstatehandler_test_app TEST - SOURCES - clusterstatehandler_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_pcommon - searchcore_grouping - GTest::GTest -) -vespa_add_test(NAME searchcore_clusterstatehandler_test_app COMMAND searchcore_clusterstatehandler_test_app) diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp b/searchcore/src/tests/proton/documentdb/clusterstatehandler_test.cpp index 539ee9ce3aa..d9d7bd279fc 100644 --- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler_test.cpp @@ -8,9 +8,6 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/log/log.h> -LOG_SETUP("cluster_state_handler_test"); - using namespace proton; using document::BucketId; using storage::lib::Distribution; @@ -18,6 +15,7 @@ using storage::spi::BucketIdListResult; using storage::spi::ClusterState; using storage::spi::Result; +namespace { struct MyClusterStateChangedHandler : public IClusterStateChangedHandler { std::shared_ptr<IBucketStateCalculator> _calc; void @@ -65,6 +63,8 @@ struct ClusterStateHandlerTest : testing::Test { } }; +} + TEST_F(ClusterStateHandlerTest, cluster_state_change_is_notified) { const auto& calc = set_cluster_state(basic_state); @@ -114,6 +114,3 @@ TEST_F(ClusterStateHandlerTest, modified_buckets_are_returned) EXPECT_EQ(1u, _bucketListHandler.getList().size()); EXPECT_EQ(bucket3, _bucketListHandler.getList()[0]); } - -GTEST_MAIN_RUN_ALL_TESTS() - diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/.gitignore b/searchcore/src/tests/proton/documentdb/combiningfeedview/.gitignore deleted file mode 100644 index 3302e827c3e..00000000000 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_combiningfeedview_test_app diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt deleted file mode 100644 index 223befab60d..00000000000 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_combiningfeedview_test_app TEST - SOURCES - combiningfeedview_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - searchcore_grouping - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_combiningfeedview_test_app COMMAND searchcore_combiningfeedview_test_app) diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview_test.cpp index c713108dcc0..4ad0f993d1b 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview_test.cpp @@ -10,9 +10,6 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/log/log.h> -LOG_SETUP("combiningfeedview_test"); - using document::DocumentTypeRepo; using document::DocumentUpdate; using document::test::makeBucketSpace; @@ -23,6 +20,7 @@ using namespace proton; using FeedViewVector = std::vector<IFeedView::SP>; +namespace { struct MyFeedView : public test::DummyFeedView { using SP = std::shared_ptr<MyFeedView>; @@ -128,18 +126,8 @@ struct Fixture MySubDb _notReady; test::BucketStateCalculator::SP _calc; CombiningFeedView _view; - Fixture() : - _builder(), - _bucketDB(std::make_shared<bucketdb::BucketDBOwner>()), - _ready(_builder.getRepo(), _bucketDB, SubDbType::READY), - _removed(_builder.getRepo(), _bucketDB, SubDbType::REMOVED), - _notReady(_builder.getRepo(), _bucketDB, SubDbType::NOTREADY), - _calc(new test::BucketStateCalculator()), - _view(getVector(_ready, _removed, _notReady), makeBucketSpace(), _calc) - { - _builder.createDoc(1, 1); - _builder.createDoc(2, 2); - } + Fixture() __attribute__((noinline)); + ~Fixture() __attribute__((noinline)); const test::UserDocuments &userDocs() const { return _builder.getDocs(); } const test::BucketDocuments &userDocs(uint32_t userId) const { return userDocs().getUserDocs(userId); } PutOperation put(uint32_t userId) { @@ -163,6 +151,22 @@ struct Fixture } }; +Fixture::Fixture() + : _builder(), + _bucketDB(std::make_shared<bucketdb::BucketDBOwner>()), + _ready(_builder.getRepo(), _bucketDB, SubDbType::READY), + _removed(_builder.getRepo(), _bucketDB, SubDbType::REMOVED), + _notReady(_builder.getRepo(), _bucketDB, SubDbType::NOTREADY), + _calc(new test::BucketStateCalculator()), + _view(getVector(_ready, _removed, _notReady), makeBucketSpace(), _calc) +{ + _builder.createDoc(1, 1); + _builder.createDoc(2, 2); +} +Fixture::~Fixture() = default; + +} + TEST_F("require that preparePut() sends to ready view", Fixture) { @@ -425,9 +429,3 @@ TEST_F("require that compactLidSpace() is sent to correct feed view", Fixture) EXPECT_EQUAL(99u, f._removed._view->_wantedLidLimit); EXPECT_EQUAL(0u, f._notReady._view->_wantedLidLimit); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} - diff --git a/searchcore/src/tests/proton/documentdb/configurer/.gitignore b/searchcore/src/tests/proton/documentdb/configurer/.gitignore deleted file mode 100644 index 3714f1b204d..00000000000 --- a/searchcore/src/tests/proton/documentdb/configurer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_configurer_test_app diff --git a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt deleted file mode 100644 index 964677784e9..00000000000 --- a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_configurer_test_app TEST - SOURCES - configurer_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_reprocessing - searchcore_index - searchcore_docsummary - searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_documentmetastore - searchcore_bucketdb - searchcore_flushengine - searchcore_pcommon - searchcore_grouping - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_configurer_test_app COMMAND searchcore_configurer_test_app) diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer_test.cpp index 7c87e16bb35..d49dbcdec2e 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer_test.cpp @@ -1,7 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> - #include <vespa/config-summary.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> @@ -42,6 +40,8 @@ #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <filesystem> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace config; using namespace document; @@ -69,6 +69,7 @@ using Configurer = SearchableDocSubDBConfigurer; using ConfigurerUP = std::unique_ptr<SearchableDocSubDBConfigurer>; using DocumenttypesConfigSP = proton::DocumentDBConfig::DocumenttypesConfigSP; +namespace { const vespalib::string BASE_DIR("baseDir"); const vespalib::string DOC_TYPE("invalid"); @@ -146,7 +147,7 @@ ViewSet::~ViewSet() = default; struct EmptyConstantValueFactory : public vespalib::eval::ConstantValueFactory { vespalib::eval::ConstantValue::UP create(const vespalib::string &, const vespalib::string &) const override { - return vespalib::eval::ConstantValue::UP(nullptr); + return {}; } }; @@ -178,13 +179,13 @@ struct Fixture const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, IDocumentDBReferenceResolver& resolver, - SerialNum serial_num); + SerialNum serial_num) const; IReprocessingInitializer::UP reconfigure(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, IDocumentDBReferenceResolver& resolver, uint32_t docid_limit, - SerialNum serial_num); + SerialNum serial_num) const; }; Fixture::Fixture() @@ -204,7 +205,9 @@ Fixture::Fixture() _configurer = std::make_unique<Configurer>(_views._summaryMgr, _views.searchView, _views.feedView, _queryLimiter, _constantValueFactory, _clock.nowRef(), "test", 0); } -Fixture::~Fixture() = default; +Fixture::~Fixture() { + std::filesystem::remove_all(std::filesystem::path(BASE_DIR)); +} void Fixture::initViewSet(ViewSet &views) @@ -255,7 +258,7 @@ Fixture::reconfigure(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, IDocumentDBReferenceResolver& resolver, - SerialNum serial_num) + SerialNum serial_num) const { EXPECT_FALSE(reconfig_params.shouldAttributeManagerChange()); uint32_t docid_limit = 1; @@ -271,7 +274,7 @@ Fixture::reconfigure(const DocumentDBConfig& new_config_snapshot, const ReconfigParams& reconfig_params, IDocumentDBReferenceResolver& resolver, uint32_t docid_limit, - SerialNum serial_num) + SerialNum serial_num) const { AttributeCollectionSpecFactory attr_spec_factory(AllocStrategy(), false); auto prepared_reconfig = _configurer->prepare_reconfig(new_config_snapshot, attr_spec_factory, reconfig_params, docid_limit, serial_num); @@ -279,7 +282,7 @@ Fixture::reconfigure(const DocumentDBConfig& new_config_snapshot, return _configurer->reconfigure(new_config_snapshot, old_config_snapshot, reconfig_params, resolver, *prepared_reconfig, serial_num); } -using MySummaryAdapter = test::MockSummaryAdapter; +using MySummaryAdapter = proton::test::MockSummaryAdapter; struct MyFastAccessFeedView { @@ -293,56 +296,51 @@ struct MyFastAccessFeedView std::shared_ptr<PendingLidTrackerBase> _pendingLidsForCommit; VarHolder<FastAccessFeedView::SP> _feedView; - explicit MyFastAccessFeedView(IThreadingService &writeService) - : _fileHeaderContext(), - _docIdLimit(0), - _writeService(writeService), - _hwInfo(), - _dmsc(), - _gidToLidChangeHandler(make_shared<DummyGidToLidChangeHandler>()), - _pendingLidsForCommit(std::make_shared<PendingLidTracker>()), - _feedView() - { - init(); - } - + explicit MyFastAccessFeedView(IThreadingService &writeService) __attribute__((noinline)); ~MyFastAccessFeedView(); - void init() { - MySummaryAdapter::SP summaryAdapter = std::make_shared<MySummaryAdapter>(); - Schema::SP schema = std::make_shared<Schema>(); - _dmsc = make_shared<DocumentMetaStoreContext>(std::make_shared<bucketdb::BucketDBOwner>()); - std::shared_ptr<const DocumentTypeRepo> repo = createRepo(); - StoreOnlyFeedView::Context storeOnlyCtx(summaryAdapter, schema, _dmsc, repo, - _pendingLidsForCommit, *_gidToLidChangeHandler, _writeService); - StoreOnlyFeedView::PersistentParams params(1, 1, DocTypeName(DOC_TYPE), 0, SubDbType::NOTREADY); - auto mgr = make_shared<AttributeManager>(BASE_DIR, "test.subdb", TuneFileAttributes(), - _fileHeaderContext, std::make_shared<search::attribute::Interlock>(), - _writeService.field_writer(), _writeService.shared(), _hwInfo); - auto writer = std::make_shared<AttributeWriter>(mgr); - FastAccessFeedView::Context fastUpdateCtx(writer, _docIdLimit); - _feedView.set(std::make_shared<FastAccessFeedView>(std::move(storeOnlyCtx), params, fastUpdateCtx)); - } + void init() __attribute__((noinline)); }; +MyFastAccessFeedView::MyFastAccessFeedView(IThreadingService &writeService) + : _fileHeaderContext(), + _docIdLimit(0), + _writeService(writeService), + _hwInfo(), + _dmsc(), + _gidToLidChangeHandler(make_shared<DummyGidToLidChangeHandler>()), + _pendingLidsForCommit(std::make_shared<PendingLidTracker>()), + _feedView() +{ + init(); +} + MyFastAccessFeedView::~MyFastAccessFeedView() = default; +void +MyFastAccessFeedView::init() { + MySummaryAdapter::SP summaryAdapter = std::make_shared<MySummaryAdapter>(); + Schema::SP schema = std::make_shared<Schema>(); + _dmsc = make_shared<DocumentMetaStoreContext>(std::make_shared<bucketdb::BucketDBOwner>()); + std::shared_ptr<const DocumentTypeRepo> repo = createRepo(); + StoreOnlyFeedView::Context storeOnlyCtx(summaryAdapter, schema, _dmsc, repo, + _pendingLidsForCommit, *_gidToLidChangeHandler, _writeService); + StoreOnlyFeedView::PersistentParams params(1, 1, DocTypeName(DOC_TYPE), 0, SubDbType::NOTREADY); + auto mgr = make_shared<AttributeManager>(BASE_DIR, "test.subdb", TuneFileAttributes(), + _fileHeaderContext, std::make_shared<search::attribute::Interlock>(), + _writeService.field_writer(), _writeService.shared(), _hwInfo); + auto writer = std::make_shared<AttributeWriter>(mgr); + FastAccessFeedView::Context fastUpdateCtx(writer, _docIdLimit); + _feedView.set(std::make_shared<FastAccessFeedView>(std::move(storeOnlyCtx), params, fastUpdateCtx)); +} + struct FastAccessFixture { TransportAndExecutorService _service; MyFastAccessFeedView _view; FastAccessDocSubDBConfigurer _configurer; - FastAccessFixture() - : _service(1), - _view(_service.write()), - _configurer(_view._feedView, "test") - { - std::filesystem::remove_all(std::filesystem::path(BASE_DIR)); - std::filesystem::create_directory(std::filesystem::path(BASE_DIR)); - } - ~FastAccessFixture() { - _service.shutdown(); - } + FastAccessFixture() __attribute__((noinline)); + ~FastAccessFixture() __attribute__((noinline)); IReprocessingInitializer::UP reconfigure(const DocumentDBConfig& new_config_snapshot, @@ -351,6 +349,19 @@ struct FastAccessFixture SerialNum serial_num); }; +FastAccessFixture::FastAccessFixture() + : _service(1), + _view(_service.write()), + _configurer(_view._feedView, "test") +{ + std::filesystem::remove_all(std::filesystem::path(BASE_DIR)); + std::filesystem::create_directory(std::filesystem::path(BASE_DIR)); +} +FastAccessFixture::~FastAccessFixture() { + _service.shutdown(); +} + + IReprocessingInitializer::UP FastAccessFixture::reconfigure(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, @@ -367,14 +378,14 @@ FastAccessFixture::reconfigure(const DocumentDBConfig& new_config_snapshot, DocumentDBConfig::SP createConfig() { - return test::DocumentDBConfigBuilder(0, make_shared<Schema>(), "client", DOC_TYPE). + return proton::test::DocumentDBConfigBuilder(0, make_shared<Schema>(), "client", DOC_TYPE). repo(createRepo()).build(); } DocumentDBConfig::SP createConfig(const Schema::SP &schema) { - return test::DocumentDBConfigBuilder(0, schema, "client", DOC_TYPE). + return proton::test::DocumentDBConfigBuilder(0, schema, "client", DOC_TYPE). repo(createRepo()).build(); } @@ -491,6 +502,8 @@ FastAccessFeedViewComparer::FastAccessFeedViewComparer(FastAccessFeedView::SP ol {} FastAccessFeedViewComparer::~FastAccessFeedViewComparer() = default; +} + TEST_F("require that we can reconfigure index searchable", Fixture) { ViewPtrs o = f._views.getViewPtrs(); @@ -782,9 +795,3 @@ TEST("require that summary manager should change if relevant config changed") TEST_DO(assertSummaryManagerShouldChange(CCR().setStoreChanged(true))); TEST_DO(assertSummaryManagerShouldChange(CCR().setSchemaChanged(true))); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); - std::filesystem::remove_all(std::filesystem::path(BASE_DIR)); -} diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/.gitignore b/searchcore/src/tests/proton/documentdb/document_scan_iterator/.gitignore deleted file mode 100644 index 6c961d2f232..00000000000 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_document_scan_iterator_test_app diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt deleted file mode 100644 index 3f891843c1c..00000000000 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_scan_iterator_test_app TEST - SOURCES - document_scan_iterator_test.cpp - DEPENDS - searchcore_server - searchcore_feedoperation - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon -) -vespa_add_test(NAME searchcore_document_scan_iterator_test_app COMMAND searchcore_document_scan_iterator_test_app) diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator_test.cpp index e3ab2737fe5..23ae1c3cb74 100644 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator_test.cpp @@ -3,9 +3,10 @@ #include <vespa/searchcore/proton/server/document_scan_iterator.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/documentid.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace document; using namespace proton; @@ -81,8 +82,3 @@ TEST_F("require that only lids > lid limit are returned", Fixture) f.add({1,2,3,4,5,6,7,8}); assertLidSet({5,6,7,8}, f.scan(4, 4)); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/.gitignore b/searchcore/src/tests/proton/documentdb/document_subdbs/.gitignore deleted file mode 100644 index e47d2bafa0e..00000000000 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_document_subdbs_test_app diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt deleted file mode 100644 index a021a8a40b6..00000000000 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_subdbs_test_app TEST - SOURCES - document_subdbs_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_initializer - searchcore_reprocessing - searchcore_index - searchcore_docsummary - searchcore_persistenceengine - searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_flushengine - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - searchcore_grouping - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_document_subdbs_test_app COMMAND searchcore_document_subdbs_test_app) diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs_test.cpp index da8e842b9e4..f3f5c2063a0 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs_test.cpp @@ -38,8 +38,6 @@ #include <vespa/config-bucketspaces.h> #include <vespa/config/subscription/sourcespec.h> #include <vespa/document/test/make_bucket_space.h> -#include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/vespalib/util/hw_info.h> @@ -47,6 +45,9 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace cloud::config::filedistribution; using namespace document; @@ -84,15 +85,16 @@ using FastAccessContext = FastAccessDocSubDB::Context; using SearchableContext = SearchableDocSubDB::Context; using AttributeGuardList = std::vector<AttributeGuard>; +namespace { const std::string DOCTYPE_NAME = "searchdocument"; const std::string SUB_NAME = "subdb"; const std::string BASE_DIR = "basedir"; const SerialNum CFG_SERIAL = 5; -struct ConfigDir1 { static vespalib::string dir() { return TEST_PATH("cfg1"); } }; -struct ConfigDir2 { static vespalib::string dir() { return TEST_PATH("cfg2"); } }; -struct ConfigDir3 { static vespalib::string dir() { return TEST_PATH("cfg3"); } }; -struct ConfigDir4 { static vespalib::string dir() { return TEST_PATH("cfg4"); } }; +struct ConfigDir1 { static vespalib::string dir() { return TEST_PATH("document_subdbs/cfg1"); } }; +struct ConfigDir2 { static vespalib::string dir() { return TEST_PATH("document_subdbs/cfg2"); } }; +struct ConfigDir3 { static vespalib::string dir() { return TEST_PATH("document_subdbs/cfg3"); } }; +struct ConfigDir4 { static vespalib::string dir() { return TEST_PATH("document_subdbs/cfg4"); } }; struct MySubDBOwner : public IDocumentSubDBOwner { @@ -146,7 +148,7 @@ struct MyDocumentDBReferenceResolver : public IDocumentDBReferenceResolver { struct MyStoreOnlyConfig { StoreOnlyConfig _cfg; - MyStoreOnlyConfig(SubDbType subDbType) + explicit MyStoreOnlyConfig(SubDbType subDbType) : _cfg(DocTypeName(DOCTYPE_NAME), SUB_NAME, BASE_DIR, @@ -179,7 +181,7 @@ MyStoreOnlyContext::MyStoreOnlyContext(IThreadingService &writeService, IBucketDBHandlerInitializer &bucketDBHandlerInitializer) : _owner(), _syncProxy(), _getSerialNum(), _fileHeader(), _metrics(DOCTYPE_NAME, 1), _configMutex(), _hwInfo(), - _ctx(_owner, _syncProxy, _getSerialNum, _fileHeader, writeService, bucketDB, + _ctx(_owner, _syncProxy, _getSerialNum, _fileHeader, writeService, std::move(bucketDB), bucketDBHandlerInitializer, _metrics, _configMutex, _hwInfo) { } @@ -189,7 +191,7 @@ template <bool FastAccessAttributesOnly> struct MyFastAccessConfig { FastAccessConfig _cfg; - MyFastAccessConfig(SubDbType subDbType) + explicit MyFastAccessConfig(SubDbType subDbType) : _cfg(MyStoreOnlyConfig(subDbType)._cfg, true, true, FastAccessAttributesOnly) { } @@ -216,7 +218,7 @@ struct MyFastAccessContext MyFastAccessContext::MyFastAccessContext(IThreadingService &writeService, std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, IBucketDBHandlerInitializer & bucketDBHandlerInitializer) - : _storeOnlyCtx(writeService, bucketDB, bucketDBHandlerInitializer), + : _storeOnlyCtx(writeService, std::move(bucketDB), bucketDBHandlerInitializer), _attributeMetrics(nullptr), _wireService(), _ctx(_storeOnlyCtx._ctx, _attributeMetrics, _wireService, std::make_shared<search::attribute::Interlock>()) @@ -226,7 +228,7 @@ MyFastAccessContext::~MyFastAccessContext() = default; struct MySearchableConfig { FastAccessConfig _cfg; - MySearchableConfig(SubDbType subDbType) + explicit MySearchableConfig(SubDbType subDbType) : _cfg(MyFastAccessConfig<false>(subDbType)._cfg) { } @@ -254,7 +256,7 @@ struct MySearchableContext MySearchableContext::MySearchableContext(IThreadingService &writeService, std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, IBucketDBHandlerInitializer & bucketDBHandlerInitializer) - : _fastUpdCtx(writeService, bucketDB, bucketDBHandlerInitializer), + : _fastUpdCtx(writeService, std::move(bucketDB), bucketDBHandlerInitializer), _queryLimiter(), _clock(), _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock.nowRef(), writeService.shared()) {} @@ -289,29 +291,36 @@ struct MyConfigSnapshot DocBuilder _builder; DocumentDBConfig::SP _cfg; BootstrapConfig::SP _bootstrap; - MyConfigSnapshot(FNET_Transport & transport, const Schema &schema, const vespalib::string &cfgDir) - : _schema(schema), - _builder(get_add_fields(_schema.getNumAttributeFields() > 1)), - _cfg(), - _bootstrap() - { - auto documenttypesConfig = std::make_shared<DocumenttypesConfig>(_builder.get_documenttypes_config()); - auto tuneFileDocumentDB = std::make_shared<TuneFileDocumentDB>(); - _bootstrap = std::make_shared<BootstrapConfig>(1, - documenttypesConfig, - _builder.get_repo_sp(), - std::make_shared<ProtonConfig>(), - std::make_shared<FiledistributorrpcConfig>(), - std::make_shared<BucketspacesConfig>(), - tuneFileDocumentDB, HwInfo(HwInfo::Disk(128_Gi,false,false), HwInfo::Memory(16_Gi), HwInfo::Cpu(8))); - ::config::DirSpec spec(cfgDir); - DocumentDBConfigHelper mgr(spec, "searchdocument"); - mgr.forwardConfig(_bootstrap); - mgr.nextGeneration(transport, 1ms); - _cfg = mgr.getConfig(); - } + MyConfigSnapshot(FNET_Transport & transport, Schema schema, const vespalib::string &cfgDir); + MyConfigSnapshot(const MyConfigSnapshot &) = delete; + MyConfigSnapshot & operator = (const MyConfigSnapshot &) = delete; + ~MyConfigSnapshot(); }; +MyConfigSnapshot::~MyConfigSnapshot() = default; + +MyConfigSnapshot::MyConfigSnapshot(FNET_Transport & transport, Schema schema, const vespalib::string &cfgDir) + : _schema(std::move(schema)), + _builder(get_add_fields(_schema.getNumAttributeFields() > 1)), + _cfg(), + _bootstrap() +{ + auto documenttypesConfig = std::make_shared<DocumenttypesConfig>(_builder.get_documenttypes_config()); + auto tuneFileDocumentDB = std::make_shared<TuneFileDocumentDB>(); + _bootstrap = std::make_shared<BootstrapConfig>(1, + documenttypesConfig, + _builder.get_repo_sp(), + std::make_shared<ProtonConfig>(), + std::make_shared<FiledistributorrpcConfig>(), + std::make_shared<BucketspacesConfig>(), + tuneFileDocumentDB, HwInfo(HwInfo::Disk(128_Gi,false,false), HwInfo::Memory(16_Gi), HwInfo::Cpu(8))); + ::config::DirSpec spec(cfgDir); + DocumentDBConfigHelper mgr(spec, "searchdocument"); + mgr.forwardConfig(_bootstrap); + mgr.nextGeneration(transport, 1ms); + _cfg = mgr.getConfig(); +} + template <typename Traits> struct FixtureBase { @@ -369,11 +378,11 @@ struct FixtureBase void basicReconfig(SerialNum serialNum) { runInMasterAndSync([&]() { performReconfig(serialNum, make_all_attr_schema(two_attr_schema), ConfigDir2::dir()); }); } - void reconfig(SerialNum serialNum, const Schema &reconfigSchema, const vespalib::string &reconfigConfigDir) { - runInMasterAndSync([&]() { performReconfig(serialNum, reconfigSchema, reconfigConfigDir); }); + void reconfig(SerialNum serialNum, Schema reconfigSchema, const vespalib::string &reconfigConfigDir) { + runInMasterAndSync([&]() { performReconfig(serialNum, std::move(reconfigSchema), reconfigConfigDir); }); } - void performReconfig(SerialNum serialNum, const Schema &reconfigSchema, const vespalib::string &reconfigConfigDir) { - auto newCfg = std::make_unique<MyConfigSnapshot>(_service.transport(), reconfigSchema, reconfigConfigDir); + void performReconfig(SerialNum serialNum, Schema reconfigSchema, const vespalib::string &reconfigConfigDir) { + auto newCfg = std::make_unique<MyConfigSnapshot>(_service.transport(), std::move(reconfigSchema), reconfigConfigDir); DocumentDBConfig::ComparisonResult cmpResult; cmpResult.attributesChanged = true; cmpResult.documenttypesChanged = true; @@ -506,7 +515,7 @@ assertCacheCapacity(const StoreOnlyDocSubDB & db, size_t expected_cache_capacity const auto & summaryManager = db.getSummaryManager(); EXPECT_TRUE(dynamic_cast<SummaryManager *>(summaryManager.get()) != nullptr); search::IDocumentStore & store = summaryManager->getBackingStore(); - search::DocumentStore & docStore = dynamic_cast<search::DocumentStore &>(store); + auto & docStore = dynamic_cast<search::DocumentStore &>(store); EXPECT_EQUAL(expected_cache_capacity, docStore.getCacheCapacity()); } @@ -524,6 +533,8 @@ assertStoreOnly(StoreOnlyDocSubDB & db) { EXPECT_TRUE(dynamic_cast<MinimalDocumentRetriever *>(db.getDocumentRetriever().get()) != nullptr); } +} + TEST_F("require that managers and components are instantiated", StoreOnlyFixture) { assertStoreOnly(f._subDb); @@ -812,7 +823,7 @@ struct DocumentHandler { FixtureType &_f; DocBuilder _builder; - DocumentHandler(FixtureType &f) : _f(f), _builder(get_add_fields(f.has_attr2)) {} + explicit DocumentHandler(FixtureType &f) : _f(f), _builder(get_add_fields(f.has_attr2)) {} static constexpr uint32_t BUCKET_USED_BITS = 8; static DocumentId createDocId(uint32_t docId) { @@ -997,8 +1008,9 @@ assertOperation(DocumentOperation &op, uint32_t expPrevSubDbId, uint32_t expPrev TEST_F("require that lid allocation uses lowest free lid", StoreOnlyFixture) { + using Handler = DocumentHandler<StoreOnlyFixture>; f._subDb.onReplayDone(); - DocumentHandler<StoreOnlyFixture> handler(f); + Handler handler(f); Document::UP doc; PutOperation putOp; RemoveOperationWithDocId rmOp; @@ -1012,14 +1024,14 @@ TEST_F("require that lid allocation uses lowest free lid", StoreOnlyFixture) putOp = handler.createPut(std::move(doc), Timestamp(20), 20); handler.putDoc(putOp); EXPECT_TRUE(assertOperation(putOp, 0, 0, 0, 2)); - rmOp = handler.createRemove(handler.createDocId(1), Timestamp(30), 30); + rmOp = handler.createRemove(Handler::createDocId(1), Timestamp(30), 30); handler.removeDoc(rmOp); EXPECT_TRUE(assertOperation(rmOp, 0, 1, 0, 0)); doc = handler.createEmptyDoc(3); putOp = handler.createPut(std::move(doc), Timestamp(40), 40); handler.putDoc(putOp); EXPECT_TRUE(assertOperation(putOp, 0, 0, 0, 1)); - rmOp = handler.createRemove(handler.createDocId(3), Timestamp(50), 50); + rmOp = handler.createRemove(Handler::createDocId(3), Timestamp(50), 50); handler.removeDoc(rmOp); EXPECT_TRUE(assertOperation(rmOp, 0, 1, 0, 0)); doc = handler.createEmptyDoc(2); @@ -1083,8 +1095,3 @@ TEST_F("require that underlying components are explorable", SearchableExplorerFi EXPECT_TRUE(f._explorer.get_child("attributewriter").get() != nullptr); EXPECT_TRUE(f._explorer.get_child("index").get() != nullptr); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/.gitignore b/searchcore/src/tests/proton/documentdb/documentbucketmover/.gitignore deleted file mode 100644 index 4c7bc43b278..00000000000 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_documentbucketmover_test_app diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt deleted file mode 100644 index 591dd5fb368..00000000000 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -vespa_add_library(searchcore_bucketmover_test STATIC - SOURCES - bucketmover_common.cpp -) - -vespa_add_executable(searchcore_documentbucketmover_test_app TEST - SOURCES - documentbucketmover_test.cpp - DEPENDS - searchcore_bucketmover_test - searchcore_test - searchcore_server - searchcore_feedoperation - GTest::GTest -) -vespa_add_test(NAME searchcore_documentbucketmover_test_app COMMAND searchcore_documentbucketmover_test_app) - -vespa_add_executable(searchcore_documentmover_test_app TEST - SOURCES - documentmover_test.cpp - DEPENDS - searchcore_bucketmover_test - searchcore_server - searchcore_test - searchcore_feedoperation - GTest::GTest -) -vespa_add_test(NAME searchcore_documentmover_test_app COMMAND searchcore_documentmover_test_app) diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover_test.cpp index e9407d5445e..c1dd06602ae 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover_test.cpp @@ -8,12 +8,8 @@ #include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> #include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h> - -LOG_SETUP("document_bucket_mover_test"); +#include <vespa/vespalib/gtest/gtest.h> using namespace proton; using namespace proton::move::test; @@ -28,6 +24,7 @@ using vespalib::MonitoredRefCount; using vespalib::RetainGuard; using vespalib::ThreadStackExecutor; +namespace { struct ControllerFixtureBase : public ::testing::Test { test::UserDocumentsBuilder _builder; @@ -170,6 +167,8 @@ struct OnlyReadyControllerFixture : public ControllerFixtureBase } }; +} + TEST_F(ControllerFixture, require_that_nothing_is_moved_if_bucket_state_says_so) { EXPECT_TRUE(_bmj->done()); @@ -760,5 +759,3 @@ TEST_F(MaxOutstandingMoveOpsFixture_2, require_that_bucket_move_job_is_blocked_i sync(); assertDocsMoved(3, 1); } - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index 97c0331ce0b..82f01c01b7e 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -36,10 +36,11 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <filesystem> #include <iostream> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace cloud::config::filedistribution; using namespace proton; diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.sh b/searchcore/src/tests/proton/documentdb/documentdb_test.sh deleted file mode 100755 index bcecccc45d7..00000000000 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e -$VALGRIND ./searchcore_documentdb_test_app -rm -rf typea tmp diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/.gitignore b/searchcore/src/tests/proton/documentdb/documentdbconfig/.gitignore deleted file mode 100644 index 18a34a296b9..00000000000 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_documentdbconfig_test_app diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt deleted file mode 100644 index 4f24a2ff1a9..00000000000 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentdbconfig_test_app TEST - SOURCES - documentdbconfig_test.cpp - DEPENDS - searchcore_test - searchcore_server -) -vespa_add_test(NAME searchcore_documentdbconfig_test_app COMMAND searchcore_documentdbconfig_test_app) diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig_test.cpp index 73d27672380..a8870f66005 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig_test.cpp @@ -220,5 +220,3 @@ TEST_F("require that makeDelayedAttributeAspectConfig works, field removed with auto removed = DocumentDBConfig::makeDelayedAttributeAspectConfig(f.noAttrCfg, *f.attrCfg); TEST_DO(f.assertNotDelayedConfig(*removed)); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/.gitignore b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/.gitignore deleted file mode 100644 index 482e85e5db0..00000000000 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_documentdbconfigscout_test_app diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt deleted file mode 100644 index b8170ba6c11..00000000000 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentdbconfigscout_test_app TEST - SOURCES - documentdbconfigscout_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_attribute -) -vespa_add_test(NAME searchcore_documentdbconfigscout_test_app COMMAND searchcore_documentdbconfigscout_test_app) diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout_test.cpp index 862792bf274..a945550f108 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout_test.cpp @@ -4,9 +4,10 @@ #include <vespa/searchcore/proton/server/documentdbconfig.h> #include <vespa/searchcore/proton/server/documentdbconfigscout.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-attributes.h> #include <ostream> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace document; using namespace proton; @@ -37,8 +38,7 @@ std::ostream& operator<<(std::ostream& os, const AttributesConfig::Attribute::Di } -namespace -{ +namespace { DDBCSP getConfig(int64_t generation, const Schema::SP &schema, @@ -395,5 +395,3 @@ TEST("Test that DocumentDBConfigScout::scout looks ahead") EXPECT_TRUE(assertScoutedAttributes(scoutedCfg->getAttributesConfig(). attribute)); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentmover_test.cpp index bde32da7fb0..666ac0c3ec5 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentmover_test.cpp @@ -6,13 +6,11 @@ #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/log/log.h> -LOG_SETUP("document_bucket_mover_test"); - using namespace proton; using namespace proton::move::test; using document::BucketId; +namespace { struct MySubDbTwoBuckets : public MySubDb { MySubDbTwoBuckets(test::UserDocumentsBuilder &builder, @@ -68,6 +66,8 @@ struct DocumentMoverTest : ::testing::Test } }; +} + TEST_F(DocumentMoverTest, require_that_initial_bucket_mover_is_done) { MyMoveOperationLimiter limiter; @@ -151,5 +151,3 @@ TEST_F(DocumentMoverTest, require_that_cancel_signal_rescheduling_need) { EXPECT_TRUE(_mover.bucketDone()); EXPECT_TRUE(_mover.needReschedule()); } - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/executor_threading_service/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/executor_threading_service/CMakeLists.txt deleted file mode 100644 index 5c5cc270a5f..00000000000 --- a/searchcore/src/tests/proton/documentdb/executor_threading_service/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_executor_threading_service_test_app TEST - SOURCES - executor_threading_service_test.cpp - DEPENDS - searchcore_test - searchcore_server - GTest::GTest -) -vespa_add_test(NAME searchcore_executor_threading_service_test_app COMMAND searchcore_executor_threading_service_test_app) diff --git a/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp b/searchcore/src/tests/proton/documentdb/executor_threading_service_test.cpp index 8a354ded706..c3310eb8838 100644 --- a/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp +++ b/searchcore/src/tests/proton/documentdb/executor_threading_service_test.cpp @@ -59,6 +59,3 @@ TEST_F(ExecutorThreadingServiceTest, tasks_limits_can_be_updated) EXPECT_EQ(11, service->summary().getTaskLimit()); EXPECT_EQ(7, field_writer()->first_executor()->getTaskLimit()); } - -GTEST_MAIN_RUN_ALL_TESTS() - diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index cdd275e898d..383f891d5d2 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -38,11 +38,12 @@ #include <vespa/searchlib/test/doc_builder.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> #include <filesystem> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("feedhandler_test"); diff --git a/searchcore/src/tests/proton/documentdb/feedview/.gitignore b/searchcore/src/tests/proton/documentdb/feedview/.gitignore deleted file mode 100644 index 596e11ac15a..00000000000 --- a/searchcore/src/tests/proton/documentdb/feedview/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_feedview_test_app diff --git a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt deleted file mode 100644 index cbb1612b3f0..00000000000 --- a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_feedview_test_app TEST - SOURCES - feedview_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_index - searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - searchcore_grouping - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_feedview_test_app COMMAND searchcore_feedview_test_app) diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview_test.cpp index 77ae6be3d65..e2a8b227f57 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview_test.cpp @@ -33,7 +33,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/log/log.h> -LOG_SETUP("feedview_test"); +LOG_SETUP(".feedview_test"); using document::BucketId; using document::DataType; @@ -65,6 +65,7 @@ using namespace search::index; using SerialNum = SearchableFeedView::SerialNum; using DocumentIdT = search::DocumentIdT; +namespace { struct MyLidVector : public std::vector<DocumentIdT> { MyLidVector &add(DocumentIdT lid) { push_back(lid); return *this; } @@ -518,9 +519,9 @@ struct FixtureBase test::ThreadingServiceObserver _writeService; SerialNum serial; std::shared_ptr<MyGidToLidChangeHandler> _gidToLidChangeHandler; - FixtureBase(); + FixtureBase() __attribute__((noinline)); - virtual ~FixtureBase(); + virtual ~FixtureBase() __attribute__((noinline)); const test::DocumentMetaStoreObserver &metaStoreObserver() { return _dmsc->getObserver(); @@ -650,18 +651,7 @@ struct FixtureBase return EXPECT_EQUAL(exp, _tracer._os.str()); } - DocumentContext::List - makeDummyDocs(uint32_t first, uint32_t count, uint64_t tsfirst) { - DocumentContext::List docs; - for (uint32_t i = 0; i < count; ++i) { - uint32_t id = first + i; - uint64_t ts = tsfirst + i; - vespalib::asciistream os; - os << "id:ns:searchdocument::" << id; - docs.push_back(doc(os.str(), ts)); - } - return docs; - } + DocumentContext::List makeDummyDocs(uint32_t first, uint32_t count, uint64_t tsfirst) __attribute__((noinline)); void performCompactLidSpace(uint32_t wantedLidLimit, IDestructorCallback::SP onDone) { auto &fv = getFeedView(); @@ -718,6 +708,19 @@ FixtureBase::~FixtureBase() { _service.shutdown(); } +DocumentContext::List +FixtureBase::makeDummyDocs(uint32_t first, uint32_t count, uint64_t tsfirst) { + DocumentContext::List docs; + for (uint32_t i = 0; i < count; ++i) { + uint32_t id = first + i; + uint64_t ts = tsfirst + i; + vespalib::asciistream os; + os << "id:ns:searchdocument::" << id; + docs.push_back(doc(os.str(), ts)); + } + return docs; +} + void FixtureBase::populateBeforeCompactLidSpace() { @@ -729,44 +732,46 @@ FixtureBase::populateBeforeCompactLidSpace() struct SearchableFeedViewFixture : public FixtureBase { SearchableFeedView fv; - SearchableFeedViewFixture() : - FixtureBase(), - fv(StoreOnlyFeedView::Context(sa, sc._schema, _dmsc, - sc.getRepo(), _pendingLidsForCommit, - *_gidToLidChangeHandler, _writeService), - pc.getParams(), - FastAccessFeedView::Context(aw, _docIdLimit), - SearchableFeedView::Context(iw)) - { - } - ~SearchableFeedViewFixture() override { - forceCommitAndWait(); - } + SearchableFeedViewFixture() __attribute__((noinline)); + ~SearchableFeedViewFixture() override __attribute__((noinline)); IFeedView &getFeedView() override { return fv; } }; +SearchableFeedViewFixture::SearchableFeedViewFixture() + : FixtureBase(), + fv(StoreOnlyFeedView::Context(sa, sc._schema, _dmsc, + sc.getRepo(), _pendingLidsForCommit, + *_gidToLidChangeHandler, _writeService), + pc.getParams(), + FastAccessFeedView::Context(aw, _docIdLimit), + SearchableFeedView::Context(iw)) +{ } +SearchableFeedViewFixture::~SearchableFeedViewFixture() { + forceCommitAndWait(); +} + struct FastAccessFeedViewFixture : public FixtureBase { FastAccessFeedView fv; - FastAccessFeedViewFixture() : - FixtureBase(), - fv(StoreOnlyFeedView::Context(sa, sc._schema, _dmsc, sc.getRepo(), _pendingLidsForCommit, - *_gidToLidChangeHandler, _writeService), - pc.getParams(), - FastAccessFeedView::Context(aw, _docIdLimit)) - { - } - ~FastAccessFeedViewFixture() override { - forceCommitAndWait(); - } + FastAccessFeedViewFixture() __attribute__((noinline)); + ~FastAccessFeedViewFixture() override __attribute__((noinline)); IFeedView &getFeedView() override { return fv; } }; -void -assertBucketInfo(const BucketId &ebid, - const Timestamp &ets, - uint32_t lid, - const IDocumentMetaStore &metaStore) +FastAccessFeedViewFixture::FastAccessFeedViewFixture() + : FixtureBase(), + fv(StoreOnlyFeedView::Context(sa, sc._schema, _dmsc, sc.getRepo(), _pendingLidsForCommit, + *_gidToLidChangeHandler, _writeService), + pc.getParams(), + FastAccessFeedView::Context(aw, _docIdLimit)) +{ } + +FastAccessFeedViewFixture::~FastAccessFeedViewFixture() { + forceCommitAndWait(); +} + +void assertBucketInfo(const BucketId &ebid, const Timestamp &ets, uint32_t lid, const IDocumentMetaStore &metaStore) __attribute__((noinline)); +void assertBucketInfo(const BucketId &ebid, const Timestamp &ets, uint32_t lid, const IDocumentMetaStore &metaStore) { document::GlobalId gid; EXPECT_TRUE(metaStore.getGid(lid, gid)); @@ -777,8 +782,8 @@ assertBucketInfo(const BucketId &ebid, EXPECT_EQUAL(ets, meta.timestamp); } -void -assertLidVector(const MyLidVector &exp, const MyLidVector &act) +void assertLidVector(const MyLidVector &exp, const MyLidVector &act) __attribute__((noinline)); +void assertLidVector(const MyLidVector &exp, const MyLidVector &act) { EXPECT_EQUAL(exp.size(), act.size()); for (size_t i = 0; i < exp.size(); ++i) { @@ -795,6 +800,8 @@ assertAttributeUpdate(SerialNum serialNum, const document::DocumentId &docId, EXPECT_EQUAL(lid, adapter._updateLid); } +} + TEST_F("require that put() updates document meta store with bucket info", SearchableFeedViewFixture) @@ -1287,9 +1294,3 @@ TEST_F("require that move() notifies gid to lid change handler", SearchableFeedV f.forceCommitAndWait(); TEST_DO(f.assertChangeHandler(dc2.gid(), 1u, 4u)); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} - diff --git a/searchcore/src/tests/proton/documentdb/gtest_runner.cpp b/searchcore/src/tests/proton/documentdb/gtest_runner.cpp new file mode 100644 index 00000000000..882e3a202b1 --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/gtest_runner.cpp @@ -0,0 +1,8 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> + +#include <vespa/log/log.h> +LOG_SETUP("searchcore_proton_documentdb_gtest_runner"); + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/.gitignore b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/.gitignore deleted file mode 100644 index 1e657f33c1a..00000000000 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_job_tracked_maintenance_job_test_app diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt deleted file mode 100644 index 17903d4dd2f..00000000000 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_job_tracked_maintenance_job_test_app TEST - SOURCES - job_tracked_maintenance_job_test.cpp - DEPENDS - searchcore_server - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_job_tracked_maintenance_job_test_app COMMAND searchcore_job_tracked_maintenance_job_test_app) diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job_test.cpp index e1ec3d2bb29..f81a3b724dd 100644 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp +++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job_test.cpp @@ -9,9 +9,6 @@ #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/log/log.h> -LOG_SETUP("job_tracked_maintenance_test"); - using namespace proton; using namespace vespalib; using test::SimpleJobTracker; @@ -146,5 +143,3 @@ TEST_F("require that stop calls are sent to underlying jobs", Fixture) f._trackedJob->stop(); EXPECT_TRUE(f._myJob->stopped()); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp b/searchcore/src/tests/proton/documentdb/lid_space_common.cpp index 44e663a559b..44e663a559b 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_common.cpp diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.h b/searchcore/src/tests/proton/documentdb/lid_space_common.h index 9e7635d3c3a..9e7635d3c3a 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.h +++ b/searchcore/src/tests/proton/documentdb/lid_space_common.h diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/.gitignore b/searchcore/src/tests/proton/documentdb/lid_space_compaction/.gitignore deleted file mode 100644 index c031fe6605d..00000000000 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_lid_space_compaction_test_app diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt deleted file mode 100644 index 3725a762c60..00000000000 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -vespa_add_library(searchcore_lidspace_test STATIC - SOURCES - lid_space_common.cpp -) - -vespa_add_executable(searchcore_lid_space_compaction_test_app TEST - SOURCES - lid_space_jobtest.cpp - lid_space_compaction_test.cpp - DEPENDS - searchcore_lidspace_test - searchcore_test - searchcore_server - searchcore_initializer - searchcore_feedoperation - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - GTest::GTest -) -vespa_add_test(NAME searchcore_lid_space_compaction_test_app COMMAND searchcore_lid_space_compaction_test_app) - -vespa_add_executable(searchcore_lid_space_handler_test_app TEST - SOURCES - lid_space_handler_test.cpp - DEPENDS - searchcore_lidspace_test - searchcore_test - searchcore_server - searchcore_initializer - searchcore_feedoperation - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - GTest::GTest -) -vespa_add_test(NAME searchcore_lid_space_handler_test_app COMMAND searchcore_lid_space_handler_test_app) diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction_test.cpp index 984af9870a6..681caad38a4 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction_test.cpp @@ -271,5 +271,3 @@ TEST_F(MaxOutstandingJobTest, job_is_blocked_if_it_has_too_many_outstanding_move assertJobContext(4, 7, 3, 7, 1); sync(); } - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_handler_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_handler_test.cpp index c5a1bc370c1..ca88510dc4d 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_handler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_handler_test.cpp @@ -4,6 +4,7 @@ #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/vespalib/gtest/gtest.h> +namespace { struct HandlerTest : public ::testing::Test { DocBuilder _docBuilder; std::shared_ptr<bucketdb::BucketDBOwner> _bucketDB; @@ -26,6 +27,8 @@ HandlerTest::HandlerTest() HandlerTest::~HandlerTest() = default; +} + TEST_F(HandlerTest, handler_uses_doctype_and_subdb_name) { EXPECT_EQ("test.dummysubdb", _handler.getName()); @@ -58,4 +61,3 @@ TEST_F(HandlerTest, createMoveOperation_works_as_expected) EXPECT_EQ(timestamp, op->getTimestamp()); } -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp b/searchcore/src/tests/proton/documentdb/lid_space_jobtest.cpp index eea523870c3..eea523870c3 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_jobtest.cpp diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h b/searchcore/src/tests/proton/documentdb/lid_space_jobtest.h index 100458bb867..100458bb867 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h +++ b/searchcore/src/tests/proton/documentdb/lid_space_jobtest.h diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/.gitignore b/searchcore/src/tests/proton/documentdb/maintenancecontroller/.gitignore deleted file mode 100644 index 7ce70f9cbcd..00000000000 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -searchcore_frozenbucketsmap_test_app -searchcore_maintenancecontroller_test_app diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt deleted file mode 100644 index 0b4220902dc..00000000000 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_maintenancecontroller_test_app TEST - SOURCES - maintenancecontroller_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_feedoperation - searchcore_matching - searchcore_attribute - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - searchcore_persistenceengine - searchcore_grouping - searchcore_proton_metrics - searchlib_test -) -vespa_add_test(NAME searchcore_maintenancecontroller_test_app COMMAND searchcore_maintenancecontroller_test_app) diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller_test.cpp index 6c67848ae51..891af3eda39 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller_test.cpp @@ -41,12 +41,8 @@ #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <unistd.h> #include <thread> -#include <vespa/log/log.h> -LOG_SETUP("maintenancecontroller_test"); - using namespace proton; using namespace vespalib::slime; using document::BucketId; @@ -74,6 +70,7 @@ using BlockedReason = IBlockableMaintenanceJob::BlockedReason; using BucketIdVector = BucketId::List; constexpr vespalib::duration TIMEOUT = 60s; +constexpr vespalib::duration DELAY = 10ms; namespace { @@ -151,40 +148,13 @@ struct MyDocumentRetriever : public DocumentRetrieverBaseForTest { MyDocumentSubDB &_subDB; - explicit MyDocumentRetriever(MyDocumentSubDB &subDB) noexcept - : _subDB(subDB) - { - } - - const document::DocumentTypeRepo & - getDocumentTypeRepo() const override - { - LOG_ABORT("should not be reached"); - } - - void - getBucketMetaData(const storage::spi::Bucket &, - DocumentMetaData::Vector &) const override - { - LOG_ABORT("should not be reached"); - } - DocumentMetaData - getDocumentMetaData(const DocumentId &) const override - { - return DocumentMetaData(); - } - - Document::UP - getFullDocument(DocumentIdT lid) const override - { - return _subDB.getDocument(lid); - } + explicit MyDocumentRetriever(MyDocumentSubDB &subDB) noexcept : _subDB(subDB) { } - CachedSelect::SP - parseSelect(const vespalib::string &) const override - { - return CachedSelect::SP(); - } + const document::DocumentTypeRepo & getDocumentTypeRepo() const override { abort(); } + void getBucketMetaData(const storage::spi::Bucket &, DocumentMetaData::Vector &) const override { abort(); } + DocumentMetaData getDocumentMetaData(const DocumentId &) const override { return {}; } + Document::UP getFullDocument(DocumentIdT lid) const override { return _subDB.getDocument(lid); } + CachedSelect::SP parseSelect(const vespalib::string &) const override { return {}; } }; @@ -268,7 +238,6 @@ struct MySimpleJob : public BlockableMaintenanceJob { } void block() { setBlocked(BlockedReason::FROZEN_BUCKET); } bool run() override { - LOG(info, "MySimpleJob::run()"); _latch.countDown(); ++_runCnt; return true; @@ -284,7 +253,6 @@ struct MySplitJob : public MySimpleJob { } bool run() override { - LOG(info, "MySplitJob::run()"); _latch.countDown(); ++_runCnt; return _latch.getCount() == 0; @@ -304,7 +272,7 @@ struct MyLongRunningJob : public BlockableMaintenanceJob void block() { setBlocked(BlockedReason::FROZEN_BUCKET); } bool run() override { _firstRun.countDown(); - usleep(10000); + std::this_thread::sleep_for(1ms); return false; } }; @@ -885,9 +853,9 @@ TEST_F("require that document pruner is active", MaintenanceControllerFixture) ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(10u, f._removed.getNumUsedLids()); EXPECT_EQUAL(10u, f._removed.getDocumentCount()); - f.setPruneConfig(DocumentDBPruneConfig(200ms, 900s)); - for (uint32_t i = 0; i < 600; ++i) { - std::this_thread::sleep_for(100ms); + f.setPruneConfig(DocumentDBPruneConfig(DELAY, 900s)); + for (uint32_t i = 0; i < 60000; ++i) { + std::this_thread::sleep_for(1ms); ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); if (f._removed.getNumUsedLids() != 10u) break; @@ -902,9 +870,9 @@ TEST_F("require that heartbeats are scheduled", MaintenanceControllerFixture) { f.notifyClusterStateChanged(); f.startMaintenance(); - f.setHeartBeatConfig(DocumentDBHeartBeatConfig(200ms)); - for (uint32_t i = 0; i < 600; ++i) { - std::this_thread::sleep_for(100ms); + f.setHeartBeatConfig(DocumentDBHeartBeatConfig(DELAY)); + for (uint32_t i = 0; i < 60000; ++i) { + std::this_thread::sleep_for(1ms); if (f._fh.getHeartBeats() != 0u) break; } @@ -913,7 +881,7 @@ TEST_F("require that heartbeats are scheduled", MaintenanceControllerFixture) TEST_F("require that a simple maintenance job is executed", MaintenanceControllerFixture) { - auto job = std::make_unique<MySimpleJob>(200ms, 200ms, 3); + auto job = std::make_unique<MySimpleJob>(DELAY, DELAY, 3); MySimpleJob &myJob = *job; f._mc.registerJob(std::move(job)); f._injectDefaultJobs = false; @@ -925,7 +893,7 @@ TEST_F("require that a simple maintenance job is executed", MaintenanceControlle TEST_F("require that a split maintenance job is executed", MaintenanceControllerFixture) { - auto job = std::make_unique<MySplitJob>(200ms, TIMEOUT * 2, 3); + auto job = std::make_unique<MySplitJob>(DELAY, TIMEOUT * 2, 3); MySplitJob &myJob = *job; f._mc.registerJob(std::move(job)); f._injectDefaultJobs = false; @@ -937,13 +905,15 @@ TEST_F("require that a split maintenance job is executed", MaintenanceController TEST_F("require that blocked jobs are not executed", MaintenanceControllerFixture) { - auto job = std::make_unique<MySimpleJob>(200ms, 200ms, 0); + auto job = std::make_unique<MySimpleJob>(DELAY, DELAY, 0); MySimpleJob &myJob = *job; myJob.block(); f._mc.registerJob(std::move(job)); f._injectDefaultJobs = false; f.startMaintenance(); - std::this_thread::sleep_for(2s); + for (uint32_t napCount = 0; (myJob._runCnt != 0) && (napCount < 200); napCount++) { + std::this_thread::sleep_for(10ms); + } EXPECT_EQUAL(0u, myJob._runCnt); } @@ -951,7 +921,7 @@ TEST_F("require that maintenance controller state list jobs", MaintenanceControl { { auto job1 = std::make_unique<MySimpleJob>(TIMEOUT * 2, TIMEOUT * 2, 0); - auto job2 = std::make_unique<MyLongRunningJob>(200ms, 200ms); + auto job2 = std::make_unique<MyLongRunningJob>(DELAY, DELAY); auto &longRunningJob = dynamic_cast<MyLongRunningJob &>(*job2); f._mc.registerJob(std::move(job1)); f._mc.registerJob(std::move(job2)); @@ -1022,8 +992,3 @@ TEST_F("require that delay for prune removed documents is set based on interval assertPruneRemovedDocumentsConfig(300s, 301s, 301s, f); assertPruneRemovedDocumentsConfig(299s, 299s, 299s, f); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/tests/proton/documentdb/move_operation_limiter/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/move_operation_limiter/CMakeLists.txt deleted file mode 100644 index 2f10bd9f025..00000000000 --- a/searchcore/src/tests/proton/documentdb/move_operation_limiter/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_move_operation_limiter_test_app TEST - SOURCES - move_operation_limiter_test.cpp - DEPENDS - searchcore_server -) -vespa_add_test(NAME searchcore_move_operation_limiter_test_app COMMAND searchcore_move_operation_limiter_test_app) diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/.gitignore b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/.gitignore deleted file mode 100644 index eaabc7b9279..00000000000 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -Makefile -.depend -*_test -searchcore_storeonlyfeedview_test_app diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt deleted file mode 100644 index 58a4afc4a72..00000000000 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_storeonlyfeedview_test_app TEST - SOURCES - storeonlyfeedview_test.cpp - DEPENDS - searchcore_test - searchcore_server - searchcore_feedoperation - searchcore_documentmetastore - searchcore_bucketdb - searchcore_proton_metrics - searchcore_pcommon -) -vespa_add_test(NAME searchcore_storeonlyfeedview_test_app COMMAND searchcore_storeonlyfeedview_test_app) diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview_test.cpp index db30c8e03fb..8938f1a0745 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview_test.cpp @@ -19,9 +19,6 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/log/log.h> -LOG_SETUP("storeonlyfeedview_test"); - using document::BucketId; using document::DataType; using document::Document; @@ -314,8 +311,7 @@ struct MoveFixture : public FixtureBase<MoveOperationFeedView> { }); // First we wait for everything propagated to MinimalFeedView while (ctx.use_count() > (expected + 1)) { - LOG(info, "use_count = %ld", ctx.use_count()); - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(10ms); } // And then we must wait for everyone else to finish up too. feedview->waitFor(expected*2); @@ -447,5 +443,3 @@ TEST_F("require that heartbeat propagates and commits meta store", Fixture) } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/threading_service_config/CMakeLists.txt deleted file mode 100644 index f03d0597367..00000000000 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_threading_service_config_test_app TEST - SOURCES - threading_service_config_test.cpp - DEPENDS - searchcore_server -) -vespa_add_test(NAME searchcore_threading_service_config_test_app COMMAND searchcore_threading_service_config_test_app) diff --git a/searchcore/src/tests/proton/documentdb/vespatest_runner.cpp b/searchcore/src/tests/proton/documentdb/vespatest_runner.cpp new file mode 100644 index 00000000000..1e4e79047c3 --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/vespatest_runner.cpp @@ -0,0 +1,8 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Unit tests for predicate_index. +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/log/log.h> +LOG_SETUP("proton_common_test"); + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index dc9dcd3e0b0..d43d1184e2c 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -13,9 +13,10 @@ #include <vespa/searchlib/common/flush_token.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <mutex> #include <thread> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("flushengine_test"); diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 7abac088011..fd80a05860b 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -189,7 +189,7 @@ TEST_F("require that tls replay cost is correct for zero operations to replay", TEST_F("require that flush cost is correct for zero flush targets", CandidatesFixture) { - EXPECT_EQUAL(0, f.builder.build().getFlushTargetsWriteCost()); + EXPECT_EQUAL(0.0, f.builder.build().getFlushTargetsWriteCost()); } TEST_F("require that flush cost is sum of flush targets", CandidatesFixture) diff --git a/searchcore/src/tests/proton/index/index_writer/.gitignore b/searchcore/src/tests/proton/index/index_writer/.gitignore deleted file mode 100644 index bbada541cf0..00000000000 --- a/searchcore/src/tests/proton/index/index_writer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_index_writer_test_app diff --git a/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt b/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt deleted file mode 100644 index d9fb7ab6018..00000000000 --- a/searchcore/src/tests/proton/index/index_writer/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_index_writer_test_app TEST - SOURCES - index_writer_test.cpp - DEPENDS - searchcore_test - searchcore_index - searchcore_pcommon -) -vespa_add_test(NAME searchcore_index_writer_test_app COMMAND searchcore_index_writer_test_app) diff --git a/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp b/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp index 312c5b6eaa1..1db8a2d93b3 100644 --- a/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp +++ b/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp @@ -63,7 +63,7 @@ struct Fixture { TEST_F("require that constant value can be retrieved from repo", Fixture) { - EXPECT_EQUAL(3, f.repo.getConstant("foo")->value().as_double()); + EXPECT_EQUAL(3.0, f.repo.getConstant("foo")->value().as_double()); } TEST_F("require that non-existing constant value in repo returns nullptr", Fixture) diff --git a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp index b17453c4b6d..1c7e7083ee7 100644 --- a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp +++ b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp @@ -1,10 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> - #include <vespa/eval/eval/value_cache/constant_value.h> #include <vespa/searchcore/proton/matching/indexenvironment.h> #include <vespa/searchlib/fef/onnx_models.h> #include <vespa/searchlib/fef/ranking_expressions.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace proton::matching; using search::fef::FieldInfo; diff --git a/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp b/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp index 6f398469fd3..9ad495a7955 100644 --- a/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp +++ b/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/match_phase_limiter.h> #include <vespa/searchcore/proton/matching/rangequerylocator.h> #include <vespa/searchlib/queryeval/termasstring.h> @@ -17,6 +16,8 @@ #include <vespa/searchlib/engine/trace.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace proton::matching; using namespace search::engine; diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index b917e69ce7b..5f3db842d72 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -30,8 +30,9 @@ #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/thread_bundle.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/query/tree/querytreecreator.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("query_test"); @@ -237,7 +238,8 @@ public: template <class TermType> void checkNode(const TermType &n, int estimatedHitCount, bool empty) { EXPECT_EQUAL(empty, (estimatedHitCount == 0)); - EXPECT_EQUAL((double)estimatedHitCount / doc_count, n.field(0).getDocFreq()); + EXPECT_EQUAL(static_cast<uint64_t>(estimatedHitCount), n.field(0).get_doc_freq().frequency); + EXPECT_EQUAL(static_cast<uint64_t>(doc_count), n.field(0).get_doc_freq().count); } void visit(ProtonNumberTerm &n) override { checkNode(n, 1, false); } @@ -383,7 +385,8 @@ public: EXPECT_EQUAL(term_data.numFields(), n.numFields()); for (size_t i = 0; i < term_data.numFields(); ++i) { const ITermFieldData &term_field_data = term_data.field(i); - EXPECT_APPROX(2.0 / doc_count, term_field_data.getDocFreq(), 1.0e-6); + EXPECT_EQUAL(2u, term_field_data.get_doc_freq().frequency); + EXPECT_EQUAL(static_cast<uint64_t>(doc_count), term_field_data.get_doc_freq().count); EXPECT_TRUE(!n.field(i).attribute_field); EXPECT_EQUAL(field_id + i, term_field_data.getFieldId()); } @@ -823,7 +826,8 @@ TEST("requireThatNoDocsGiveZeroDocFrequency") { Blueprint::UP blueprint = BlueprintBuilder::build(requestContext, node, context); EXPECT_EQUAL(1u, node.numFields()); - EXPECT_EQUAL(0.0, node.field(0).getDocFreq()); + EXPECT_EQUAL(0u, node.field(0).get_doc_freq().frequency); + EXPECT_EQUAL(1u, node.field( 0).get_doc_freq().count); } TEST("requireThatWeakAndBlueprintsAreCreatedCorrectly") { diff --git a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp index 3153bcccb42..483034a112d 100644 --- a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp +++ b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp @@ -8,8 +8,10 @@ #include <vespa/searchcore/proton/matching/match_tools.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> + #include <vespa/log/log.h> LOG_SETUP("sessionmanager_test"); diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/.gitignore b/searchcore/src/tests/proton/metrics/metrics_engine/.gitignore deleted file mode 100644 index 98ae77cb458..00000000000 --- a/searchcore/src/tests/proton/metrics/metrics_engine/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_metrics_engine_test_app diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt b/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt deleted file mode 100644 index 6cc8e133ece..00000000000 --- a/searchcore/src/tests/proton/metrics/metrics_engine/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_metrics_engine_test_app TEST - SOURCES - metrics_engine_test.cpp - DEPENDS - searchcore_flushengine - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_metrics_engine_test_app COMMAND searchcore_metrics_engine_test_app) diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 0f1e2e9487a..ea8fbcb57dc 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -17,8 +17,9 @@ #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <set> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using document::BucketId; using document::BucketSpace; diff --git a/searchcore/src/tests/proton/proton_config_fetcher/.cvsignore b/searchcore/src/tests/proton/proton_config_fetcher/.cvsignore deleted file mode 100644 index 13fb04d2a35..00000000000 --- a/searchcore/src/tests/proton/proton_config_fetcher/.cvsignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -config_test diff --git a/searchcore/src/tests/proton/proton_config_fetcher/.gitignore b/searchcore/src/tests/proton/proton_config_fetcher/.gitignore deleted file mode 100644 index 72c49479fc1..00000000000 --- a/searchcore/src/tests/proton/proton_config_fetcher/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_config_test_app diff --git a/searchcore/src/tests/proton/proton_config_fetcher/CMakeLists.txt b/searchcore/src/tests/proton/proton_config_fetcher/CMakeLists.txt deleted file mode 100644 index a91c7914bce..00000000000 --- a/searchcore/src/tests/proton/proton_config_fetcher/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_proton_config_fetcher_test_app TEST - SOURCES - proton_config_fetcher_test.cpp - DEPENDS - searchcore_test - searchcore_server -) -vespa_add_test(NAME searchcore_proton_config_fetcher_test_app COMMAND searchcore_proton_config_fetcher_test_app) diff --git a/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt deleted file mode 100644 index 5f1314f0e18..00000000000 --- a/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_proton_disk_layout_test_app TEST - SOURCES - proton_disk_layout_test.cpp - DEPENDS - searchcore_test - searchcore_server -) -vespa_add_test(NAME searchcore_proton_disk_layout_test_app COMMAND searchcore_proton_disk_layout_test_app) diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp index 70211909296..bbe4b10851e 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> @@ -9,6 +8,9 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/searchcore/proton/reference/gid_to_lid_mapper.h> #include <vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> + #include <vespa/log/log.h> LOG_SETUP("gid_to_lid_mapper_test"); diff --git a/searchcore/src/tests/proton/server/.gitignore b/searchcore/src/tests/proton/server/.gitignore index dc96b15f5fe..c96dc3b275f 100644 --- a/searchcore/src/tests/proton/server/.gitignore +++ b/searchcore/src/tests/proton/server/.gitignore @@ -2,8 +2,3 @@ .depend Makefile test_data -searchcore_attribute_metrics_test_app -searchcore_documentretriever_test_app -searchcore_feeddebugger_test_app -searchcore_feedstates_test_app -searchcore_memoryconfigstore_test_app diff --git a/searchcore/src/tests/proton/server/CMakeLists.txt b/searchcore/src/tests/proton/server/CMakeLists.txt index c522f4ede9c..b9fb096bf3a 100644 --- a/searchcore/src/tests/proton/server/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/CMakeLists.txt @@ -1,44 +1,18 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentretriever_test_app TEST +vespa_add_executable(searchcore_proton_server_vespa_test_app TEST SOURCES + vespa_testrunner.cpp documentretriever_test.cpp - DEPENDS - searchcore_server - searchcore_attribute - searchcore_feedoperation - searchcore_documentmetastore - searchcore_bucketdb - searchcore_pcommon - searchcore_persistenceengine -) -vespa_add_test(NAME searchcore_documentretriever_test_app COMMAND searchcore_documentretriever_test_app) -vespa_add_executable(searchcore_feeddebugger_test_app TEST - SOURCES feeddebugger_test.cpp - DEPENDS - searchcore_server - searchcore_pcommon -) -vespa_add_test(NAME searchcore_feeddebugger_test_app COMMAND searchcore_feeddebugger_test_app) -vespa_add_executable(searchcore_feedstates_test_app TEST - SOURCES feedstates_test.cpp + health_adapter_test.cpp + memory_flush_config_updater_test.cpp + memoryconfigstore_test.cpp + move_operation_limiter_test.cpp + threading_service_config_test.cpp DEPENDS - searchcore_test searchcore_server - searchcore_bucketdb - searchcore_persistenceengine searchcore_feedoperation - searchcore_pcommon - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_feedstates_test_app COMMAND searchcore_feedstates_test_app) -vespa_add_executable(searchcore_memoryconfigstore_test_app TEST - SOURCES - memoryconfigstore_test.cpp - DEPENDS searchcore_test - searchcore_server - searchcore_pcommon ) -vespa_add_test(NAME searchcore_memoryconfigstore_test_app COMMAND searchcore_memoryconfigstore_test_app) +vespa_add_test(NAME searchcore_proton_server_vespa_test_app COMMAND searchcore_proton_server_vespa_test_app) diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index b05964fe7e0..426c709cf52 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -40,7 +40,6 @@ #include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> @@ -48,11 +47,10 @@ #include <vespa/eval/eval/test/value_compare.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/test.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> -#include <vespa/log/log.h> -LOG_SETUP("document_retriever_test"); - using document::ArrayFieldValue; using document::FieldValue; using document::BucketId; @@ -669,5 +667,3 @@ TEST("require that fieldset can figure out their attributeness and rember it") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/server/feeddebugger_test.cpp b/searchcore/src/tests/proton/server/feeddebugger_test.cpp index 4d1bbdb2f03..1058979305e 100644 --- a/searchcore/src/tests/proton/server/feeddebugger_test.cpp +++ b/searchcore/src/tests/proton/server/feeddebugger_test.cpp @@ -1,12 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for feeddebugger. -#include <vespa/log/log.h> -LOG_SETUP("feeddebugger_test"); - #include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/common/feeddebugger.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using document::DocumentId; using std::string; @@ -46,8 +44,7 @@ TEST("require that when environment variable is not set, debugging is off") { EXPECT_FALSE(debugger.isDebugging()); } -TEST("require that setting an environment variable turns on lid-specific" - " debugging.") { +TEST("require that setting an environment variable turns on lid-specific debugging.") { EnvSaver save_lid_env(lid_env_name); EnvSaver save_docid_env(docid_env_name); setenv(lid_env_name, "1,3,5", true); @@ -61,8 +58,7 @@ TEST("require that setting an environment variable turns on lid-specific" EXPECT_EQUAL(ns_log::Logger::info, debugger.getDebugLevel(5, 0)); } -TEST("require that setting an environment variable turns on docid-specific" - " debugging.") { +TEST("require that setting an environment variable turns on docid-specific debugging.") { EnvSaver save_lid_env(lid_env_name); EnvSaver save_docid_env(docid_env_name); setenv(docid_env_name, "id:ns:type::test:foo,id:ns:type::test:bar,id:ns:type::test:baz", true); @@ -80,5 +76,3 @@ TEST("require that setting an environment variable turns on docid-specific" } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index c46510c5181..bd88a5b8038 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -16,11 +16,9 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/foreground_thread_executor.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/buffer.h> - -#include <vespa/log/log.h> -LOG_SETUP("feedstates_test"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using document::BucketId; using document::DocumentId; @@ -154,5 +152,3 @@ TEST_F("require that replay progress is tracked", Fixture) } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/server/health_adapter/.gitignore b/searchcore/src/tests/proton/server/health_adapter/.gitignore deleted file mode 100644 index c82499f49b7..00000000000 --- a/searchcore/src/tests/proton/server/health_adapter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_health_adapter_test_app diff --git a/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt b/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt deleted file mode 100644 index c65481e3751..00000000000 --- a/searchcore/src/tests/proton/server/health_adapter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_health_adapter_test_app TEST - SOURCES - health_adapter_test.cpp - DEPENDS - searchcore_server - searchcore_pcommon -) -vespa_add_test(NAME searchcore_health_adapter_test_app COMMAND searchcore_health_adapter_test_app) diff --git a/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp b/searchcore/src/tests/proton/server/health_adapter_test.cpp index aa247140dc5..2e153516986 100644 --- a/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp +++ b/searchcore/src/tests/proton/server/health_adapter_test.cpp @@ -57,5 +57,3 @@ TEST_FF("require that multiple failure messages are concatenated", MyStatusProdu EXPECT_FALSE(f2.getHealth().ok); EXPECT_EQUAL(std::string("c1: xxx, c3: zzz"), f2.getHealth().msg); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater_test.cpp index 0dc8390d7d9..4aac075caa6 100644 --- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp +++ b/searchcore/src/tests/proton/server/memory_flush_config_updater_test.cpp @@ -236,5 +236,3 @@ TEST_F("require that more disk bloat is allowed while node state is retired or m f.notifyDiskMemUsage(belowLimit(), belowLimit()); TEST_DO(f.assertStrategyDiskConfig(DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT)); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp index c0245cc1a9d..ccb893b65a2 100644 --- a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp +++ b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp @@ -1,9 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for memoryconfigstore. -#include <vespa/log/log.h> -LOG_SETUP("memoryconfigstore_test"); - #include <vespa/searchcommon/common/schema.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> @@ -106,5 +103,3 @@ TEST("require that MemoryConfigStores preserves state of " } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp b/searchcore/src/tests/proton/server/move_operation_limiter_test.cpp index 766158bd178..dee5fd2fd2c 100644 --- a/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp +++ b/searchcore/src/tests/proton/server/move_operation_limiter_test.cpp @@ -5,9 +5,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <queue> -#include <vespa/log/log.h> -LOG_SETUP("move_operation_limiter_test"); - using namespace proton; struct MyBlockableMaintenanceJob : public IBlockableMaintenanceJob { @@ -110,8 +107,3 @@ TEST_F("require that destructor callback has reference to limiter via shared ptr f.endOp(); EXPECT_FALSE(f.job.blocked); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/server/threading_service_config_test.cpp index 7a16e6b17cb..567a723de39 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/server/threading_service_config_test.cpp @@ -5,9 +5,6 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/hw_info.h> -#include <vespa/log/log.h> -LOG_SETUP("threading_service_config_test"); - using namespace proton; using ProtonConfig = vespa::config::search::core::ProtonConfig; using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder; @@ -63,8 +60,3 @@ TEST_FF("require that config can be somewhat updated", Fixture(), Fixture(3000, cfg1.update(cfg2); assertConfig(3000u, 1000u, cfg1); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/tests/proton/server/vespa_testrunner.cpp b/searchcore/src/tests/proton/server/vespa_testrunner.cpp new file mode 100644 index 00000000000..1e4e79047c3 --- /dev/null +++ b/searchcore/src/tests/proton/server/vespa_testrunner.cpp @@ -0,0 +1,8 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Unit tests for predicate_index. +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/log/log.h> +LOG_SETUP("proton_common_test"); + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp index 2ab81058e43..6d1d80b495d 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> #include <vespa/searchlib/engine/searchreply.h> @@ -7,6 +6,8 @@ #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/util/compressor.h> #include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp index 9ece37665cd..18c942b80a0 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp @@ -4,6 +4,7 @@ #include "groupingcontext.h" #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/expression/attributenode.h> +#include <vespa/searchlib/aggregation/modifiers.h> #include <vespa/vespalib/util/issue.h> #include <vespa/vespalib/util/stringfmt.h> @@ -30,7 +31,7 @@ bool GroupingManager::empty() const { } void -GroupingManager::init(const IAttributeContext &attrCtx) +GroupingManager::init(const IAttributeContext &attrCtx, const document::DocumentType * documentType) { GroupingContext::GroupingList list; GroupingContext::GroupingList &groupingList(_groupingContext.getGroupingList()); @@ -48,7 +49,8 @@ GroupingManager::init(const IAttributeContext &attrCtx) an.enableEnumOptimization(true); } } - ConfigureStaticParams stuff(&attrCtx, nullptr); + aggregation::NonAttribute2DocumentAccessor nonAttributes2DocumentAccess(attrCtx); + ConfigureStaticParams stuff(&attrCtx, documentType); grouping.configureStaticStuff(stuff); list.push_back(groupingList[i]); } catch (const std::exception & e) { diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h index b0856ae7189..99281549a8f 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h @@ -8,6 +8,7 @@ namespace search { struct RankedHit; class BitVector; } +namespace document { class DocumentType; } namespace search::grouping { @@ -43,7 +44,7 @@ public: * * @param attrCtx attribute context **/ - void init(const attribute::IAttributeContext &attrCtx); + void init(const attribute::IAttributeContext &attrCtx, const document::DocumentType * documentType); /** * Perform actual grouping on the given results. diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp index e3f8919e747..8d845766966 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp @@ -16,19 +16,21 @@ using search::attribute::IAttributeContext; GroupingSession::GroupingSession(const SessionId &sessionId, GroupingContext & groupingContext, - const IAttributeContext &attrCtx) + const IAttributeContext &attrCtx, + const document::DocumentType * documentType) : _sessionId(sessionId), _mgrContext(std::make_unique<GroupingContext>(groupingContext)), _groupingManager(std::make_unique<GroupingManager>(*_mgrContext)), _timeOfDoom(groupingContext.getTimeOfDoom()) { - init(groupingContext, attrCtx); + init(groupingContext, attrCtx, documentType); } GroupingSession::~GroupingSession() = default; void -GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext &attrCtx) +GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext &attrCtx, + const document::DocumentType * documentType) { GroupingList & sessionList(groupingContext.getGroupingList()); for (auto g : sessionList) { @@ -41,7 +43,7 @@ GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext } _mgrContext->addGrouping(std::move(g)); } - _groupingManager->init(attrCtx); + _groupingManager->init(attrCtx, documentType); } void @@ -53,7 +55,8 @@ GroupingSession::prepareThreadContextCreation(size_t num_threads) } GroupingContext::UP -GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext &attrCtx) +GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext &attrCtx, + const document::DocumentType * documentType) { auto ctx = std::make_unique<GroupingContext>(*_mgrContext); if (thread_id == 0) { @@ -63,7 +66,7 @@ GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext & } else { ctx->deserialize(_mgrContext->getResult().peek(), _mgrContext->getResult().size()); GroupingManager man(*ctx); - man.init(attrCtx); + man.init(attrCtx, documentType); } return ctx; } diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.h b/searchcore/src/vespa/searchcore/grouping/groupingsession.h index e3e600161d0..d24f0b4ef3d 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingsession.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.h @@ -7,6 +7,7 @@ #include <vector> #include <map> +namespace document { class DocumentType; } namespace search::aggregation { class Grouping; } namespace search::grouping { @@ -44,7 +45,8 @@ public: **/ GroupingSession(const SessionId & sessionId, GroupingContext & groupingContext, - const attribute::IAttributeContext &attrCtx); + const attribute::IAttributeContext &attrCtx, + const document::DocumentType * documentType); GroupingSession(const GroupingSession &) = delete; GroupingSession &operator=(const GroupingSession &) = delete; @@ -63,7 +65,7 @@ public: * @param groupingContext The current grouping context. * @param attrCtx attribute context. **/ - void init(GroupingContext & groupingContext, const attribute::IAttributeContext &attrCtx); + void init(GroupingContext & groupingContext, const attribute::IAttributeContext &attrCtx, const document::DocumentType * documentType); /** * This function is called to prepare for creation of individual @@ -85,7 +87,8 @@ public: * @param thread_id thread id * @param attrCtx attribute context. **/ - std::unique_ptr<GroupingContext> createThreadContext(size_t thread_id, const attribute::IAttributeContext &attrCtx); + std::unique_ptr<GroupingContext> createThreadContext(size_t thread_id, const attribute::IAttributeContext &attrCtx, + const document::DocumentType * documentType); /** * Return the GroupingManager to use when performing grouping. diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp index 3337c1adb9b..1aad609963d 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp @@ -16,12 +16,6 @@ using namespace search::docsummary; namespace proton { -namespace { - -const vespalib::string DOCUMENT_ID_FIELD("documentid"); - -} - DocumentStoreAdapter:: DocumentStoreAdapter(const search::IDocumentStore & docStore, const DocumentTypeRepo &repo) diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h index 308a85baf96..757ce863245 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h @@ -16,7 +16,7 @@ private: public: DocumentStoreAdapter(const search::IDocumentStore &docStore, const document::DocumentTypeRepo &repo); - ~DocumentStoreAdapter(); + ~DocumentStoreAdapter() override; std::unique_ptr<const search::docsummary::IDocsumStoreDocument> get_document(uint32_t docId) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp index e96e231d46a..623a17b89f4 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp @@ -74,7 +74,7 @@ ResultProcessor::ResultProcessor(IAttributeContext &attrContext, _wasMerged(false) { if (!_groupingContext.empty()) { - _groupingSession = std::make_unique<GroupingSession>(sessionId, _groupingContext, attrContext); + _groupingSession = std::make_unique<GroupingSession>(sessionId, _groupingContext, attrContext, nullptr); } } @@ -98,7 +98,7 @@ ResultProcessor::createThreadContext(const vespalib::Doom & hardDoom, size_t thr auto result = std::make_unique<PartialResult>((_offset + _hits), sort->hasSortData()); search::grouping::GroupingContext::UP groupingContext; if (_groupingSession) { - groupingContext = _groupingSession->createThreadContext(thread_id, _attrContext); + groupingContext = _groupingSession->createThreadContext(thread_id, _attrContext, nullptr); } return std::make_unique<Context>(_metaStore.getValidLids(), std::move(sort), std::move(result), std::move(groupingContext)); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h index e11e09ef338..1cf6284295f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h @@ -41,7 +41,7 @@ public: Sort(const Sort &) = delete; Sort & operator = (const Sort &) = delete; Sort(uint32_t partitionId, const vespalib::Doom & doom, IAttributeContext &ac, const vespalib::string &ss); - bool hasSortData() const { + bool hasSortData() const noexcept { return (sorter == (const FastS_IResultSorter *) &sortSpec); } }; @@ -51,7 +51,7 @@ public: **/ struct GroupingSource : vespalib::DualMergeDirector::Source { GroupingContext *ctx; - GroupingSource(GroupingContext *g) : ctx(g) {} + explicit GroupingSource(GroupingContext *g) noexcept : ctx(g) {} void merge(Source &s) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h index afeacfe6a4f..a95c7b7a8a8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h +++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h @@ -46,7 +46,7 @@ private: std::unique_ptr<SearchSessionCache> _search_map; public: - SessionManager(uint32_t maxSizeGrouping); + explicit SessionManager(uint32_t maxSizeGrouping); ~SessionManager(); void insert(search::grouping::GroupingSession::UP session); diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h index 5ac170cd6f3..a98e98bc66a 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h @@ -14,7 +14,7 @@ class ImportedAttributesRepo; * Interface used by a given document db to resolve all references to parent document dbs. */ struct IDocumentDBReferenceResolver { - virtual ~IDocumentDBReferenceResolver() {} + virtual ~IDocumentDBReferenceResolver() = default; virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, const search::IAttributeManager &oldAttrMgr, const std::shared_ptr<search::IDocumentMetaStoreContext> &documentMetaStore, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index b073fb2133d..3fc3202ebd7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -202,9 +202,9 @@ fillInPositionFields(Document &doc, DocumentIdT lid, const DocumentRetriever::Po class PopulateVisitor : public search::IDocumentVisitor { public: - PopulateVisitor(const DocumentRetriever & retriever, search::IDocumentVisitor & visitor) : - _retriever(retriever), - _visitor(visitor) + PopulateVisitor(const DocumentRetriever & retriever, search::IDocumentVisitor & visitor) noexcept + : _retriever(retriever), + _visitor(visitor) { } void visit(uint32_t lid, document::Document::UP doc) override { if (doc) { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 4bf21b814fe..78101ca6afe 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -108,7 +108,7 @@ DocumentSubDBCollection::~DocumentSubDBCollection() void DocumentSubDBCollection::createRetrievers() { - RetrieversSP retrievers(std::make_shared<std::vector<IDocumentRetriever::SP>>()); + RetrieversSP retrievers(std::make_shared<std::vector<std::shared_ptr<IDocumentRetriever>>>()); retrievers->reserve(_subDBs.size()); for (auto subDb : _subDBs) { retrievers->emplace_back(subDb->getDocumentRetriever()); diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp index 27913eca62b..02fd222b9ad 100644 --- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp @@ -18,9 +18,6 @@ using namespace vespalib::slime; namespace proton { -EmptySearchView::EmptySearchView() = default; - - DocsumReply::UP EmptySearchView::getDocsums(const DocsumRequest &req) { diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h index 8dc5f10b717..cda593a9600 100644 --- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h @@ -11,12 +11,9 @@ class EmptySearchView : public ISearchHandler public: using SP = std::shared_ptr<EmptySearchView>; - EmptySearchView(); - std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override; - - std::unique_ptr<SearchReply> - match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override; + std::unique_ptr<SearchReply> match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override; +private: }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 4972cc790c5..1e96f604530 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -201,8 +201,7 @@ FastAccessDocSubDB::FastAccessDocSubDB(const Config &cfg, const Context &ctx) _fastAccessAttributesOnly(cfg._fastAccessAttributesOnly), _initAttrMgr(), _fastAccessFeedView(), - _configurer(_fastAccessFeedView, - getSubDbName()), + _configurer(_fastAccessFeedView, getSubDbName()), _subAttributeMetrics(ctx._subAttributeMetrics), _addMetrics(cfg._addMetrics), _metricsWireService(ctx._metricsWireService), @@ -306,12 +305,12 @@ FastAccessDocSubDB::getAttributeManager() const return extractAttributeManager(_fastAccessFeedView.get()); } -IDocumentRetriever::UP +std::shared_ptr<IDocumentRetriever> FastAccessDocSubDB::getDocumentRetriever() { FastAccessFeedView::SP feedView = _fastAccessFeedView.get(); proton::IAttributeManager::SP attrMgr = extractAttributeManager(feedView); - return std::make_unique<FastAccessDocumentRetriever>(feedView, attrMgr); + return std::make_shared<FastAccessDocumentRetriever>(feedView, attrMgr); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index 4337185a8d3..692667ff2b4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -104,7 +104,7 @@ protected: public: FastAccessDocSubDB(const Config &cfg, const Context &ctx); - ~FastAccessDocSubDB(); + ~FastAccessDocSubDB() override; std::unique_ptr<DocumentSubDbInitializer> createInitializer(const DocumentDBConfig &configSnapshot, SerialNum configSerialNum, @@ -121,7 +121,7 @@ public: std::shared_ptr<IAttributeWriter> get_attribute_writer() const override; std::shared_ptr<IAttributeManager> getAttributeManager() const override; - IDocumentRetriever::UP getDocumentRetriever() override; + std::shared_ptr<IDocumentRetriever> getDocumentRetriever() override; void onReplayDone() override; void onReprocessDone(SerialNum serialNum) override; SerialNum getOldestFlushedSerial() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.cpp index c2a190cc494..323b91d017c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.cpp @@ -4,7 +4,7 @@ namespace proton { -FastAccessDocumentRetriever::FastAccessDocumentRetriever(FastAccessFeedView::SP feedView, IAttributeManager::SP attrMgr) +FastAccessDocumentRetriever::FastAccessDocumentRetriever(FastAccessFeedView::SP feedView, IAttributeManager::SP attrMgr) noexcept : DocumentRetriever(feedView->getPersistentParams()._docTypeName, *feedView->getDocumentTypeRepo(), *feedView->getSchema(), diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h index 6a9a385ce5a..83884f8f327 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h @@ -20,7 +20,7 @@ private: IAttributeManager::SP _attrMgr; public: - FastAccessDocumentRetriever(FastAccessFeedView::SP feedView, IAttributeManager::SP attrMgr); + FastAccessDocumentRetriever(FastAccessFeedView::SP feedView, IAttributeManager::SP attrMgr) noexcept; ~FastAccessDocumentRetriever() override; uint32_t getDocIdLimit() const override { return _feedView->getDocIdLimit().get(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 60e844dad60..2249cb5392a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -68,8 +68,8 @@ public: using OnDone = std::shared_ptr<vespalib::IDestructorCallback>; using SessionManager = matching::SessionManager; public: - IDocumentSubDB() { } - virtual ~IDocumentSubDB() { } + IDocumentSubDB() noexcept { } + virtual ~IDocumentSubDB() = default; virtual uint32_t getSubDbId() const = 0; virtual vespalib::string getName() const = 0; @@ -124,7 +124,7 @@ public: virtual void pruneRemovedFields(SerialNum serialNum) = 0; virtual void setIndexSchema(const SchemaSP &schema, SerialNum serialNum) = 0; virtual search::SearchableStats getSearchableStats() const = 0; - virtual std::unique_ptr<IDocumentRetriever> getDocumentRetriever() = 0; + virtual std::shared_ptr<IDocumentRetriever> getDocumentRetriever() = 0; virtual matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const = 0; virtual void close() = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 5e9d9585ca9..fd62baa5bcb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -162,8 +162,8 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const if (initializer && initializer->hasReprocessors()) { tasks.emplace_back(createReprocessingTask(*initializer, newConfigSnapshot.getDocumentTypeRepoSP())); } - proton::IAttributeManager::SP newMgr = getAttributeManager(); if (_addMetrics) { + proton::IAttributeManager::SP newMgr = getAttributeManager(); reconfigureAttributeMetrics(*newMgr, *oldMgr); } } else { @@ -309,10 +309,10 @@ SearchableDocSubDB::getSearchableStats() const return _indexMgr ? _indexMgr->getSearchableStats() : search::SearchableStats(); } -IDocumentRetriever::UP +std::shared_ptr<IDocumentRetriever> SearchableDocSubDB::getDocumentRetriever() { - return std::make_unique<FastAccessDocumentRetriever>(_rFeedView.get(), _rSearchView.get()->getAttributeManager()); + return std::make_shared<FastAccessDocumentRetriever>(_rFeedView.get(), _rSearchView.get()->getAttributeManager()); } MatchingStats diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index eb27406e228..01b78c6bb4d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -129,7 +129,7 @@ public: void setIndexSchema(const Schema::SP &schema, SerialNum serialNum) override; size_t getNumActiveDocs() const override; search::SearchableStats getSearchableStats() const override ; - IDocumentRetriever::UP getDocumentRetriever() override; + std::shared_ptr<IDocumentRetriever> getDocumentRetriever() override; matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const override; void close() override; std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h index 41b84edc887..15990f8a55d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h @@ -15,9 +15,9 @@ private: std::shared_ptr<DocumentDB> _documentDB; vespalib::RetainGuard _retainGuard; public: - SearchHandlerProxy(std::shared_ptr<DocumentDB> documentDB); - + explicit SearchHandlerProxy(std::shared_ptr<DocumentDB> documentDB); ~SearchHandlerProxy() override; + std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & request) override; std::unique_ptr<SearchReply> match(const SearchRequest &req, ThreadBundle &threadBundle) const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 0c217eeeeba..0e383c47e9b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -548,10 +548,10 @@ StoreOnlyDocSubDB::getSearchableStats() const return {}; } -IDocumentRetriever::UP +std::shared_ptr<IDocumentRetriever> StoreOnlyDocSubDB::getDocumentRetriever() { - return std::make_unique<MinimalDocumentRetriever>(_docTypeName, _iFeedView.get()->getDocumentTypeRepo(), + return std::make_shared<MinimalDocumentRetriever>(_docTypeName, _iFeedView.get()->getDocumentTypeRepo(), *_metaStoreCtx, _iSummaryMgr->getBackingStore(), _subDbType != SubDbType::REMOVED); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index d530e44755c..db7b59611d6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -232,7 +232,7 @@ public: void pruneRemovedFields(SerialNum serialNum) override; void setIndexSchema(const Schema::SP &schema, SerialNum serialNum) override; search::SearchableStats getSearchableStats() const override; - IDocumentRetriever::UP getDocumentRetriever() override; + std::shared_ptr<IDocumentRetriever> getDocumentRetriever() override; matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const override; void close() override; std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override; diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt index 5474f1ac157..b7cd9d35b1e 100644 --- a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt @@ -8,6 +8,7 @@ vespa_add_library(searchcore_test STATIC bucketstatecalculator.cpp clusterstatehandler.cpp documentdb_config_builder.cpp + dummy_document_sub_db.cpp dummy_feed_view.cpp dummy_flush_target.cpp dummydbowner.cpp diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.cpp b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.cpp new file mode 100644 index 00000000000..f70bdb0bff6 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.cpp @@ -0,0 +1,18 @@ +#include "dummy_document_sub_db.h" + +namespace proton::test { + +DummyDocumentSubDb::DummyDocumentSubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, uint32_t subDbId) + : _subDbId(subDbId), + _metaStoreCtx(std::move(bucketDB)), + _summaryManager(), + _indexManager(), + _summaryAdapter(), + _indexWriter(), + _service(1), + _pendingLidTracker() +{ } + +DummyDocumentSubDb::~DummyDocumentSubDb() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 7d6ea39a65a..35ada75b001 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -30,18 +30,8 @@ struct DummyDocumentSubDb : public IDocumentSubDB mutable TransportAndExecutorService _service; PendingLidTracker _pendingLidTracker; - DummyDocumentSubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, uint32_t subDbId) - : _subDbId(subDbId), - _metaStoreCtx(std::move(bucketDB)), - _summaryManager(), - _indexManager(), - _summaryAdapter(), - _indexWriter(), - _service(1), - _pendingLidTracker() - { - } - ~DummyDocumentSubDb() override { } + DummyDocumentSubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, uint32_t subDbId); + ~DummyDocumentSubDb() override; void close() override { } uint32_t getSubDbId() const override { return _subDbId; } vespalib::string getName() const override { return "dummysubdb"; } @@ -59,11 +49,11 @@ struct DummyDocumentSubDb : public IDocumentSubDB IReprocessingTask::List applyConfig(const DocumentDBConfig &, const DocumentDBConfig &, SerialNum, const ReconfigParams &, IDocumentDBReferenceResolver &, const DocumentSubDBReconfig&) override { - return IReprocessingTask::List(); + return {}; } void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &, OnDone) override { } - ISearchHandler::SP getSearchView() const override { return ISearchHandler::SP(); } - IFeedView::SP getFeedView() const override { return IFeedView::SP(); } + ISearchHandler::SP getSearchView() const override { return {}; } + IFeedView::SP getFeedView() const override { return {}; } void clearViews() override {} const ISummaryManager::SP &getSummaryManager() const override { return _summaryManager; } std::shared_ptr<IAttributeWriter> get_attribute_writer() const override { return {}; } @@ -74,7 +64,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB const IIndexWriter::SP &getIndexWriter() const override { return _indexWriter; } IDocumentMetaStoreContext &getDocumentMetaStoreContext() override { return _metaStoreCtx; } const IDocumentMetaStoreContext &getDocumentMetaStoreContext() const override { return _metaStoreCtx; } - IFlushTargetList getFlushTargets() override { return IFlushTargetList(); } + IFlushTargetList getFlushTargets() override { return {}; } size_t getNumDocs() const override { return 0; } size_t getNumActiveDocs() const override { return 0; } bool hasDocument(const document::DocumentId &) override { return false; } @@ -85,16 +75,16 @@ struct DummyDocumentSubDb : public IDocumentSubDB void pruneRemovedFields(SerialNum) override { } void setIndexSchema(const Schema::SP &, SerialNum) override { } search::SearchableStats getSearchableStats() const override { - return search::SearchableStats(); + return {}; } - IDocumentRetriever::UP getDocumentRetriever() override { - return IDocumentRetriever::UP(); + std::shared_ptr<IDocumentRetriever> getDocumentRetriever() override { + return {}; } matching::MatchingStats getMatcherStats(const vespalib::string &) const override { - return matching::MatchingStats(); + return {}; } std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override { - return std::shared_ptr<IDocumentDBReference>(); + return {}; } PendingLidTrackerBase &getUncommittedLidsTracker() override { diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.cpp index a4f5fea1750..b9ed2cc10f2 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mock_gid_to_lid_change_handler.h" +#include <vespa/vespalib/testkit/test_master.hpp> namespace proton::test { @@ -14,4 +15,27 @@ MockGidToLidChangeHandler::MockGidToLidChangeHandler() noexcept MockGidToLidChangeHandler::~MockGidToLidChangeHandler() = default; +void +MockGidToLidChangeHandler::addListener(std::unique_ptr<IGidToLidChangeListener> listener) { + _adds.emplace_back(listener->getDocTypeName(), listener->getName()); + _listeners.push_back(std::move(listener)); +} + +void +MockGidToLidChangeHandler::removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) { + _removes.emplace_back(docTypeName, keepNames); +} + +void +MockGidToLidChangeHandler::assertAdds(const std::vector<AddEntry> &expAdds) const +{ + EXPECT_EQUAL(expAdds, _adds); +} + +void +MockGidToLidChangeHandler::assertRemoves(const std::vector<RemoveEntry> &expRemoves) const +{ + EXPECT_EQUAL(expRemoves, _removes); +} + } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h index 288f6ebfebd..1fd27e07f49 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h @@ -4,9 +4,9 @@ #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> #include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> +#include <vespa/document/base/globalid.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/document/base/globalid.h> namespace proton::test { @@ -28,28 +28,15 @@ public: MockGidToLidChangeHandler() noexcept; ~MockGidToLidChangeHandler() override; - void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override { - _adds.emplace_back(listener->getDocTypeName(), listener->getName()); - _listeners.push_back(std::move(listener)); - } - - void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) override { - _removes.emplace_back(docTypeName, keepNames); - } + void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override; + void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) override; void notifyPut(IDestructorCallbackSP, document::GlobalId, uint32_t, SerialNum) override { } void notifyRemoves(IDestructorCallbackSP, const std::vector<document::GlobalId> &, SerialNum) override { } std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() override { return {}; } - void assertAdds(const std::vector<AddEntry> &expAdds) - { - EXPECT_EQUAL(expAdds, _adds); - } - - void assertRemoves(const std::vector<RemoveEntry> &expRemoves) - { - EXPECT_EQUAL(expRemoves, _removes); - } + void assertAdds(const std::vector<AddEntry> &expAdds) const; + void assertRemoves(const std::vector<RemoveEntry> &expRemoves) const; const std::vector<std::unique_ptr<IGidToLidChangeListener>> &getListeners() const { return _listeners; } }; diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index b9c80a3c04d..5eecb2b79c4 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -21,11 +21,12 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/vespalib/util/rand48.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("memoryindexstress_test"); diff --git a/searchlib/src/apps/vespa-query-analyzer/vespa-query-analyzer.cpp b/searchlib/src/apps/vespa-query-analyzer/vespa-query-analyzer.cpp index 6a746d59daa..dfe7a57ceba 100644 --- a/searchlib/src/apps/vespa-query-analyzer/vespa-query-analyzer.cpp +++ b/searchlib/src/apps/vespa-query-analyzer/vespa-query-analyzer.cpp @@ -13,6 +13,7 @@ #include <variant> #include <vector> #include <map> +#include <unistd.h> using namespace vespalib::slime::convenience; using vespalib::make_string_short::fmt; diff --git a/searchlib/src/tests/aggregator/attr_test.cpp b/searchlib/src/tests/aggregator/attr_test.cpp index 2d2e798817a..bd3e756ea6e 100644 --- a/searchlib/src/tests/aggregator/attr_test.cpp +++ b/searchlib/src/tests/aggregator/attr_test.cpp @@ -1,6 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> - #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/extendableattributes.h> @@ -8,6 +6,8 @@ #include <vespa/searchlib/expression/arrayatlookupfunctionnode.h> #include <vespa/searchlib/expression/interpolatedlookupfunctionnode.h> +#include <vespa/vespalib/testkit/test_kit.h> + #include <vespa/log/log.h> LOG_SETUP("attr_test"); @@ -147,7 +147,7 @@ TEST_F("testArrayAtInt", IntAttrFixture()) { EXPECT_TRUE(et.execute(0, HitRank(0.0))); EXPECT_EQUAL(et.getResult()->getInteger(), f1.doc0attr[i]); EXPECT_TRUE(et.execute(1, HitRank(0.0))); - EXPECT_EQUAL(et.getResult()->getInteger(), f1.doc1attr[i]); + EXPECT_EQUAL(static_cast<double>(et.getResult()->getInteger()), f1.doc1attr[i]); } } diff --git a/searchlib/src/tests/aggregator/perdocexpr_test.cpp b/searchlib/src/tests/aggregator/perdocexpr_test.cpp index 61c5a4f8de9..e22b0561b0b 100644 --- a/searchlib/src/tests/aggregator/perdocexpr_test.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr_test.cpp @@ -7,7 +7,6 @@ #include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/objects/objectdumper.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> @@ -17,6 +16,8 @@ #include <cmath> #include <iostream> #include <list> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("per_doc_expr_test"); diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 2494b514c56..57b88692e12 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("attribute_test"); diff --git a/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp b/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp index 1d66eefaff7..33d36d03351 100644 --- a/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp +++ b/searchlib/src/tests/attribute/bitvector_search_cache/bitvector_search_cache_test.cpp @@ -1,9 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/bitvector_search_cache.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/vespalib/util/memoryusage.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search; using namespace search::attribute; diff --git a/searchlib/src/tests/attribute/changevector/changevector_test.cpp b/searchlib/src/tests/attribute/changevector/changevector_test.cpp index 9dd96b1b72c..0d3db6cca74 100644 --- a/searchlib/src/tests/attribute/changevector/changevector_test.cpp +++ b/searchlib/src/tests/attribute/changevector/changevector_test.cpp @@ -1,8 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> - #include <vespa/searchlib/attribute/changevector.hpp> #include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> + using namespace search; using Change = ChangeTemplate<NumericChangeData<long>>; diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index 53fe3c4046a..2359a0286d3 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/tensor/i_tensor_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/test/imported_attribute_fixture.h> +#include <vespa/vespalib/testkit/test_master.hpp> using search::attribute::IAttributeVector; using search::tensor::ITensorAttribute; @@ -191,8 +192,8 @@ TEST_F("Singled-valued floating point attribute values can be retrieved via refe {{DocId(2), dummy_gid(3), DocId(3), 10.5f}, {DocId(4), dummy_gid(8), DocId(8), 3.14f}}); - EXPECT_EQUAL(10.5f, f.get_imported_attr()->getFloat(DocId(2))); - EXPECT_EQUAL(3.14f, f.get_imported_attr()->getFloat(DocId(4))); + EXPECT_EQUAL(10.5, f.get_imported_attr()->getFloat(DocId(2))); + EXPECT_EQUAL(3.14, f.get_imported_attr()->getFloat(DocId(4))); } TEST_F("Multi-valued floating point attribute values can be retrieved via reference", Fixture) { diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index 7c38c322bc8..7d02949c994 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -7,8 +7,9 @@ #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/test/imported_attribute_fixture.h> #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> -#include <vespa/vespalib/test/insertion_operators.h> #include <vespa/searchlib/queryeval/executeinfo.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_master.hpp> namespace search::attribute { diff --git a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp index 868c85c7962..22170461f25 100644 --- a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp +++ b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp @@ -1,10 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/posting_list_merger.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/size_literals.h> #include <algorithm> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using vespalib::btree::BTreeNoLeafData; using search::attribute::PostingListMerger; diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index 0278c4f32ef..ea18960c8ff 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/attribute_blueprint_factory.h> #include <vespa/searchlib/attribute/attribute_weighted_set_blueprint.h> #include <vespa/searchlib/attribute/attributecontext.h> @@ -16,6 +15,8 @@ #include <vespa/searchlib/attribute/enumstore.hpp> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/util/normalize_class_name.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("attribute_weighted_set_blueprint_test"); diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index cce72837dad..4e6e01f11da 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -21,7 +21,6 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/threadstackexecutor.h> @@ -34,6 +33,8 @@ #include <vespa/eval/eval/test/value_compare.h> #include <vespa/fastos/file.h> #include <filesystem> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("tensorattribute_test"); diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index ef9e801cda3..04c0c082a69 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/searchlib/common/growablebitvector.h> #include <vespa/searchlib/common/partialbitvector.h> @@ -13,6 +12,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/simple_thread_bundle.h> #include <algorithm> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search; diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index a35aeb5e851..5dcab0b8810 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -1,10 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/searchlib/docstore/value.h> #include <vespa/vespalib/stllike/cache_stats.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search; using CompressionConfig = vespalib::compression::CompressionConfig; diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 8f506b7ca2d..8ccd4e57eed 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -5,12 +5,12 @@ #include <vespa/searchlib/docstore/writeablefilechunk.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <iomanip> -#include <iostream> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 72bdd533719..79fbefa03a3 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> @@ -22,6 +21,8 @@ #include <filesystem> #include <iomanip> #include <random> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using document::BucketId; using document::StringFieldValue; diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index d2b07c6c552..3941cc606fb 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -15,6 +15,7 @@ #include <vespa/searchlib/test/make_attribute_map_lookup_node.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("attribute_node_test"); diff --git a/searchlib/src/tests/features/bm25/bm25_test.cpp b/searchlib/src/tests/features/bm25/bm25_test.cpp index f175d21c001..85514a9ced7 100644 --- a/searchlib/src/tests/features/bm25/bm25_test.cpp +++ b/searchlib/src/tests/features/bm25/bm25_test.cpp @@ -175,7 +175,7 @@ struct Bm25ExecutorTest : public ::testing::Test { } double idf(uint32_t matching_doc_count) const { - return Bm25Executor::calculate_inverse_document_frequency(matching_doc_count, total_doc_count); + return Bm25Executor::calculate_inverse_document_frequency({matching_doc_count, total_doc_count}); } feature_t score(feature_t num_occs, feature_t field_length, double inverse_doc_freq) const { @@ -226,11 +226,17 @@ TEST_F(Bm25ExecutorTest, uses_average_field_length_from_shared_state_if_found) TEST_F(Bm25ExecutorTest, calculates_inverse_document_frequency) { EXPECT_DOUBLE_EQ(std::log(1 + (99 + 0.5) / (1 + 0.5)), - Bm25Executor::calculate_inverse_document_frequency(1, 100)); + Bm25Executor::calculate_inverse_document_frequency({1, 100})); EXPECT_DOUBLE_EQ(std::log(1 + (60 + 0.5) / (40 + 0.5)), - Bm25Executor::calculate_inverse_document_frequency(40, 100)); + Bm25Executor::calculate_inverse_document_frequency({40, 100})); EXPECT_DOUBLE_EQ(std::log(1 + (0.5) / (100 + 0.5)), - Bm25Executor::calculate_inverse_document_frequency(100, 100)); + Bm25Executor::calculate_inverse_document_frequency({100, 100})); + EXPECT_DOUBLE_EQ(std::log(1 + (0.5) / (100 + 0.5)), + Bm25Executor::calculate_inverse_document_frequency({200, 100})); + EXPECT_DOUBLE_EQ(std::log(1 + (99 + 0.5) / (1 + 0.5)), + Bm25Executor::calculate_inverse_document_frequency({0, 100})); + EXPECT_DOUBLE_EQ(std::log(1 + (0.5) / (1 + 0.5)), + Bm25Executor::calculate_inverse_document_frequency({0, 0})); } TEST_F(Bm25ExecutorTest, k1_param_can_be_overriden) diff --git a/searchlib/src/tests/features/constant/constant_test.cpp b/searchlib/src/tests/features/constant/constant_test.cpp index 3377965536b..ae9c698372b 100644 --- a/searchlib/src/tests/features/constant/constant_test.cpp +++ b/searchlib/src/tests/features/constant/constant_test.cpp @@ -1,7 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <iostream> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/ftlib.h> @@ -13,6 +11,8 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using search::feature_t; using namespace search::fef; @@ -28,8 +28,7 @@ using vespalib::eval::Value; using vespalib::eval::ValueType; using vespalib::make_string_short::fmt; -namespace -{ +namespace { Value::UP make_tensor(const TensorSpec &spec) { return SimpleValue::from_spec(spec); diff --git a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp index 735ec5d67b7..45b7f089445 100644 --- a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp +++ b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> @@ -9,6 +8,8 @@ #include <vespa/searchlib/query/weight.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using search::feature_t; using namespace search::fef; diff --git a/searchlib/src/tests/features/prod_features_test.cpp b/searchlib/src/tests/features/prod_features_test.cpp index fb00b4ff5e6..729ee3fca98 100644 --- a/searchlib/src/tests/features/prod_features_test.cpp +++ b/searchlib/src/tests/features/prod_features_test.cpp @@ -2121,10 +2121,10 @@ TEST_F(ProdFeaturesTest, test_term) ASSERT_TRUE(ft.setup()); RankResult exp; - exp.addScore("term(1).significance", util::getSignificance(0.50)). + exp.addScore("term(1).significance", util::calculate_legacy_significance({50, 100})). addScore("term(1).weight", 200.0f). addScore("term(1).connectedness", 0.7f). - addScore("term(2).significance", util::getSignificance(0.25)). + addScore("term(2).significance", util::calculate_legacy_significance({25, 100})). addScore("term(2).weight", 400.0f). addScore("term(2).connectedness", 0.1f). // default connectedness setEpsilon(10e-6); @@ -2215,21 +2215,22 @@ Test::assertTermDistance(const TermDistanceCalculator::Result & exp, TEST_F(ProdFeaturesTest, test_utils) { - { // getSignificance - EXPECT_NEAR(util::getSignificance(0.0), 1, EPS); - EXPECT_NEAR(util::getSignificance(0.0 + 1.0e-7), 1, EPS); - EXPECT_NEAR(util::getSignificance(1.0), 0.5, EPS); - EXPECT_NEAR(util::getSignificance(1.0 + 1.0e-7), 0.5, EPS); + { // calculate_legacy_significance + constexpr uint64_t N = 1000000; // The "normal" corpus size for legacy significance + EXPECT_NEAR(util::calculate_legacy_significance({0, N}), 1, EPS); + EXPECT_NEAR(util::calculate_legacy_significance({1, N}), 1, EPS); + EXPECT_NEAR(util::calculate_legacy_significance({ N, N}), 0.5, EPS); + EXPECT_NEAR(util::calculate_legacy_significance({ N + 1, N}), 0.5, EPS); feature_t last = 1; for (uint32_t i = 2; i <= 100; i = i + 1) { - feature_t s = util::getSignificance(i * 1.0e-6); + feature_t s = util::calculate_legacy_significance({i, N}); EXPECT_GT(s, 0); EXPECT_LT(s, 1); EXPECT_LT(s, last); last = s; } for (uint32_t i = 999900; i <= 1000000; i = i + 1) { - feature_t s = util::getSignificance(i * 1.0e-6); + feature_t s = util::calculate_legacy_significance({i, N}); EXPECT_GT(s, 0); EXPECT_LT(s, 1); EXPECT_LT(s, last); diff --git a/searchlib/src/tests/features/subqueries/subqueries_test.cpp b/searchlib/src/tests/features/subqueries/subqueries_test.cpp index 6aaa5f3a4a6..8be9b0d969f 100644 --- a/searchlib/src/tests/features/subqueries/subqueries_test.cpp +++ b/searchlib/src/tests/features/subqueries/subqueries_test.cpp @@ -127,32 +127,32 @@ TEST_FF("require that setup fails for unknown field", TEST_F("require that not searching a field will give it 0 subqueries", RankFixture(0, 3)) { - EXPECT_EQUAL(0, f1.getSubqueries(10)); + EXPECT_EQUAL(0.0, f1.getSubqueries(10)); } TEST_F("require that subqueries can be obtained", RankFixture(1, 0)) { f1.setFooSubqueries(0, 10, 0x1234); - EXPECT_EQUAL(0x1234, f1.getSubqueries(10)); + EXPECT_EQUAL(static_cast<double>(0x1234), f1.getSubqueries(10)); } TEST_F("require that msb subqueries can be obtained", RankFixture(1, 0, "subqueries(foo).msb")) { f1.setFooSubqueries(0, 10, 0x123412345678ULL); - EXPECT_EQUAL(0x1234, f1.getSubqueries(10)); + EXPECT_EQUAL(static_cast<double>(0x1234), f1.getSubqueries(10)); } TEST_F("require that multiple subqueries are accumulated", RankFixture(3, 0)) { f1.setFooSubqueries(0, 10, 1); f1.setFooSubqueries(1, 10, 2); f1.setFooSubqueries(2, 10, 4); - EXPECT_EQUAL(7, f1.getSubqueries(10)); + EXPECT_EQUAL(7.0, f1.getSubqueries(10)); } TEST_F("require that stale subqueries are ignored", RankFixture(3, 0)) { f1.setFooSubqueries(0, 10, 1); f1.setFooSubqueries(1, 9, 2); f1.setFooSubqueries(2, 10, 4); - EXPECT_EQUAL(5, f1.getSubqueries(10)); + EXPECT_EQUAL(5.0, f1.getSubqueries(10)); } TEST_F("require that subqueries from other fields are ignored", @@ -161,7 +161,7 @@ TEST_F("require that subqueries from other fields are ignored", f1.setFooSubqueries(1, 10, 2); f1.setBarSubqueries(0, 10, 4); f1.setBarSubqueries(1, 10, 8); - EXPECT_EQUAL(3, f1.getSubqueries(10)); + EXPECT_EQUAL(3.0, f1.getSubqueries(10)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/features/util/CMakeLists.txt b/searchlib/src/tests/features/util/CMakeLists.txt index 0eee4d3b7ac..1315734eee7 100644 --- a/searchlib/src/tests/features/util/CMakeLists.txt +++ b/searchlib/src/tests/features/util/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchlib_util_test_app TEST util_test.cpp DEPENDS vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_util_test_app COMMAND searchlib_util_test_app) diff --git a/searchlib/src/tests/features/util/util_test.cpp b/searchlib/src/tests/features/util/util_test.cpp index 7f3d8ad209f..69baed111f1 100644 --- a/searchlib/src/tests/features/util/util_test.cpp +++ b/searchlib/src/tests/features/util/util_test.cpp @@ -1,8 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/utils.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace search; using namespace search::fef; @@ -10,6 +10,14 @@ using namespace search::fef::test; using namespace search::features; using namespace search::features::util; +namespace search::features::util { + +void PrintTo(const DocumentFrequency& document_frequency, std::ostream* os) { + *os << "{" << document_frequency.frequency << "," << document_frequency.count << "}"; +} + +} + SimpleTermData make_term(uint32_t uid) { SimpleTermData term; term.setUniqueId(uid); @@ -30,31 +38,57 @@ struct TermLabelFixture { } }; -TEST_F("require that label can be mapped to term", TermLabelFixture) { - EXPECT_EQUAL((ITermData*)&f1.queryEnv.getTerms()[0], util::getTermByLabel(f1.queryEnv, "foo")); - EXPECT_EQUAL((ITermData*)0, util::getTermByLabel(f1.queryEnv, "bar")); - EXPECT_EQUAL((ITermData*)&f1.queryEnv.getTerms()[2], util::getTermByLabel(f1.queryEnv, "baz")); - EXPECT_EQUAL((ITermData*)0, util::getTermByLabel(f1.queryEnv, "fox")); - EXPECT_EQUAL((ITermData*)0, util::getTermByLabel(f1.queryEnv, "unknown")); +TEST(UtilsTest, require_that_label_can_be_mapped_to_term) +{ + TermLabelFixture f1; + EXPECT_EQ((ITermData*)&f1.queryEnv.getTerms()[0], getTermByLabel(f1.queryEnv, "foo")); + EXPECT_EQ((ITermData*)0, getTermByLabel(f1.queryEnv, "bar")); + EXPECT_EQ((ITermData*)&f1.queryEnv.getTerms()[2], getTermByLabel(f1.queryEnv, "baz")); + EXPECT_EQ((ITermData*)0, getTermByLabel(f1.queryEnv, "fox")); + EXPECT_EQ((ITermData*)0, getTermByLabel(f1.queryEnv, "unknown")); } template <typename T> -void verifyStrToNum() { - EXPECT_EQUAL(-17, static_cast<long>(strToNum<T>("-17"))); - EXPECT_EQUAL(-1, static_cast<long>(strToNum<T>("-1"))); - EXPECT_EQUAL(0, static_cast<long>(strToNum<T>("0"))); - EXPECT_EQUAL(1, static_cast<long>(strToNum<T>("1"))); - EXPECT_EQUAL(17, static_cast<long>(strToNum<T>("17"))); - EXPECT_EQUAL(0, static_cast<long>(strToNum<T>("0x0"))); - EXPECT_EQUAL(1, static_cast<long>(strToNum<T>("0x1"))); - EXPECT_EQUAL(27, static_cast<long>(strToNum<T>("0x1b"))); +void verifyStrToNum(const std::string& label) { + SCOPED_TRACE(label); + EXPECT_EQ(-17, static_cast<long>(strToNum<T>("-17"))); + EXPECT_EQ(-1, static_cast<long>(strToNum<T>("-1"))); + EXPECT_EQ(0, static_cast<long>(strToNum<T>("0"))); + EXPECT_EQ(1, static_cast<long>(strToNum<T>("1"))); + EXPECT_EQ(17, static_cast<long>(strToNum<T>("17"))); + EXPECT_EQ(0, static_cast<long>(strToNum<T>("0x0"))); + EXPECT_EQ(1, static_cast<long>(strToNum<T>("0x1"))); + EXPECT_EQ(27, static_cast<long>(strToNum<T>("0x1b"))); +} + +TEST(UtilsTest, verify_str2Num) +{ + verifyStrToNum<int8_t>("int8_t"); + verifyStrToNum<int16_t>("int16_t"); + verifyStrToNum<int32_t>("int32_t"); + verifyStrToNum<int64_t>("int64_t"); } -TEST("verify str2Num") { - verifyStrToNum<int8_t>(); - verifyStrToNum<int16_t>(); - verifyStrToNum<int32_t>(); - verifyStrToNum<int64_t>(); +TEST(UtilsTest, lookup_document_frequency) +{ + using OptDF = std::optional<DocumentFrequency>; + IndexEnvironment index_env;; + QueryEnvironment query_env(&index_env); + query_env.getTerms() = std::vector<SimpleTermData>{make_term(0), make_term(5), make_term(6), make_term(10)}; + // Properties not used due to bad unique id + query_env.getProperties().add("vespa.term.0.docfreq", "11"); + query_env.getProperties().add("vespa.term.0.docfreq", "17"); + // Incomplete properties, thus not used + query_env.getProperties().add("vespa.term.6.docfreq", "5"); + // Complete properties + query_env.getProperties().add("vespa.term.10.docfreq", "10"); + query_env.getProperties().add("vespa.term.10.docfreq", "15"); + auto& terms = query_env.getTerms(); + EXPECT_EQ(4, terms.size()); + EXPECT_EQ(OptDF(), lookup_document_frequency(query_env, terms[0])); // bad unique id + EXPECT_EQ(OptDF(), lookup_document_frequency(query_env, terms[1])); // missing properties + EXPECT_EQ(OptDF(), lookup_document_frequency(query_env, terms[2])); // incomplete properties + EXPECT_EQ(OptDF({10, 15}), lookup_document_frequency(query_env, terms[3])); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp index e8eec096cd4..b6a01f23949 100644 --- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -261,14 +261,14 @@ TEST("Access subqueries") { testSetup(state); state.f3->reset(10); state.f3->setSubqueries(10, 42); - EXPECT_EQUAL(42ULL, state.f3->getSubqueries()); + EXPECT_EQUAL(42UL, state.f3->getSubqueries()); state.f3->enableRawScore(); - EXPECT_EQUAL(0ULL, state.f3->getSubqueries()); + EXPECT_EQUAL(0UL, state.f3->getSubqueries()); state.f3->reset(11); state.f3->appendPosition(TermFieldMatchDataPosition()); state.f3->setSubqueries(11, 42); - EXPECT_EQUAL(0ULL, state.f3->getSubqueries()); + EXPECT_EQUAL(0UL, state.f3->getSubqueries()); } TEST("require that TermFieldMatchData can be tagged as needed or not") { diff --git a/searchlib/src/tests/grouping/grouping_test.cpp b/searchlib/src/tests/grouping/grouping_test.cpp index 0caebab6607..952f5d2a5db 100644 --- a/searchlib/src/tests/grouping/grouping_test.cpp +++ b/searchlib/src/tests/grouping/grouping_test.cpp @@ -8,6 +8,8 @@ #include <vespa/searchlib/aggregation/hitsaggregationresult.h> #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/aggregation/predicates.h> +#include <vespa/searchlib/aggregation/modifiers.h> +#include <vespa/searchlib/expression/documentfieldnode.h> #include <vespa/searchlib/expression/fixedwidthbucketfunctionnode.h> #include <vespa/searchlib/test/make_attribute_map_lookup_node.h> #include <vespa/searchcommon/common/undefinedvalues.h> @@ -51,7 +53,7 @@ public: add(val); } } - AttrBuilder(const std::string &name) + explicit AttrBuilder(const std::string &name) : _attr(new A(name)), _attrSP(_attr) { @@ -127,19 +129,19 @@ private: ResultBuilder _result; IAttributeContext::UP _attrCtx; - AggregationContext(const AggregationContext &); - AggregationContext &operator=(const AggregationContext &); - public: AggregationContext(); + AggregationContext(const AggregationContext &) = delete; + AggregationContext &operator=(const AggregationContext &) = delete; ~AggregationContext(); ResultBuilder &result() { return _result; } - void add(AttributeVector::SP attr) { + void add(const AttributeVector::SP & attr) { _attrMan.add(attr); } void setup(Grouping &g) { - g.configureStaticStuff(ConfigureStaticParams(_attrCtx.get(), 0)); + g.configureStaticStuff(ConfigureStaticParams(_attrCtx.get(), nullptr)); } + const IAttributeContext & attrCtx() const { return *_attrCtx; } }; AggregationContext::AggregationContext() : _attrMan(), _result(), _attrCtx(_attrMan.createContext()) {} @@ -688,7 +690,7 @@ TEST("testAggregationGroupCapping") EXPECT_TRUE(testAggregation(ctx, request, expect)); } { - AddFunctionNode *add = new AddFunctionNode(); + auto add = std::make_unique<AddFunctionNode>(); add->addArg(MU<AggregationRefNode>(0)); add->appendArg(MU<ConstantNode>(MU<Int64ResultNode>(3))); @@ -697,7 +699,7 @@ TEST("testAggregationGroupCapping") .setLastLevel(1) .addLevel(std::move(GroupingLevel().setMaxGroups(3).setExpression(MU<AttributeNode>("attr")) .addAggregationResult(createAggr<SumAggregationResult>(MU<AttributeNode>("attr"))) - .addOrderBy(ExpressionNode::UP(add), false))); + .addOrderBy(std::move(add), false))); Group expect; expect.addChild(Group().setId(Int64ResultNode(7)).setRank(RawRank(7)) @@ -1826,4 +1828,31 @@ TEST("testAttributeMapLookup") testAggregationSimple(ctx, MaxAggregationResult(), Int64ResultNode(100), "smap{attribute(key2)}.weight"); } +TEST("test that non-attributes are converted to document field nodes") { + AggregationContext ctx; + ctx.add(IntAttrBuilder("attr").sp()); + + Grouping attrRequest; + attrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("attr")))); + aggregation::NonAttribute2DocumentAccessor optional2DocumentAccessor(ctx.attrCtx()); + attrRequest.select(optional2DocumentAccessor, optional2DocumentAccessor); + EXPECT_TRUE(attrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(AttributeNode::classId)); + + Grouping nonAttrRequest; + nonAttrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("non-attr")))); + nonAttrRequest.select(optional2DocumentAccessor, optional2DocumentAccessor); + EXPECT_TRUE(nonAttrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(DocumentFieldNode::classId)); +} + +TEST("test that attributes can be unconditionally converted to document field nodes") { + AggregationContext ctx; + ctx.add(IntAttrBuilder("attr").sp()); + + Grouping attrRequest; + attrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("attr")))); + aggregation::Attribute2DocumentAccessor attr2DocumentAccessor; + attrRequest.select(attr2DocumentAccessor, attr2DocumentAccessor); + EXPECT_TRUE(attrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(DocumentFieldNode::classId)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/grouping/hyperloglog_test.cpp b/searchlib/src/tests/grouping/hyperloglog_test.cpp index 281bad55735..9df9d4322a1 100644 --- a/searchlib/src/tests/grouping/hyperloglog_test.cpp +++ b/searchlib/src/tests/grouping/hyperloglog_test.cpp @@ -1,13 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for hyperloglog. -#include <vespa/log/log.h> -LOG_SETUP("hyperloglog_test"); - #include <vespa/searchlib/grouping/hyperloglog.h> #include <vespa/vespalib/objects/nboserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> + +#include <vespa/log/log.h> +LOG_SETUP("hyperloglog_test"); using vespalib::NBOSerializer; using vespalib::nbostream; diff --git a/searchlib/src/tests/grouping/sketch_test.cpp b/searchlib/src/tests/grouping/sketch_test.cpp index ae1052e9f59..38ca80b2171 100644 --- a/searchlib/src/tests/grouping/sketch_test.cpp +++ b/searchlib/src/tests/grouping/sketch_test.cpp @@ -4,8 +4,9 @@ #include <vespa/searchlib/grouping/sketch.h> #include <vespa/vespalib/objects/nboserializer.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("sketch_test"); diff --git a/searchlib/src/tests/predicate/predicate_index_test.cpp b/searchlib/src/tests/predicate/predicate_index_test.cpp index d0af12f93c7..5a48e696af8 100644 --- a/searchlib/src/tests/predicate/predicate_index_test.cpp +++ b/searchlib/src/tests/predicate/predicate_index_test.cpp @@ -5,12 +5,13 @@ #include <vespa/searchlib/predicate/simple_index.hpp> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/util/data_buffer_writer.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search; using namespace search::predicate; diff --git a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp index d8c9691d421..9dd9a0cefe9 100644 --- a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp +++ b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp @@ -2,10 +2,10 @@ // Unit tests for predicate_interval_store. #include <vespa/searchlib/predicate/predicate_interval_store.h> - #include <vespa/searchlib/predicate/predicate_index.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vector> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search; using namespace search::predicate; diff --git a/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp b/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp index 4d71f585910..167b310b556 100644 --- a/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp +++ b/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp @@ -377,7 +377,7 @@ TEST("require that default open range works") { EXPECT_EQUAL(0u, result.features.size()); ASSERT_EQUAL(1u, result.range_features.size()); EXPECT_EQUAL("foo", result.range_features[0].label.make_string()); - EXPECT_EQUAL(LLONG_MIN, result.range_features[0].from); + EXPECT_EQUAL(LONG_MIN, result.range_features[0].from); EXPECT_EQUAL(39, result.range_features[0].to); } diff --git a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp index 4ae94c17804..a64081d41c6 100644 --- a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp +++ b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp @@ -73,18 +73,18 @@ TEST("require that PredicateQueryItem stack dump item can be read") { buf.appendCompressedNumber(2); appendString(buf, "key1"); appendString(buf, "value1"); - buf.Put64ToInet(-1ULL); + buf.Put64ToInet(-1UL); appendString(buf, "key2"); appendString(buf, "value2"); - buf.Put64ToInet(0xffffULL); + buf.Put64ToInet(0xffffUL); buf.appendCompressedNumber(2); appendString(buf, "key3"); - buf.Put64ToInet(42ULL); - buf.Put64ToInet(-1ULL); + buf.Put64ToInet(42UL); + buf.Put64ToInet(-1UL); appendString(buf, "key4"); - buf.Put64ToInet(84ULL); - buf.Put64ToInet(0xffffULL); + buf.Put64ToInet(84UL); + buf.Put64ToInet(0xffffUL); SimpleQueryStackDumpIterator query_stack(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); @@ -97,7 +97,7 @@ TEST("require that PredicateQueryItem stack dump item can be read") { ASSERT_EQUAL(2u, term.getFeatures().size()); ASSERT_EQUAL(2u, term.getRangeFeatures().size()); ASSERT_EQUAL("value1", term.getFeatures()[0].getValue()); - ASSERT_EQUAL(0xffffffffffffffffULL, + ASSERT_EQUAL(0xffffffffffffffffUL, term.getFeatures()[0].getSubQueryBitmap()); ASSERT_EQUAL("key2", term.getFeatures()[1].getKey()); ASSERT_EQUAL(42u, term.getRangeFeatures()[0].getValue()); diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 77d0099afdb..e88dfa3f386 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mysearch.h" -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/flow.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> @@ -8,6 +7,8 @@ #include <vespa/vespalib/objects/visit.h> #include <vespa/vespalib/data/slime/slime.h> #include <algorithm> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("blueprint_test"); diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index a8707bb6f7e..86297923503 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mysearch.h" -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/flow.h> @@ -22,6 +21,8 @@ #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/normalize_class_name.h> #include <filesystem> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("blueprint_test"); @@ -999,9 +1000,9 @@ TEST("test WeakAnd Blueprint") { auto &s = dynamic_cast<WeakAndSearch&>(*search); EXPECT_EQUAL(456u, s.getN()); ASSERT_EQUAL(3u, s.getTerms().size()); - EXPECT_GREATER(s.get_max_score(0), 0.0); - EXPECT_GREATER(s.get_max_score(1), 0.0); - EXPECT_GREATER(s.get_max_score(2), 0.0); + EXPECT_GREATER(s.get_max_score(0), 0l); + EXPECT_GREATER(s.get_max_score(1), 0l); + EXPECT_GREATER(s.get_max_score(2), 0l); wand::Terms terms = s.getTerms(); std::sort(terms.begin(), terms.end(), WeightOrder()); EXPECT_EQUAL(120, terms[0].weight); diff --git a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp index 1ae6cff447e..201cd7ab54e 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp @@ -126,7 +126,7 @@ TEST_F("require that blueprint with zstar-compressed estimates non-empty.", Fixt } void -runQuery(Fixture & f, std::vector<uint32_t> expected, bool expectCachedSize, uint32_t expectedKV) { +runQuery(Fixture & f, std::vector<uint32_t> expected, uint32_t expectCachedSize, uint32_t expectedKV) { PredicateBlueprint blueprint(f.field, f.guard(), f.query); blueprint.basic_plan(true, 100); blueprint.fetchPostings(ExecuteInfo::FULL); diff --git a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp index b02df263f82..3430539e513 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp @@ -367,8 +367,7 @@ TEST("require that subquery bitmap is unpacked to subqueries.") { search.initFullRange(); EXPECT_TRUE(search.seek(2)); search.unpack(2); - EXPECT_EQUAL(0xffffffffffffffffULL, - static_cast<unsigned long long>(data.getSubqueries())); + EXPECT_EQUAL(0xffffffffffffffffUL, data.getSubqueries()); } diff --git a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp index 9d93ade71d9..2bb255e2a1d 100644 --- a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp +++ b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/simpleresult.h> @@ -9,6 +8,8 @@ #include <vespa/searchcommon/attribute/i_search_context.h> #include <vespa/searchlib/attribute/searchcontextelementiterator.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search::fef; using namespace search::queryeval; diff --git a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp index f4de5bb0086..460de0b3cc7 100644 --- a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp +++ b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> +#include "rise_wand.h" +#include "rise_wand.hpp" #include <vespa/searchlib/queryeval/andsearch.h> #include <vespa/searchlib/queryeval/andnotsearch.h> #include <vespa/searchlib/queryeval/fake_search.h> @@ -11,8 +12,8 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/util/stringfmt.h> -#include "rise_wand.h" -#include "rise_wand.hpp" +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search::fef; using namespace search::queryeval; diff --git a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp index 4fdcc7f1d8a..6c6ca7ce543 100644 --- a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp +++ b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/plugin/setup.h> @@ -7,6 +6,8 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/asciistream.h> #include <regex> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search::features; using namespace search::fef::test; diff --git a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp index 085fe8d3149..5a24f55c953 100644 --- a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp +++ b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp @@ -1,7 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using search::attribute::Config; using search::attribute::BasicType; diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index 0e173691f99..498d1f673b0 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/tensor/dense_tensor_store.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> @@ -9,6 +8,9 @@ #include <vespa/eval/eval/test/value_compare.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> + #include <vespa/log/log.h> LOG_SETUP("dense_tensor_store_test"); diff --git a/searchlib/src/tests/transactionlog/chunks_test.cpp b/searchlib/src/tests/transactionlog/chunks_test.cpp index e057e853d0d..144dffc82fe 100644 --- a/searchlib/src/tests/transactionlog/chunks_test.cpp +++ b/searchlib/src/tests/transactionlog/chunks_test.cpp @@ -1,8 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/transactionlog/chunks.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <atomic> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("translog_chunks_test"); diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index 07f43e98cd3..e4a7b66e7b1 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -525,11 +525,11 @@ testVisitOverGeneratedDomain(const vespalib::string & testDir) { createDomainTest(tls, name); auto s1 = openDomainTest(tls, name); fillDomainTest(s1.get(), name); - EXPECT_EQUAL(0, getMaxSessionRunTime(tlss.tls, "test1")); + EXPECT_EQUAL(0.0, getMaxSessionRunTime(tlss.tls, "test1")); visitDomainTest(tls, s1.get(), name); double maxSessionRunTime = getMaxSessionRunTime(tlss.tls, "test1"); LOG(info, "testVisitOverGeneratedDomain(): maxSessionRunTime=%f", maxSessionRunTime); - EXPECT_GREATER(maxSessionRunTime, 0); + EXPECT_GREATER(maxSessionRunTime, 0.0); } void diff --git a/searchlib/src/tests/util/slime_output_raw_buf_adapter/slime_output_raw_buf_adapter_test.cpp b/searchlib/src/tests/util/slime_output_raw_buf_adapter/slime_output_raw_buf_adapter_test.cpp index 9c9c28f6b8b..c9e52d9e845 100644 --- a/searchlib/src/tests/util/slime_output_raw_buf_adapter/slime_output_raw_buf_adapter_test.cpp +++ b/searchlib/src/tests/util/slime_output_raw_buf_adapter/slime_output_raw_buf_adapter_test.cpp @@ -1,8 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace search; using namespace vespalib::slime::convenience; diff --git a/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp b/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp index 0749f19fceb..89e6e6685e9 100644 --- a/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/expression/documentfieldnode.h> #include <vespa/searchlib/expression/interpolated_document_field_lookup_node.h> #include <vespa/searchlib/expression/interpolatedlookupfunctionnode.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> using namespace search::expression; @@ -59,6 +60,14 @@ Attribute2DocumentAccessor::getReplacementNode(const AttributeNode &attributeNod return std::make_unique<DocumentFieldNode>(attributeNode.getAttributeName()); } +std::unique_ptr<ExpressionNode> +NonAttribute2DocumentAccessor::getReplacementNode(const expression::AttributeNode &attributeNode) { + if (_attrCtx.getAttribute(attributeNode.getAttributeName()) == nullptr) { + return Attribute2DocumentAccessor::getReplacementNode(attributeNode); + } + return {}; +} + } // this function was added by ../../forcelink.sh diff --git a/searchlib/src/vespa/searchlib/aggregation/modifiers.h b/searchlib/src/vespa/searchlib/aggregation/modifiers.h index e55ca28bd62..9b0f5d8186d 100644 --- a/searchlib/src/vespa/searchlib/aggregation/modifiers.h +++ b/searchlib/src/vespa/searchlib/aggregation/modifiers.h @@ -7,10 +7,12 @@ #include <functional> namespace search::expression { + class ExpressionNode; + class AttributeNode; +} -class ExpressionNode; -class AttributeNode; - +namespace search::attribute { + class IAttributeContext; } namespace search::aggregation { @@ -28,8 +30,17 @@ private: class Attribute2DocumentAccessor : public AttributeNodeReplacer { +protected: + ExpressionNodeUP getReplacementNode(const expression::AttributeNode &attributeNode) override; +}; + +class NonAttribute2DocumentAccessor : public Attribute2DocumentAccessor +{ +public: + explicit NonAttribute2DocumentAccessor(const attribute::IAttributeContext &attrCtx) noexcept : _attrCtx(attrCtx) {} private: ExpressionNodeUP getReplacementNode(const expression::AttributeNode &attributeNode) override; + const attribute::IAttributeContext &_attrCtx; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributecontext.h b/searchlib/src/vespa/searchlib/attribute/attributecontext.h index a02e05abe4f..bd98031ee66 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributecontext.h +++ b/searchlib/src/vespa/searchlib/attribute/attributecontext.h @@ -29,7 +29,7 @@ private: const IAttributeVector *getAttribute(AttributeMap & map, const string & name, bool stableEnum) const; const IAttributeVector *getAttributeMtSafe(AttributeMap & map, const string & name, bool stableEnum) const; public: - AttributeContext(const IAttributeManager & manager); + explicit AttributeContext(const IAttributeManager & manager); ~AttributeContext() override; // Implements IAttributeContext diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index 03d2e94b5d0..6e0fb1f9c39 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/fef/objectstore.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stash.h> +#include <algorithm> #include <cmath> #include <stdexcept> @@ -18,6 +19,7 @@ namespace search::features { using fef::AnyWrapper; using fef::Blueprint; +using fef::DocumentFrequency; using fef::FeatureExecutor; using fef::FeatureNameBuilder; using fef::FieldInfo; @@ -36,8 +38,11 @@ get_inverse_document_frequency(const ITermFieldData& term_field, const ITermData& term) { - double fallback = Bm25Executor::calculate_inverse_document_frequency(term_field.get_matching_doc_count(), - term_field.get_total_doc_count()); + auto doc_freq = util::lookup_document_frequency(env, term); + if (doc_freq.has_value()) { + return Bm25Executor::calculate_inverse_document_frequency(doc_freq.value()); + } + double fallback = Bm25Executor::calculate_inverse_document_frequency(term_field.get_doc_freq()); return util::lookupSignificance(env, term, fallback); } @@ -68,10 +73,13 @@ Bm25Executor::Bm25Executor(const fef::FieldInfo& field, } double -Bm25Executor::calculate_inverse_document_frequency(uint32_t matching_doc_count, uint32_t total_doc_count) noexcept +Bm25Executor::calculate_inverse_document_frequency(DocumentFrequency doc_freq) noexcept { - return std::log(1 + (static_cast<double>(total_doc_count - matching_doc_count + 0.5) / - static_cast<double>(matching_doc_count + 0.5))); + double frequency = doc_freq.frequency; + double count = doc_freq.count; + count = std::max(1.0, count); + frequency = std::min(std::max(1.0, frequency), count); + return std::log(1 + ((count - frequency + 0.5) / (frequency + 0.5))); } void diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.h b/searchlib/src/vespa/searchlib/features/bm25_feature.h index 637d656990b..4a2f0e12452 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.h +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.h @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/fef/blueprint.h> +#include <vespa/searchlib/fef/document_frequency.h> #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/vespalib/util/trinary.h> @@ -39,7 +40,7 @@ public: double k1_param, double b_param); - double static calculate_inverse_document_frequency(uint32_t matching_doc_count, uint32_t total_doc_count) noexcept; + static double calculate_inverse_document_frequency(search::fef::DocumentFrequency doc_freq) noexcept; void handle_bind_match_data(const fef::MatchData& match_data) override; void execute(uint32_t docId) override; diff --git a/searchlib/src/vespa/searchlib/features/queryterm.cpp b/searchlib/src/vespa/searchlib/features/queryterm.cpp index d3a8a4f6cf4..21ec40d3a81 100644 --- a/searchlib/src/vespa/searchlib/features/queryterm.cpp +++ b/searchlib/src/vespa/searchlib/features/queryterm.cpp @@ -12,8 +12,7 @@ QueryTerm QueryTermFactory::create(const IQueryEnvironment & env, uint32_t termIdx, bool lookupConnectedness) { const ITermData *termData = env.getTerm(termIdx); - feature_t fallback = util::getSignificance(*termData); - feature_t significance = features::util::lookupSignificance(env, termIdx, fallback); + feature_t significance = util::get_legacy_significance(env, *termData); feature_t connectedness = 0; if (lookupConnectedness) { connectedness = util::lookupConnectedness(env, termIdx); diff --git a/searchlib/src/vespa/searchlib/features/termfeature.cpp b/searchlib/src/vespa/searchlib/features/termfeature.cpp index e3ce4f26833..a27b6fbe9ad 100644 --- a/searchlib/src/vespa/searchlib/features/termfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termfeature.cpp @@ -21,8 +21,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, _significance(0) { if (_termData != nullptr) { - feature_t fallback = util::getSignificance(*_termData); - _significance = util::lookupSignificance(env, termId, fallback); + _significance = util::get_legacy_significance(env, *_termData); } } diff --git a/searchlib/src/vespa/searchlib/features/utils.cpp b/searchlib/src/vespa/searchlib/features/utils.cpp index 0c3bbcb0ffa..70c689109bf 100644 --- a/searchlib/src/vespa/searchlib/features/utils.cpp +++ b/searchlib/src/vespa/searchlib/features/utils.cpp @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/issue.h> +#include <algorithm> +#include <cassert> #include <charconv> #include <cmath> #include <ostream> @@ -100,56 +102,47 @@ lookupSignificance(const search::fef::IQueryEnvironment& env, const ITermData& t return fallback; } +static const double N = 1000000.0; + feature_t -lookupSignificance(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) +calculate_legacy_significance(DocumentFrequency doc_freq) { - const ITermData* term = env.getTerm(termId); - if (term == nullptr) { - return fallback; + if (doc_freq.count == 0) { + return 0.5; // Corner case, no documents } - return lookupSignificance(env, *term, fallback); + double frequency = doc_freq.frequency; + double count = doc_freq.count; + // Rescale frequency and count to corpus of N documents. + frequency = std::min(std::max(1.0, frequency * N / count), N); + count = N; + double logcount = std::log(count); + double logfrequency = std::log(frequency); + // Using traditional formula for inverse document frequency, see + // https://en.wikipedia.org/wiki/Tf%E2%80%93idf#Inverse_document_frequency + double idf = logcount - logfrequency; + // We normalize against document frequency 1 in corpus of N documents. + double normalized_idf = idf / logcount; // normalized to range [0;1] + double renormalized_idf = 0.5 + 0.5 * normalized_idf; // normalized to range [0.5;1] + return renormalized_idf; } -double -getRobertsonSparckJonesWeight(double docCount, double docsInCorpus) +DocumentFrequency +aggregate_max(DocumentFrequency lhs, DocumentFrequency rhs) { - return std::log((docsInCorpus - docCount + 0.5)/(docCount + 0.5)); + return { std::max(lhs.frequency, rhs.frequency), std::max(lhs.count, rhs.count)}; } -static const double N = 1000000.0; - feature_t -getSignificance(double docFreq) -{ - if (docFreq < (1.0/N)) { - docFreq = 1.0/N; - } - if (docFreq > 1.0) { - docFreq = 1.0; - } - double d = std::log(docFreq)/std::log(1.0/N); - return 0.5 + 0.5 * d; -#if 0 - double n = docFreq * N; - n = (n == 0) ? 1 : (n > N ? N : n); - double a = getRobertsonSparckJonesWeight(1, N + 1); - double b = getRobertsonSparckJonesWeight(N + 1, N + 1); - double w = getRobertsonSparckJonesWeight(n, N + 1); - return ((w - b)/(a - b)); -#endif -} - -feature_t -getSignificance(const search::fef::ITermData& termData) +calculate_legacy_significance(const ITermData& termData) { using FRA = search::fef::ITermFieldRangeAdapter; - double df = 0; + DocumentFrequency df(0, 0); for (FRA iter(termData); iter.valid(); iter.next()) { - df = std::max(df, iter.get().getDocFreq()); + df = aggregate_max(df, iter.get().get_doc_freq()); } - feature_t signif = getSignificance(df); - LOG(debug, "getSignificance %e %f [ %e %f ] = %e", df, df, df * N, df * N, signif); + feature_t signif = calculate_legacy_significance(df); + LOG(debug, "calculate_legacy_significance %" PRIu64 " %" PRIu64 " = %e", df.frequency, df.count, signif); return signif; } @@ -167,7 +160,7 @@ lookupTable(const search::fef::IIndexEnvironment & env, const vespalib::string & return retval; } -const search::fef::ITermData * +const ITermData * getTermByLabel(const search::fef::IQueryEnvironment &env, const vespalib::string &label) { // Labeling the query item with unique id '5' with the label 'foo' @@ -195,4 +188,33 @@ getTermByLabel(const search::fef::IQueryEnvironment &env, const vespalib::string return 0; } +std::optional<DocumentFrequency> +lookup_document_frequency(const search::fef::IQueryEnvironment& env, const ITermData& term) +{ + vespalib::asciistream os; + auto unique_id = term.getUniqueId(); + if (unique_id != 0) { + os << "vespa.term." << unique_id << ".docfreq"; + Property p = env.getProperties().lookup(os.str()); + if (p.size() == 2) { + // we have a defined document frequency + auto document_frequency = strToNum<uint64_t>(p.getAt(0)); + auto document_count = strToNum<uint64_t>(p.getAt(1)); + return DocumentFrequency(document_frequency, document_count); + } + } + return {}; +} + +feature_t +get_legacy_significance(const IQueryEnvironment& env, const ITermData& term) +{ + auto docfreq = lookup_document_frequency(env, term); + if (docfreq.has_value()) { + return calculate_legacy_significance(docfreq.value()); + } + feature_t fallback = calculate_legacy_significance(term); + return lookupSignificance(env, term, fallback); +} + } diff --git a/searchlib/src/vespa/searchlib/features/utils.h b/searchlib/src/vespa/searchlib/features/utils.h index e90592fbe5d..4aca1a1635d 100644 --- a/searchlib/src/vespa/searchlib/features/utils.h +++ b/searchlib/src/vespa/searchlib/features/utils.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/searchlib/fef/document_frequency.h> #include <vespa/searchlib/fef/iqueryenvironment.h> #include <vespa/searchlib/fef/table.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> @@ -10,6 +11,7 @@ #include <vespa/searchlib/common/feature.h> #include <vespa/vespalib/util/string_hash.h> #include <limits> +#include <optional> namespace search::features::util { @@ -107,30 +109,12 @@ feature_t lookupConnectedness(const search::fef::IQueryEnvironment & env, feature_t lookupSignificance(const search::fef::IQueryEnvironment& env, const search::fef::ITermData& term, feature_t fallback); /** - * Returns the significance of the given term. - * Uses the property map of the query environment to lookup this data. - * - * @param env The query environment. - * @param termId The term id. - * @param fallback The value to return if the significance was not found in the property map. - * @return The significance. - */ -feature_t lookupSignificance(const search::fef::IQueryEnvironment & env, uint32_t termId, feature_t fallback = 0.0f); - -/** - * Returns the Robertson-Sparck-Jones weight based on the given document count - * (number of documents containing the term) and the number of documents in the corpus. - * This weight is a variant of inverse document frequency. - */ -double getRobertsonSparckJonesWeight(double docCount, double docsInCorpus); - -/** - * Returns the significance based on the given scaled number of documents containing the term. + * Returns the significance based on the given document frequency * - * @param docFreq The scaled number of documents containing the term. - * @return The significance. + * @param doc_freq The document frequency + * @return The significance. */ -feature_t getSignificance(double docFreq); +feature_t calculate_legacy_significance(search::fef::DocumentFrequency doc_freq); /** * Returns the significance based on max known frequency of the term @@ -138,7 +122,7 @@ feature_t getSignificance(double docFreq); * @param termData Data for the term * @return The significance. */ -feature_t getSignificance(const search::fef::ITermData &termData); +feature_t calculate_legacy_significance(const search::fef::ITermData& termData); /** * Lookups a table by using the properties and the table manager in the given index environment. @@ -198,4 +182,10 @@ getTermFieldHandle(const search::fef::IQueryEnvironment &env, uint32_t termId, u const search::fef::ITermData * getTermByLabel(const search::fef::IQueryEnvironment &env, const vespalib::string &label); +std::optional<search::fef::DocumentFrequency> +lookup_document_frequency(const search::fef::IQueryEnvironment& env, const search::fef::ITermData& term); + +feature_t +get_legacy_significance(const search::fef::IQueryEnvironment& env, const search::fef::ITermData& term); + } diff --git a/searchlib/src/vespa/searchlib/fef/document_frequency.h b/searchlib/src/vespa/searchlib/fef/document_frequency.h new file mode 100644 index 00000000000..0dde5c6d0c2 --- /dev/null +++ b/searchlib/src/vespa/searchlib/fef/document_frequency.h @@ -0,0 +1,26 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> + +namespace search::fef { + +/* + * Struct containing the raw data used to calculate significance. + */ +struct DocumentFrequency { + uint64_t frequency; // number of documents containing the word + uint64_t count; // total number of documents + + DocumentFrequency(uint64_t document_frequency_in, uint64_t document_count_in) + : frequency(document_frequency_in), + count(document_count_in) + { + } + bool operator==(const DocumentFrequency& rhs) const noexcept { + return frequency == rhs.frequency && count == rhs.count; + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/fef/itermfielddata.h b/searchlib/src/vespa/searchlib/fef/itermfielddata.h index 44d8dde19cd..7c7bc982ab1 100644 --- a/searchlib/src/vespa/searchlib/fef/itermfielddata.h +++ b/searchlib/src/vespa/searchlib/fef/itermfielddata.h @@ -2,6 +2,7 @@ #pragma once +#include "document_frequency.h" #include "handle.h" #include "match_data_details.h" @@ -40,13 +41,12 @@ public: uint32_t get_total_doc_count() const noexcept { return _total_doc_count; } /** - * Obtain the document frequency. This is a value between 0 and 1 - * indicating the ratio of the matching documents to the corpus. + * Obtain the document frequency. * * @return document frequency **/ - double getDocFreq() const noexcept { - return (double)get_matching_doc_count() / (double)get_total_doc_count(); + DocumentFrequency get_doc_freq() const noexcept { + return { get_matching_doc_count(), get_total_doc_count() }; } /** diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h index f0a359b98df..18fee3c7725 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/searchlib/fef/document_frequency.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/features/bm25_feature.h> @@ -448,7 +449,7 @@ public: Bm25TermFrequencyScorer(uint32_t num_docs, float range) noexcept : _num_docs(num_docs), _range(range), - _max_idf(Bm25Executor::calculate_inverse_document_frequency(1, _num_docs)) + _max_idf(Bm25Executor::calculate_inverse_document_frequency({1, _num_docs})) { } double apply_range(double idf) const noexcept { return (1.0 - _range)*_max_idf + _range * idf; @@ -456,7 +457,7 @@ public: // weight * scaled_bm25_idf, scaled to fixedpoint score_t calculateMaxScore(double estHits, double weight) const noexcept { return score_t(TermFrequencyScorer_TERM_SCORE_FACTOR * weight * - apply_range(Bm25Executor::calculate_inverse_document_frequency(estHits, _num_docs))); + apply_range(Bm25Executor::calculate_inverse_document_frequency({static_cast<uint64_t>(estHits), _num_docs}))); } score_t calculateMaxScore(const Term &term) const noexcept { diff --git a/searchlib/src/vespa/searchlib/test/ft_test_app_base.cpp b/searchlib/src/vespa/searchlib/test/ft_test_app_base.cpp index 1b60ca2cd33..6e7ba5366be 100644 --- a/searchlib/src/vespa/searchlib/test/ft_test_app_base.cpp +++ b/searchlib/src/vespa/searchlib/test/ft_test_app_base.cpp @@ -3,6 +3,7 @@ #include "ft_test_app_base.h" #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP(".ft_test_app_base"); diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index 535d0ae7d78..282d9fc9e77 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -52,6 +52,13 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { } @Override + public Integer zookeeperJuteMaxBuffer() { + return Optional.ofNullable(System.getenv("VESPA_CONFIGSERVER_ZOOKEEPER_JUTE_MAX_BUFFER")) + .map(Integer::parseInt) + .orElse(104857600); + } + + @Override public Optional<Long> sessionLifeTimeSecs() { return getInstallVariable("session_lifetime", Long::parseLong); } diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index f41dae89eec..a20902422b0 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -1,9 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> -#include <tests/common/storage_config_set.h> +#include <vespa/config-stor-distribution.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/datatype/documenttype.h> @@ -25,7 +26,6 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/testkit/test_path.h> -#include <vespa/config-stor-distribution.h> #include <future> #include <vespa/log/log.h> @@ -116,16 +116,17 @@ public: BucketManagerTest::~BucketManagerTest() = default; -#define ASSERT_DUMMYLINK_REPLY_COUNT(link, count) \ - if (link->getNumReplies() != count) { \ - std::ostringstream ost; \ - ost << "Expected there to be " << count << " replies in link, but " \ - << "found " << link->getNumReplies() << ":\n"; \ - for (uint32_t i=0; i<link->getNumReplies(); ++i) { \ - ost << link->getReply(i)->getType() << "\n"; \ - } \ - FAIL() << ost.str(); \ +void check_dummy_link_reply_count(const DummyStorageLink& link, size_t expected_count) { + if (link.getNumReplies() != expected_count) { + std::ostringstream ost; + ost << "Expected there to be " << expected_count << " replies in link, but " + << "found " << link.getNumReplies() << ":\n"; + for (uint32_t i = 0; i < link.getNumReplies(); ++i) { + ost << link.getReply(i)->getType() << "\n"; + } + FAIL() << ost.str(); } +} void BucketManagerTest::setupTestEnvironment() { @@ -313,7 +314,7 @@ TEST_F(BucketManagerTest, DISABLED_request_bucket_info_with_state) { { LOG(info, "Waiting for response from 3 request bucket info messages"); _top->waitForMessages(3, 5); - ASSERT_DUMMYLINK_REPLY_COUNT(_top, 3); + ASSERT_NO_FATAL_FAILURE(check_dummy_link_reply_count(*_top, 3)); std::map<uint64_t, api::RequestBucketInfoReply::SP> replies; for (uint32_t i=0; i<3; ++i) { replies[_top->getReply(i)->getMsgId()] @@ -357,7 +358,7 @@ TEST_F(BucketManagerTest, request_bucket_info_with_list) { _top->sendDown(cmd); _top->waitForMessages(1, 5); - ASSERT_DUMMYLINK_REPLY_COUNT(_top, 1); + ASSERT_NO_FATAL_FAILURE(check_dummy_link_reply_count(*_top, 1)); auto reply = std::dynamic_pointer_cast<api::RequestBucketInfoReply>(_top->getReply(0)); _top->reset(); ASSERT_TRUE(reply.get()); @@ -527,7 +528,7 @@ class ConcurrentOperationFixture { public: explicit ConcurrentOperationFixture(BucketManagerTest& self) : _self(self), - _state(std::make_shared<lib::ClusterState>("distributor:1 storage:1")) + _state(std::make_shared<lib::ClusterState>("version:2 distributor:1 storage:1")) { _self.setupTestEnvironment(); _self._top->open(); diff --git a/storage/src/tests/common/testnodestateupdater.cpp b/storage/src/tests/common/testnodestateupdater.cpp index f9671617352..5d1d7a085b9 100644 --- a/storage/src/tests/common/testnodestateupdater.cpp +++ b/storage/src/tests/common/testnodestateupdater.cpp @@ -25,17 +25,28 @@ TestNodeStateUpdater::getClusterStateBundle() const } void +TestNodeStateUpdater::patch_distribution(std::shared_ptr<const lib::Distribution> distribution) +{ + _clusterStateBundle = _clusterStateBundle->clone_with_new_distribution( + lib::DistributionConfigBundle::of(std::move(distribution))); +} + +void TestNodeStateUpdater::setClusterState(std::shared_ptr<const lib::ClusterState> c) { - setClusterStateBundle(std::make_shared<const lib::ClusterStateBundle>(*c)); + setClusterStateBundle(std::make_shared<const lib::ClusterStateBundle>(std::move(c))); } void TestNodeStateUpdater::setClusterStateBundle(std::shared_ptr<const lib::ClusterStateBundle> clusterStateBundle) { + auto existing_distr = _clusterStateBundle->distribution_config_bundle(); _clusterStateBundle = std::move(clusterStateBundle); - for (uint32_t i = 0; i < _listeners.size(); ++i) { - _listeners[i]->handleNewState(); + if (!_clusterStateBundle->has_distribution_config() && existing_distr) { + _clusterStateBundle = _clusterStateBundle->clone_with_new_distribution(existing_distr); + } + for (auto* listener : _listeners) { + listener->handleNewState(); } } diff --git a/storage/src/tests/common/testnodestateupdater.h b/storage/src/tests/common/testnodestateupdater.h index e5418c238d5..6a00e9a2264 100644 --- a/storage/src/tests/common/testnodestateupdater.h +++ b/storage/src/tests/common/testnodestateupdater.h @@ -1,11 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::TestNodeStateUpdater - * \ingroup common - * - * \brief Test implementation of the node state updater. - */ - #pragma once #include <vespa/storage/common/nodestateupdater.h> @@ -14,6 +7,7 @@ namespace storage::lib { class ClusterState; class ClusterStateBundle; + class Distribution; } namespace storage { @@ -53,6 +47,7 @@ public: _current = std::make_shared<lib::NodeState>(state); } + void patch_distribution(std::shared_ptr<const lib::Distribution> distribution); void setClusterState(std::shared_ptr<const lib::ClusterState> c); void setClusterStateBundle(std::shared_ptr<const lib::ClusterStateBundle> clusterStateBundle); diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index d811f100aec..b2b82a46850 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -6,10 +6,10 @@ #include <vespa/storage/config/config-stor-distributormanager.h> #include <vespa/storage/config/config-stor-visitordispatcher.h> #include <vespa/config-stor-distribution.h> -#include <vespa/config-fleetcontroller.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/time.h> #include <vespa/config/subscription/configuri.h> @@ -59,6 +59,7 @@ TestStorageApp::TestStorageApp(StorageComponentRegisterImpl::UP compReg, auto distr = std::make_shared<lib::Distribution>( lib::Distribution::getDefaultDistributionConfig(redundancy, nodeCount)); _compReg.setDistribution(distr); + _nodeStateUpdater.patch_distribution(distr); } TestStorageApp::~TestStorageApp() = default; @@ -69,6 +70,7 @@ TestStorageApp::setDistribution(Redundancy redundancy, NodeCount nodeCount) auto distr = std::make_shared<lib::Distribution>( lib::Distribution::getDefaultDistributionConfig(redundancy, nodeCount)); _compReg.setDistribution(distr); + _nodeStateUpdater.patch_distribution(distr); } void @@ -83,6 +85,12 @@ TestStorageApp::setClusterState(const lib::ClusterState& c) _nodeStateUpdater.setClusterState(std::make_shared<lib::ClusterState>(c)); } +void +TestStorageApp::set_cluster_state_bundle(std::shared_ptr<const lib::ClusterStateBundle> state_bundle) +{ + _nodeStateUpdater.setClusterStateBundle(std::move(state_bundle)); +} + namespace { NodeIndex node_index_from_config(const config::ConfigUri& uri) { diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 04fa6996e15..c423761a9a2 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -73,6 +73,7 @@ public: void setDistribution(Redundancy, NodeCount); void setTypeRepo(std::shared_ptr<const document::DocumentTypeRepo> repo); void setClusterState(const lib::ClusterState&); + void set_cluster_state_bundle(std::shared_ptr<const lib::ClusterStateBundle>); // Utility functions for getting a hold of currently used bits. Practical // to avoid adding extra components in the tests. @@ -81,7 +82,7 @@ public: std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() { return _compReg.getTypeRepo(); } const document::BucketIdFactory& getBucketIdFactory() { return _compReg.getBucketIdFactory(); } TestNodeStateUpdater& getStateUpdater() { return _nodeStateUpdater; } - std::shared_ptr<lib::Distribution> & getDistribution() { return _compReg.getDistribution(); } + std::shared_ptr<const lib::Distribution> getDistribution() { return _compReg.getDistribution(); } TestNodeStateUpdater& getNodeStateUpdater() { return _nodeStateUpdater; } uint16_t getIndex() const { return _compReg.getIndex(); } const NodeIdentity& node_identity() const noexcept { return _node_identity; } diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 8982b02f2b7..6993882f479 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -53,6 +53,7 @@ struct ChangedBucketOwnershipHandlerTest : Test { void applyDistribution(Redundancy, NodeCount); void applyClusterState(const lib::ClusterState&); + void apply_cluster_state_bundle(std::shared_ptr<const lib::ClusterStateBundle>); document::BucketId nextOwnedBucket( uint16_t wantedOwner, @@ -84,6 +85,21 @@ struct ChangedBucketOwnershipHandlerTest : Test { return lib::ClusterState("distributor:4 storage:1 .0.s:d"); } + static std::shared_ptr<const lib::DistributionConfigBundle> make_distr_bundle(uint16_t node_count) { + return lib::DistributionConfigBundle::of(lib::Distribution::getDefaultDistributionConfig(1, node_count)); + } + + static std::shared_ptr<const lib::ClusterStateBundle> make_state_bundle_with_config( + vespalib::stringref state_str, uint16_t node_count) + { + return std::make_shared<const lib::ClusterStateBundle>( + std::make_shared<const lib::ClusterState>(state_str), + lib::ClusterStateBundle::BucketSpaceStateMapping{}, + std::nullopt, + make_distr_bundle(node_count), + false); + } + void SetUp() override; }; @@ -194,6 +210,7 @@ hasOnlySetSystemStateCmdQueued(DummyStorageLink& link) { void ChangedBucketOwnershipHandlerTest::applyDistribution(Redundancy redundancy, NodeCount nodeCount) { + // TODO set distribution via state bundle instead _app->setDistribution(redundancy, nodeCount); _handler->storageDistributionChanged(); } @@ -205,6 +222,13 @@ ChangedBucketOwnershipHandlerTest::applyClusterState(const lib::ClusterState& st _handler->reloadClusterState(); } +void +ChangedBucketOwnershipHandlerTest::apply_cluster_state_bundle(std::shared_ptr<const lib::ClusterStateBundle> state_bundle) +{ + _app->set_cluster_state_bundle(std::move(state_bundle)); + _handler->reloadClusterState(); +} + TEST_F(ChangedBucketOwnershipHandlerTest, enumerate_buckets_belonging_on_changed_nodes) { lib::ClusterState stateBefore("distributor:4 storage:1"); applyDistribution(Redundancy(1), NodeCount(4)); @@ -225,7 +249,7 @@ TEST_F(ChangedBucketOwnershipHandlerTest, enumerate_buckets_belonging_on_changed EXPECT_TRUE(hasAbortedNoneOf(cmd, node2Buckets)); // Handler must swallow abort replies - _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply().release())); + _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply())); EXPECT_EQ(size_t(0), _top->getNumReplies()); } @@ -312,7 +336,7 @@ TEST_F(ChangedBucketOwnershipHandlerTest, ownership_changed_on_distributor_up_ed EXPECT_TRUE(hasAbortedNoneOf(cmd, node2Buckets)); // Handler must swallow abort replies - _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply().release())); + _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply())); EXPECT_EQ(0, _top->getNumReplies()); } @@ -345,6 +369,32 @@ TEST_F(ChangedBucketOwnershipHandlerTest, distribution_config_change_updates_own sendAndExpectAbortedCreateBucket(2); } +TEST_F(ChangedBucketOwnershipHandlerTest, distribution_config_via_state_bundle_change_updates_ownership) { + apply_cluster_state_bundle(make_state_bundle_with_config("version:2 distributor:3 storage:1", 3)); + // Apply new distribution config containing only 1 distributor, meaning + // any messages sent from >1 must be aborted. + // This test case is a bit dodgy since the CC should never send a state with more nodes in it than + // the distribution config allows for when _it_ is responsible for also sending the config. + apply_cluster_state_bundle(make_state_bundle_with_config("version:3 distributor:3 storage:1", 1)); + sendAndExpectAbortedCreateBucket(2); +} + +TEST_F(ChangedBucketOwnershipHandlerTest, ignore_internal_config_once_state_bundle_with_config_received) { + apply_cluster_state_bundle(make_state_bundle_with_config("version:2 distributor:3 storage:3", 1)); + applyDistribution(Redundancy(1), NodeCount(3)); + // Bundle config says 1 node, internal config says 3. Trust the bundle(tm). + sendAndExpectAbortedCreateBucket(2); +} + +TEST_F(ChangedBucketOwnershipHandlerTest, revert_to_internal_config_if_distribution_no_longer_received_in_state_bundle) { + apply_cluster_state_bundle(make_state_bundle_with_config("version:2 distributor:1 storage:3", 3)); + + applyDistribution(Redundancy(1), NodeCount(1)); // not yet used + applyClusterState(lib::ClusterState("version:3 distributor:3 storage:3")); // no bundle config; revert to internal + + sendAndExpectAbortedCreateBucket(2); +} + /** * Generate and dispatch a message of the given type with the provided * arguments as if that message was sent from distributor 1. Messages will diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index 79246cb3ce1..45ad44b1152 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -10,6 +10,7 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/storageserver/statemanager.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/config-stor-distribution.h> #include <vespa/vespalib/gtest/gtest.h> using storage::lib::NodeState; @@ -38,6 +39,23 @@ struct StateManagerTest : Test, NodeStateReporter { return cmd; } + static std::shared_ptr<const lib::ClusterStateBundle> make_state_bundle_with_config( + vespalib::stringref state_str, uint16_t num_nodes) + { + auto state = std::make_shared<const ClusterState>(state_str); + auto distr = lib::DistributionConfigBundle::of(lib::Distribution::getDefaultDistributionConfig(1, num_nodes)); + return std::make_shared<lib::ClusterStateBundle>(std::move(state), + lib::ClusterStateBundle::BucketSpaceStateMapping{}, + std::nullopt, std::move(distr), false); + } + + + static std::shared_ptr<api::SetSystemStateCommand> make_set_state_cmd_with_config( + vespalib::stringref state_str, uint16_t num_nodes) + { + return std::make_shared<api::SetSystemStateCommand>(make_state_bundle_with_config(state_str, num_nodes)); + } + void get_single_reply(std::shared_ptr<api::StorageReply>& reply_out); void get_only_ok_reply(std::shared_ptr<api::StorageReply>& reply_out); void force_current_cluster_state_version(uint32_t version, uint16_t cc_index); @@ -56,6 +74,10 @@ struct StateManagerTest : Test, NodeStateReporter { void report(vespalib::JsonStream &) const override {} void extract_cluster_state_version_from_host_info(uint32_t& version_out); + + static vespalib::string to_string(const lib::Distribution::DistributionConfig& cfg) { + return lib::Distribution(cfg).serialized(); + } }; StateManagerTest::StateManagerTest() @@ -148,26 +170,107 @@ StateManagerTest::extract_cluster_state_version_from_host_info(uint32_t& version version_out = clusterStateVersionCursor.asLong(); } -TEST_F(StateManagerTest, cluster_state) { - std::shared_ptr<api::StorageReply> reply; - // Verify initial state on startup - auto currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); +TEST_F(StateManagerTest, cluster_state_and_config_has_expected_values_at_bootstrap) { + auto initial_bundle = _manager->getClusterStateBundle(); + auto currentState = initial_bundle->getBaselineClusterState(); EXPECT_EQ("cluster:d", currentState->toString(false)); EXPECT_EQ(currentState->getVersion(), 0); + // Distribution config should be equal to the config the node is running with. + ASSERT_TRUE(initial_bundle->has_distribution_config()); + EXPECT_EQ(to_string(initial_bundle->distribution_config_bundle()->config()), + _node->getComponentRegister().getDistribution()->serialized()); + auto currentNodeState = _manager->getCurrentNodeState(); EXPECT_EQ("s:d", currentNodeState->toString(false)); +} + +TEST_F(StateManagerTest, can_receive_state_bundle_without_distribution_config) { + ClusterState send_state("version:2 distributor:1 storage:4 .2.s:m"); + auto cmd = std::make_shared<api::SetSystemStateCommand>(send_state); + _upper->sendDown(cmd); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(send_state, *current_bundle->getBaselineClusterState()); + // Distribution config should be unchanged from bootstrap. + ASSERT_TRUE(current_bundle->has_distribution_config()); + EXPECT_EQ(to_string(current_bundle->distribution_config_bundle()->config()), + _node->getComponentRegister().getDistribution()->serialized()); + + auto current_node_state = _manager->getCurrentNodeState(); + EXPECT_EQ("s:m", current_node_state->toString(false)); +} + +TEST_F(StateManagerTest, can_receive_state_bundle_with_distribution_config) { + auto cmd = make_set_state_cmd_with_config("version:2 distributor:1 storage:4 .2.s:m", 5); + EXPECT_NE(to_string(cmd->getClusterStateBundle().distribution_config_bundle()->config()), + _node->getComponentRegister().getDistribution()->serialized()); + _upper->sendDown(cmd); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(*current_bundle, cmd->getClusterStateBundle()); // also compares distribution configs +} - ClusterState sendState("storage:4 .2.s:m"); - auto cmd = std::make_shared<api::SetSystemStateCommand>(sendState); +TEST_F(StateManagerTest, receiving_cc_bundle_with_distribution_config_disables_node_distribution_config_propagation) { + auto cmd = make_set_state_cmd_with_config("version:2 distributor:1 storage:4 .2.s:m", 5); _upper->sendDown(cmd); + std::shared_ptr<api::StorageReply> reply; ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + // Explicitly setting distribution config should not propagate to the active state bundle + // since we've flipped to expecting config from the cluster controllers instead. + auto distr = std::make_shared<lib::Distribution>(lib::Distribution::getDefaultDistributionConfig(2, 7)); + _node->getComponentRegister().setDistribution(distr); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(*current_bundle, cmd->getClusterStateBundle()); // unchanged +} + +TEST_F(StateManagerTest, internal_distribution_config_is_propagated_if_none_yet_received_from_cc) { + _upper->sendDown(make_set_state_cmd("version:10 distributor:1 storage:4", 0)); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + auto expected_bundle = make_state_bundle_with_config("version:10 distributor:1 storage:4", 7); + // Explicitly set internal config + _node->getComponentRegister().setDistribution(expected_bundle->distribution_config_bundle()->default_distribution_sp()); + _manager->storageDistributionChanged(); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(*current_bundle, *expected_bundle); +} + +TEST_F(StateManagerTest, revert_to_internal_config_if_cc_no_longer_sends_distribution_config) { + // Initial state bundle _with_ distribution config + auto cmd = make_set_state_cmd_with_config("version:2 distributor:1 storage:4 .2.s:m", 5); + _upper->sendDown(cmd); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(to_string(current_bundle->distribution_config_bundle()->config()), + to_string(cmd->getClusterStateBundle().distribution_config_bundle()->config())); + + // CC then sends a new bundle _without_ config + _upper->sendDown(make_set_state_cmd("version:3 distributor:1 storage:4", 0)); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + // Config implicitly reverted to the active internal config + current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(to_string(current_bundle->distribution_config_bundle()->config()), + _node->getComponentRegister().getDistribution()->serialized()); - currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); - EXPECT_EQ(sendState, *currentState); + // Explicitly set internal config + auto expected_bundle = make_state_bundle_with_config("version:3 distributor:1 storage:4", 7); + _node->getComponentRegister().setDistribution(expected_bundle->distribution_config_bundle()->default_distribution_sp()); + _manager->storageDistributionChanged(); - currentNodeState = _manager->getCurrentNodeState(); - EXPECT_EQ("s:m", currentNodeState->toString(false)); + // Internal config shall have taken effect, overriding that of the initial bundle + current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(*current_bundle, *expected_bundle); } TEST_F(StateManagerTest, accept_lower_state_versions_if_strict_requirement_disabled) { diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 1f20a19ec51..280afe8fb91 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -526,12 +526,12 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac using RBISP = std::shared_ptr<api::RequestBucketInfoCommand>; std::map<uint16_t, RBISP> requests; - // TODO fetch distribution from bundle as well - auto distribution(_component.getBucketSpaceRepo().get(bucketSpace).getDistribution()); - auto clusterStateBundle(_component.getStateUpdater().getClusterStateBundle()); - assert(clusterStateBundle); - lib::ClusterState::CSP clusterState(clusterStateBundle->getDerivedClusterState(bucketSpace)); - assert(clusterState.get()); + auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); + assert(clusterStateBundle && clusterStateBundle->has_distribution_config()); + auto clusterState = clusterStateBundle->getDerivedClusterState(bucketSpace); + assert(clusterState); + const auto distribution = clusterStateBundle->bucket_space_distribution_or_nullptr(bucketSpace); + assert(distribution); const auto our_hash = distribution->getNodeGraph().getDistributionConfigHash(); diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp index 92b5257b991..d66219794c7 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.cpp +++ b/storage/src/vespa/storage/common/content_bucket_space.cpp @@ -10,6 +10,7 @@ ClusterStateAndDistribution::ClusterStateAndDistribution( : _cluster_state(std::move(cluster_state)), _distribution(std::move(distribution)) { + assert(_cluster_state && _distribution); } ClusterStateAndDistribution::~ClusterStateAndDistribution() = default; @@ -48,34 +49,6 @@ ContentBucketSpace::state_and_distribution() const noexcept { return _state_and_distribution; } -void -ContentBucketSpace::setClusterState(std::shared_ptr<const lib::ClusterState> clusterState) -{ - std::lock_guard guard(_lock); - _state_and_distribution = _state_and_distribution->with_new_state(std::move(clusterState)); -} - -std::shared_ptr<const lib::ClusterState> -ContentBucketSpace::getClusterState() const -{ - std::lock_guard guard(_lock); - return _state_and_distribution->_cluster_state; -} - -void -ContentBucketSpace::setDistribution(std::shared_ptr<const lib::Distribution> distribution) -{ - std::lock_guard guard(_lock); - _state_and_distribution = _state_and_distribution->with_new_distribution(std::move(distribution)); -} - -std::shared_ptr<const lib::Distribution> -ContentBucketSpace::getDistribution() const -{ - std::lock_guard guard(_lock); - return _state_and_distribution->_distribution; -} - bool ContentBucketSpace::getNodeUpInLastNodeStateSeenByProvider() const { diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h index eb48640c97b..67ee4209d35 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.h +++ b/storage/src/vespa/storage/common/content_bucket_space.h @@ -54,14 +54,6 @@ public: void set_state_and_distribution(std::shared_ptr<const ClusterStateAndDistribution> state_and_distr) noexcept; [[nodiscard]] std::shared_ptr<const ClusterStateAndDistribution> state_and_distribution() const noexcept; - // TODO deprecate; only use atomic state+distribution setter - void setClusterState(std::shared_ptr<const lib::ClusterState> clusterState); - // TODO deprecate; only use atomic state+distribution getter - std::shared_ptr<const lib::ClusterState> getClusterState() const; - // TODO deprecate; only use atomic state+distribution setter - void setDistribution(std::shared_ptr<const lib::Distribution> distribution); - // TODO deprecate; only use atomic state+distribution getter - std::shared_ptr<const lib::Distribution> getDistribution() const; bool getNodeUpInLastNodeStateSeenByProvider() const; void setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider); diff --git a/storage/src/vespa/storage/common/storagecomponent.h b/storage/src/vespa/storage/common/storagecomponent.h index 677d5652fe3..ba12fc666a9 100644 --- a/storage/src/vespa/storage/common/storagecomponent.h +++ b/storage/src/vespa/storage/common/storagecomponent.h @@ -61,7 +61,7 @@ public: const std::shared_ptr<const document::FieldSetRepo> fieldSetRepo; }; using UP = std::unique_ptr<StorageComponent>; - using DistributionSP = std::shared_ptr<lib::Distribution>; + using DistributionSP = std::shared_ptr<const lib::Distribution>; /** * Node type is supposed to be set immediately, and never be updated. diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp index ab1cbf0b4d7..4d1daa329fd 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp @@ -23,11 +23,9 @@ ServiceLayerComponentRegisterImpl::registerServiceLayerComponent(ServiceLayerMan } void -ServiceLayerComponentRegisterImpl::setDistribution(std::shared_ptr<lib::Distribution> distribution) +ServiceLayerComponentRegisterImpl::setDistribution(std::shared_ptr<const lib::Distribution> distribution) { - _bucketSpaceRepo.get(document::FixedBucketSpaces::default_space()).setDistribution(distribution); - auto global_distr = lib::GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); - _bucketSpaceRepo.get(document::FixedBucketSpaces::global_space()).setDistribution(global_distr); + // TODO remove this override entirely? StorageComponentRegisterImpl::setDistribution(distribution); } diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index 1589192b92e..3478c0b3b9c 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -34,7 +34,7 @@ public: } void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; - void setDistribution(std::shared_ptr<lib::Distribution> distribution) override; + void setDistribution(std::shared_ptr<const lib::Distribution> distribution) override; }; } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp index bd0853a3524..ab1d80dc0b9 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp @@ -91,7 +91,7 @@ StorageComponentRegisterImpl::setBucketIdFactory(const document::BucketIdFactory } void -StorageComponentRegisterImpl::setDistribution(std::shared_ptr<lib::Distribution> distribution) +StorageComponentRegisterImpl::setDistribution(std::shared_ptr<const lib::Distribution> distribution) { std::lock_guard lock(_componentLock); _distribution = distribution; diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h index abb60051fe1..271b38399b6 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h @@ -21,16 +21,16 @@ class StorageComponentRegisterImpl { using BucketspacesConfig = vespa::config::content::core::internal::InternalBucketspacesType; - std::mutex _componentLock; - std::vector<StorageComponent*> _components; - vespalib::string _clusterName; - const lib::NodeType* _nodeType; - uint16_t _index; + std::mutex _componentLock; + std::vector<StorageComponent*> _components; + vespalib::string _clusterName; + const lib::NodeType* _nodeType; + uint16_t _index; std::shared_ptr<const document::DocumentTypeRepo> _docTypeRepo; - document::BucketIdFactory _bucketIdFactory; - std::shared_ptr<lib::Distribution> _distribution; - NodeStateUpdater* _nodeStateUpdater; - BucketspacesConfig _bucketSpacesConfig; + document::BucketIdFactory _bucketIdFactory; + std::shared_ptr<const lib::Distribution> _distribution; + NodeStateUpdater* _nodeStateUpdater; + BucketspacesConfig _bucketSpacesConfig; public: using UP = std::unique_ptr<StorageComponentRegisterImpl>; @@ -38,12 +38,12 @@ public: StorageComponentRegisterImpl(); ~StorageComponentRegisterImpl() override; - const lib::NodeType& getNodeType() const { return *_nodeType; } - uint16_t getIndex() const { return _index; } - std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() { return _docTypeRepo; } - const document::BucketIdFactory& getBucketIdFactory() { return _bucketIdFactory; } - std::shared_ptr<lib::Distribution> & getDistribution() { return _distribution; } - NodeStateUpdater& getNodeStateUpdater() { return *_nodeStateUpdater; } + [[nodiscard]] const lib::NodeType& getNodeType() const noexcept { return *_nodeType; } + [[nodiscard]] uint16_t getIndex() const noexcept { return _index; } + [[nodiscard]] std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() const noexcept { return _docTypeRepo; } + [[nodiscard]] const document::BucketIdFactory& getBucketIdFactory() const noexcept { return _bucketIdFactory; } + [[nodiscard]] const std::shared_ptr<const lib::Distribution>& getDistribution() const noexcept { return _distribution; } + [[nodiscard]] NodeStateUpdater& getNodeStateUpdater() noexcept { return *_nodeStateUpdater; } void registerStorageComponent(StorageComponent&) override; @@ -51,7 +51,7 @@ public: virtual void setNodeStateUpdater(NodeStateUpdater& updater); virtual void setDocumentTypeRepo(std::shared_ptr<const document::DocumentTypeRepo>); virtual void setBucketIdFactory(const document::BucketIdFactory&); - virtual void setDistribution(std::shared_ptr<lib::Distribution>); + virtual void setDistribution(std::shared_ptr<const lib::Distribution>); virtual void setBucketSpacesConfig(const BucketspacesConfig&); }; diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp index d5de11c7d6f..f571f6272b8 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp @@ -16,13 +16,14 @@ using document::BucketSpace; namespace storage { uint16_t -BucketOwnershipNotifier::getOwnerDistributorForBucket(const document::Bucket &bucket) const +BucketOwnershipNotifier::getOwnerDistributorForBucket(const document::Bucket& bucket) const { try { - // TODO use state updater bundle for everything? - auto distribution(_component.getBucketSpaceRepo().get(bucket.getBucketSpace()).getDistribution()); - const auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); - const auto &clusterState = *clusterStateBundle->getDerivedClusterState(bucket.getBucketSpace()); + const auto state_bundle = _component.getStateUpdater().getClusterStateBundle(); + assert(state_bundle && state_bundle->has_distribution_config()); + const auto* distribution = state_bundle->distribution_config_bundle()->bucket_space_distribution_or_nullptr_raw(bucket.getBucketSpace()); + assert(distribution); + const auto& clusterState = *state_bundle->getDerivedClusterState(bucket.getBucketSpace()); return (distribution->getIdealDistributorNode(clusterState, bucket.getBucketId())); // If we get exceptions there aren't any distributors, so they'll have // to explicitly fetch all bucket info eventually anyway. @@ -39,7 +40,7 @@ BucketOwnershipNotifier::getOwnerDistributorForBucket(const document::Bucket &bu } bool -BucketOwnershipNotifier::distributorOwns(uint16_t distributor, const document::Bucket &bucket) const +BucketOwnershipNotifier::distributorOwns(uint16_t distributor, const document::Bucket& bucket) const { return (distributor == getOwnerDistributorForBucket(bucket)); } @@ -47,7 +48,7 @@ BucketOwnershipNotifier::distributorOwns(uint16_t distributor, const document::B void BucketOwnershipNotifier::sendNotifyBucketToDistributor( uint16_t distributorIndex, - const document::Bucket &bucket, + const document::Bucket& bucket, const api::BucketInfo& infoToSend) { if (!infoToSend.valid()) { @@ -71,7 +72,7 @@ BucketOwnershipNotifier::sendNotifyBucketToDistributor( } void -BucketOwnershipNotifier::logNotification(const document::Bucket &bucket, +BucketOwnershipNotifier::logNotification(const document::Bucket& bucket, uint16_t sourceIndex, uint16_t currentOwnerIndex, const api::BucketInfo& newInfo) @@ -88,7 +89,7 @@ BucketOwnershipNotifier::logNotification(const document::Bucket &bucket, void BucketOwnershipNotifier::notifyIfOwnershipChanged( - const document::Bucket &bucket, + const document::Bucket& bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend) { @@ -111,7 +112,7 @@ BucketOwnershipNotifier::notifyIfOwnershipChanged( void BucketOwnershipNotifier::sendNotifyBucketToCurrentOwner( - const document::Bucket &bucket, + const document::Bucket& bucket, const api::BucketInfo& infoToSend) { uint16_t distributor(getOwnerDistributorForBucket(bucket)); @@ -134,7 +135,7 @@ NotificationGuard::~NotificationGuard() } void -NotificationGuard::notifyIfOwnershipChanged(const document::Bucket &bucket, +NotificationGuard::notifyIfOwnershipChanged(const document::Bucket& bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend) { @@ -142,7 +143,7 @@ NotificationGuard::notifyIfOwnershipChanged(const document::Bucket &bucket, } void -NotificationGuard::notifyAlways(const document::Bucket &bucket, +NotificationGuard::notifyAlways(const document::Bucket& bucket, const api::BucketInfo& infoToSend) { BucketToCheck bc(bucket, 0xffff, infoToSend); diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.h b/storage/src/vespa/storage/persistence/bucketownershipnotifier.h index fc3d9209f5f..93a2c1a2104 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.h +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.h @@ -11,29 +11,29 @@ namespace storage { class BucketOwnershipNotifier { - const ServiceLayerComponent & _component; - MessageSender & _sender; + const ServiceLayerComponent& _component; + MessageSender& _sender; public: BucketOwnershipNotifier(const ServiceLayerComponent& component, MessageSender& sender) : _component(component), _sender(sender) {} - bool distributorOwns(uint16_t distributor, const document::Bucket &bucket) const; - void notifyIfOwnershipChanged(const document::Bucket &bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend); - void sendNotifyBucketToCurrentOwner(const document::Bucket &bucket, const api::BucketInfo& infoToSend); + [[nodiscard]] bool distributorOwns(uint16_t distributor, const document::Bucket& bucket) const; + void notifyIfOwnershipChanged(const document::Bucket& bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend); + void sendNotifyBucketToCurrentOwner(const document::Bucket& bucket, const api::BucketInfo& infoToSend); private: enum IndexMeta { FAILED_TO_RESOLVE = 0xffff }; - void sendNotifyBucketToDistributor(uint16_t distributorIndex, const document::Bucket &bucket, + void sendNotifyBucketToDistributor(uint16_t distributorIndex, const document::Bucket& bucket, const api::BucketInfo& infoToSend); // Returns either index or FAILED_TO_RESOLVE uint16_t getOwnerDistributorForBucket(const document::Bucket &bucket) const; - void logNotification(const document::Bucket &bucket, uint16_t sourceIndex, + void logNotification(const document::Bucket& bucket, uint16_t sourceIndex, uint16_t currentOwnerIndex, const api::BucketInfo& newInfo); }; @@ -60,7 +60,7 @@ class NotificationGuard BucketOwnershipNotifier& _notifier; std::vector<BucketToCheck> _bucketsToCheck; public: - NotificationGuard(BucketOwnershipNotifier& notifier) + explicit NotificationGuard(BucketOwnershipNotifier& notifier) : _notifier(notifier), _bucketsToCheck() {} @@ -69,8 +69,8 @@ public: ~NotificationGuard(); - void notifyIfOwnershipChanged(const document::Bucket &bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend); - void notifyAlways(const document::Bucket &bucket, const api::BucketInfo& infoToSend); + void notifyIfOwnershipChanged(const document::Bucket& bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend); + void notifyAlways(const document::Bucket& bucket, const api::BucketInfo& infoToSend); }; } // storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 495497d507d..e44490388e2 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -887,9 +887,9 @@ bool FileStorManager::maintenance_in_all_spaces(const lib::Node& node) const noexcept { for (const auto& elem : _component.getBucketSpaceRepo()) { - const ContentBucketSpace& bucket_space = *elem.second; - auto derived_cluster_state = bucket_space.getClusterState(); - if (!derived_cluster_state->getNodeState(node).getState().oneOf("m")) { + const auto space_state_and_distr = elem.second->state_and_distribution(); + const auto& derived_cluster_state = space_state_and_distr->cluster_state(); + if (!derived_cluster_state.getNodeState(node).getState().oneOf("m")) { return false; } } @@ -915,8 +915,7 @@ FileStorManager::maybe_log_received_cluster_state() { if (LOG_WOULD_LOG(debug)) { auto cluster_state_bundle = _component.getStateUpdater().getClusterStateBundle(); - auto baseline_state = cluster_state_bundle->getBaselineClusterState(); - LOG(debug, "FileStorManager received baseline cluster state '%s'", baseline_state->toString().c_str()); + LOG(debug, "FileStorManager received baseline cluster state '%s'", cluster_state_bundle->toString().c_str()); } } @@ -951,16 +950,17 @@ FileStorManager::updateState() void FileStorManager::storageDistributionChanged() { - updateState(); } void FileStorManager::propagateClusterStates() { auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); - for (const auto &elem : _component.getBucketSpaceRepo()) { - // TODO also distribution! bundle and repo must be 1-1 - elem.second->setClusterState(clusterStateBundle->getDerivedClusterState(elem.first)); + assert(clusterStateBundle->has_distribution_config()); + for (const auto& elem : _component.getBucketSpaceRepo()) { + elem.second->set_state_and_distribution(std::make_shared<ClusterStateAndDistribution>( + clusterStateBundle->getDerivedClusterState(elem.first), + clusterStateBundle->bucket_space_distribution_or_nullptr(elem.first))); } } diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp index 25829f3d391..8702cb1272b 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp @@ -31,10 +31,11 @@ ChangedBucketOwnershipHandler::ChangedBucketOwnershipHandler( _stateLock(), _currentState(), // Not set yet, so ownership will not be valid _currentOwnership(std::make_shared<OwnershipState>( - _component.getBucketSpaceRepo(), _currentState)), + _currentState, lib::DistributionConfigBundle::of(_component.getDistribution()))), _abortQueuedAndPendingOnStateChange(false), _abortMutatingIdealStateOps(false), - _abortMutatingExternalLoadOps(false) + _abortMutatingExternalLoadOps(false), + _receiving_distribution_config_from_cc(false) { on_configure(bootstrap_config); _component.registerMetric(_metrics); @@ -61,16 +62,29 @@ ChangedBucketOwnershipHandler::reloadClusterState() { std::lock_guard guard(_stateLock); const auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); - setCurrentOwnershipWithStateNoLock(*clusterStateBundle); + setCurrentOwnershipWithStateNoLock(std::move(clusterStateBundle)); } void -ChangedBucketOwnershipHandler::setCurrentOwnershipWithStateNoLock( - const lib::ClusterStateBundle& newState) +ChangedBucketOwnershipHandler::setCurrentOwnershipWithStateNoLock(std::shared_ptr<const lib::ClusterStateBundle> new_state) { - _currentState = std::make_shared<const lib::ClusterStateBundle>(newState); - _currentOwnership = std::make_shared<const OwnershipState>( - _component.getBucketSpaceRepo(), _currentState); + LOG(debug, "Setting new ownership state bundle: %s", new_state->toString().c_str()); + std::shared_ptr<const lib::DistributionConfigBundle> distributions; + _currentState = std::move(new_state); + // This partially duplicates distribution config fallback logic from StateManager, but that's because + // this component has the same approach to intercepting both state commands and distribution config + // changes. + // `new_state` can come straight from a SetSystemStateCommand, which may or may not have a state bundle + // with distribution config strapped to it. + if (_currentState->has_distribution_config()) { + distributions = _currentState->distribution_config_bundle(); + } else { + distributions = lib::DistributionConfigBundle::of(_component.getDistribution()); + LOG(debug, "No distribution config in bundle; using current host config of '%s'", + distributions->default_distribution().getNodeGraph().getDistributionConfigHash().c_str()); + } + _receiving_distribution_config_from_cc = _currentState->has_distribution_config(); + _currentOwnership = std::make_shared<const OwnershipState>(_currentState, std::move(distributions)); } namespace { @@ -98,17 +112,11 @@ ChangedBucketOwnershipHandler::Metrics::Metrics(metrics::MetricSet* owner) {} ChangedBucketOwnershipHandler::Metrics::~Metrics() = default; -ChangedBucketOwnershipHandler::OwnershipState::OwnershipState(const ContentBucketSpaceRepo &contentBucketSpaceRepo, - std::shared_ptr<const lib::ClusterStateBundle> state) - : _distributions(), - _state(std::move(state)) +ChangedBucketOwnershipHandler::OwnershipState::OwnershipState(std::shared_ptr<const lib::ClusterStateBundle> state, + std::shared_ptr<const lib::DistributionConfigBundle> distributions) + : _state(std::move(state)), + _distributions(std::move(distributions)) { - for (const auto &elem : contentBucketSpaceRepo) { - auto distribution = elem.second->getDistribution(); - if (distribution) { - _distributions.emplace(elem.first, std::move(distribution)); - } - } } @@ -123,19 +131,15 @@ ChangedBucketOwnershipHandler::OwnershipState::getBaselineState() const } uint16_t -ChangedBucketOwnershipHandler::OwnershipState::ownerOf( - const document::Bucket& bucket) const +ChangedBucketOwnershipHandler::OwnershipState::ownerOf(const document::Bucket& bucket) const { - auto distributionItr = _distributions.find(bucket.getBucketSpace()); - assert(distributionItr != _distributions.end()); - const auto &distribution = *distributionItr->second; - const auto &derivedState = *_state->getDerivedClusterState(bucket.getBucketSpace()); + const auto* distribution = _distributions->bucket_space_distribution_or_nullptr_raw(bucket.getBucketSpace()); + assert(distribution); + const auto& derivedState = *_state->getDerivedClusterState(bucket.getBucketSpace()); try { - return distribution.getIdealDistributorNode(derivedState, bucket.getBucketId()); + return distribution->getIdealDistributorNode(derivedState, bucket.getBucketId()); } catch (lib::TooFewBucketBitsInUseException& e) { - LOGBP(debug, - "Too few bucket bits used for %s to be assigned to " - "a distributor.", + LOGBP(debug, "Too few bucket bits used for %s to be assigned to a distributor.", bucket.toString().c_str()); } catch (lib::NoDistributorsAvailableException& e) { LOGBP(warning, @@ -144,11 +148,9 @@ ChangedBucketOwnershipHandler::OwnershipState::ownerOf( "for available distributors before reaching this code path! " "Cluster state is '%s', distribution is '%s'", derivedState.toString().c_str(), - distribution.toString().c_str()); + distribution->toString().c_str()); } catch (const std::exception& e) { - LOG(error, - "Got unknown exception while resolving distributor: %s", - e.what()); + LOG(error, "Got unknown exception while resolving distributor: %s", e.what()); } return FAILED_TO_RESOLVE; } @@ -162,9 +164,7 @@ ChangedBucketOwnershipHandler::OwnershipState::storageNodeUp(document::BucketSpa } void -ChangedBucketOwnershipHandler::logTransition( - const lib::ClusterState& currentState, - const lib::ClusterState& newState) const +ChangedBucketOwnershipHandler::logTransition(const lib::ClusterState& currentState, const lib::ClusterState& newState) { LOG(debug, "State transition '%s' -> '%s' changes distributor bucket ownership, " @@ -269,7 +269,7 @@ public: { std::lock_guard guard(_owner._stateLock); old_ownership = _owner._currentOwnership; - _owner.setCurrentOwnershipWithStateNoLock(_command->getClusterStateBundle()); + _owner.setCurrentOwnershipWithStateNoLock(_command->cluster_state_bundle_ptr()); new_ownership = _owner._currentOwnership; } assert(new_ownership->valid()); @@ -287,7 +287,7 @@ public: new_ownership->getBaselineState().toString().c_str()); return _owner.sendDown(_command);; } - _owner.logTransition(old_ownership->getBaselineState(), new_ownership->getBaselineState()); + logTransition(old_ownership->getBaselineState(), new_ownership->getBaselineState()); metrics::MetricTimer duration_timer; auto predicate = _owner.makeLazyAbortPredicate(old_ownership, new_ownership); @@ -327,17 +327,19 @@ ChangedBucketOwnershipHandler::onSetSystemState( * Invoked whenever a distribution config change happens and is called in the * context of the config updater thread (which is why we have to lock). */ + // TODO remove this when there are no more state bundles without distribution config void ChangedBucketOwnershipHandler::storageDistributionChanged() { std::lock_guard guard(_stateLock); - _currentOwnership = std::make_shared<OwnershipState>( - _component.getBucketSpaceRepo(), _currentState); + if (!_receiving_distribution_config_from_cc) { + _currentOwnership = std::make_shared<OwnershipState>( + _currentState, lib::DistributionConfigBundle::of(_component.getDistribution())); + } } bool -ChangedBucketOwnershipHandler::isMutatingIdealStateOperation( - const api::StorageMessage& msg) const +ChangedBucketOwnershipHandler::isMutatingIdealStateOperation(const api::StorageMessage& msg) { switch (msg.getType().getId()) { case api::MessageType::CREATEBUCKET_ID: @@ -357,8 +359,7 @@ ChangedBucketOwnershipHandler::isMutatingIdealStateOperation( bool -ChangedBucketOwnershipHandler::isMutatingExternalOperation( - const api::StorageMessage& msg) const +ChangedBucketOwnershipHandler::isMutatingExternalOperation(const api::StorageMessage& msg) { switch (msg.getType().getId()) { case api::MessageType::PUT_ID: @@ -418,8 +419,7 @@ ChangedBucketOwnershipHandler::abortOperation(api::StorageCommand& cmd) } bool -ChangedBucketOwnershipHandler::isMutatingCommandAndNeedsChecking( - const api::StorageMessage& msg) const +ChangedBucketOwnershipHandler::isMutatingCommandAndNeedsChecking(const api::StorageMessage& msg) const { if (enabledIdealStateAborting() && isMutatingIdealStateOperation(msg)) { return true; diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h index 801534385f7..2c32704d64b 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h @@ -25,6 +25,7 @@ namespace lib { class ClusterState; class ClusterStateBundle; class Distribution; + class DistributionConfigBundle; } /** @@ -77,20 +78,20 @@ public: */ class OwnershipState { using BucketSpace = document::BucketSpace; - std::unordered_map<BucketSpace, std::shared_ptr<const lib::Distribution>, BucketSpace::hash> _distributions; std::shared_ptr<const lib::ClusterStateBundle> _state; + std::shared_ptr<const lib::DistributionConfigBundle> _distributions; public: using SP = std::shared_ptr<OwnershipState>; using CSP = std::shared_ptr<const OwnershipState>; - OwnershipState(const ContentBucketSpaceRepo &contentBucketSpaceRepo, - std::shared_ptr<const lib::ClusterStateBundle> state); + OwnershipState(std::shared_ptr<const lib::ClusterStateBundle> state, + std::shared_ptr<const lib::DistributionConfigBundle> distributions); ~OwnershipState(); static const uint16_t FAILED_TO_RESOLVE = 0xffff; [[nodiscard]] bool valid() const noexcept { - return (!_distributions.empty() && _state); + return (_distributions && _state); } /** @@ -124,20 +125,21 @@ private: std::atomic<bool> _abortQueuedAndPendingOnStateChange; std::atomic<bool> _abortMutatingIdealStateOps; std::atomic<bool> _abortMutatingExternalLoadOps; + bool _receiving_distribution_config_from_cc; std::unique_ptr<AbortBucketOperationsCommand::AbortPredicate> makeLazyAbortPredicate( const OwnershipState::CSP& oldOwnership, const OwnershipState::CSP& newOwnership) const; - void logTransition(const lib::ClusterState& currentState, - const lib::ClusterState& newState) const; + static void logTransition(const lib::ClusterState& currentState, + const lib::ClusterState& newState); /** * Creates a new immutable OwnershipState based on the current distribution * and the provided cluster state and assigns it to _currentOwnership. */ - void setCurrentOwnershipWithStateNoLock(const lib::ClusterStateBundle&); + void setCurrentOwnershipWithStateNoLock(std::shared_ptr<const lib::ClusterStateBundle>); /** * Grabs _stateLock and returns a shared_ptr to the current ownership @@ -147,9 +149,9 @@ private: bool isMutatingCommandAndNeedsChecking(const api::StorageMessage&) const; - bool isMutatingIdealStateOperation(const api::StorageMessage&) const; + static bool isMutatingIdealStateOperation(const api::StorageMessage&); - bool isMutatingExternalOperation(const api::StorageMessage&) const; + static bool isMutatingExternalOperation(const api::StorageMessage&); /** * Returns whether the operation in cmd has a bucket whose ownership in * the current cluster state does not match the distributor marked as diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index 417d8ad75dd..f8b4f9ef42d 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -210,7 +210,7 @@ void StorageApiRpcService::encode_rpc_v1_response(FRT_RPCRequest& request, api:: // TODO skip encoding header altogether if no relevant fields set? protobuf::ResponseHeader hdr; if (reply.getTrace().getLevel() > 0) { - hdr.set_trace_payload(reply.getTrace().encode()); + hdr.set_trace_payload(static_cast<std::string_view>(reply.getTrace().encode())); } // TODO consistent naming... encode_header_into_rpc_params(hdr, *ret); diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index a2106dce8d2..560982fb6c0 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -29,7 +29,18 @@ using vespalib::make_string_short::fmt; namespace storage { namespace { - constexpr vespalib::duration MAX_TIMEOUT = 600s; +constexpr vespalib::duration MAX_TIMEOUT = 600s; + +[[nodiscard]] std::shared_ptr<const lib::ClusterStateBundle> +make_bootstrap_state_bundle(std::shared_ptr<const lib::Distribution> config) { + return std::make_shared<const lib::ClusterStateBundle>( + std::make_shared<lib::ClusterState>(), + lib::ClusterStateBundle::BucketSpaceStateMapping{}, + std::nullopt, + lib::DistributionConfigBundle::of(std::move(config)), + false); +} + } struct StateManager::StateManagerMetrics : metrics::MetricSet { @@ -62,7 +73,8 @@ StateManager::StateManager(StorageComponentRegister& compReg, _listenerLock(), _nodeState(std::make_shared<lib::NodeState>(_component.getNodeType(), lib::State::DOWN)), _nextNodeState(), - _systemState(std::make_shared<const ClusterStateBundle>(lib::ClusterState())), + _configured_distribution(_component.getDistribution()), + _systemState(make_bootstrap_state_bundle(_configured_distribution)), _nextSystemState(), _reported_host_info_cluster_state_version(0), _stateListeners(), @@ -81,6 +93,7 @@ StateManager::StateManager(StorageComponentRegister& compReg, _noThreadTestMode(testMode), _grabbedExternalLock(false), _require_strictly_increasing_cluster_state_versions(false), + _receiving_distribution_config_from_cc(false), _notifyingListeners(false), _requested_almost_immediate_node_state_replies(false) { @@ -170,8 +183,7 @@ lib::NodeState::CSP StateManager::getCurrentNodeState() const { std::lock_guard lock(_stateLock); - return std::make_shared<const lib::NodeState> - (_systemState->getBaselineClusterState()->getNodeState(thisNode())); + return std::make_shared<const lib::NodeState>(_systemState->getBaselineClusterState()->getNodeState(thisNode())); } std::shared_ptr<const lib::ClusterStateBundle> @@ -181,6 +193,28 @@ StateManager::getClusterStateBundle() const return _systemState; } +// TODO remove when distribution config is only received from cluster controller +void +StateManager::storageDistributionChanged() +{ + { + std::lock_guard lock(_stateLock); + _configured_distribution = _component.getDistribution(); + if (_receiving_distribution_config_from_cc) { + return; // nothing more to do + } + // Avoid losing any pending state if this callback happens in the middle of a + // state update. This edge case is practically impossible to unit test today... + const auto patch_state = _nextSystemState ? _nextSystemState : _systemState; + _nextSystemState = patch_state->clone_with_new_distribution( + lib::DistributionConfigBundle::of(_configured_distribution)); + } + // We've assembled a new state bundle based on the (non-distribution carrying) state + // bundle from the cluster controller and our own internal config. Propagate it as one + // unit to the internal components. + notifyStateListeners(); +} + void StateManager::addStateListener(StateListener& listener) { @@ -316,12 +350,12 @@ using BucketSpaceToTransitionString = std::unordered_map<document::BucketSpace, document::BucketSpace::hash>; void -considerInsertDerivedTransition(const lib::State ¤tBaseline, - const lib::State &newBaseline, - const lib::State ¤tDerived, - const lib::State &newDerived, - const document::BucketSpace &bucketSpace, - BucketSpaceToTransitionString &transitions) +considerInsertDerivedTransition(const lib::State& currentBaseline, + const lib::State& newBaseline, + const lib::State& currentDerived, + const lib::State& newDerived, + const document::BucketSpace& bucketSpace, + BucketSpaceToTransitionString& transitions) { bool considerDerivedTransition = ((currentDerived != newDerived) && ((currentDerived != currentBaseline) || (newDerived != newBaseline))); @@ -333,28 +367,28 @@ considerInsertDerivedTransition(const lib::State ¤tBaseline, } BucketSpaceToTransitionString -calculateDerivedClusterStateTransitions(const ClusterStateBundle ¤tState, - const ClusterStateBundle &newState, +calculateDerivedClusterStateTransitions(const ClusterStateBundle& currentState, + const ClusterStateBundle& newState, const lib::Node node) { BucketSpaceToTransitionString result; - const lib::State ¤tBaseline = currentState.getBaselineClusterState()->getNodeState(node).getState(); - const lib::State &newBaseline = newState.getBaselineClusterState()->getNodeState(node).getState(); - for (const auto &entry : currentState.getDerivedClusterStates()) { - const lib::State ¤tDerived = entry.second->getNodeState(node).getState(); - const lib::State &newDerived = newState.getDerivedClusterState(entry.first)->getNodeState(node).getState(); + const lib::State& currentBaseline = currentState.getBaselineClusterState()->getNodeState(node).getState(); + const lib::State& newBaseline = newState.getBaselineClusterState()->getNodeState(node).getState(); + for (const auto& entry : currentState.getDerivedClusterStates()) { + const lib::State& currentDerived = entry.second->getNodeState(node).getState(); + const lib::State& newDerived = newState.getDerivedClusterState(entry.first)->getNodeState(node).getState(); considerInsertDerivedTransition(currentBaseline, newBaseline, currentDerived, newDerived, entry.first, result); } - for (const auto &entry : newState.getDerivedClusterStates()) { - const lib::State &newDerived = entry.second->getNodeState(node).getState(); - const lib::State ¤tDerived = currentState.getDerivedClusterState(entry.first)->getNodeState(node).getState(); + for (const auto& entry : newState.getDerivedClusterStates()) { + const lib::State& newDerived = entry.second->getNodeState(node).getState(); + const lib::State& currentDerived = currentState.getDerivedClusterState(entry.first)->getNodeState(node).getState(); considerInsertDerivedTransition(currentBaseline, newBaseline, currentDerived, newDerived, entry.first, result); } return result; } vespalib::string -transitionsToString(const BucketSpaceToTransitionString &transitions) +transitionsToString(const BucketSpaceToTransitionString& transitions) { if (transitions.empty()) { return ""; @@ -362,7 +396,7 @@ transitionsToString(const BucketSpaceToTransitionString &transitions) vespalib::asciistream stream; stream << "["; bool first = true; - for (const auto &entry : transitions) { + for (const auto& entry : transitions) { if (!first) { stream << ", "; } @@ -435,7 +469,7 @@ StateManager::onGetNodeState(const api::GetNodeStateCommand::SP& cmd) } void -StateManager::mark_controller_as_having_observed_explicit_node_state(const std::unique_lock<std::mutex> &, uint16_t controller_index) { +StateManager::mark_controller_as_having_observed_explicit_node_state(const std::unique_lock<std::mutex>&, uint16_t controller_index) { _controllers_observed_explicit_node_state.emplace(controller_index); } @@ -483,7 +517,16 @@ StateManager::try_set_cluster_state_bundle(std::shared_ptr<const ClusterStateBun } } _last_accepted_cluster_state_time = now; - _nextSystemState = std::move(c); + _receiving_distribution_config_from_cc = c->has_distribution_config(); + if (!c->has_distribution_config()) { + LOG(debug, "Next state bundle '%s' does not have distribution config; patching in existing config '%s'", + c->toString().c_str(), _configured_distribution->getNodeGraph().getDistributionConfigHash().c_str()); + _nextSystemState = c->clone_with_new_distribution(lib::DistributionConfigBundle::of(_configured_distribution)); + } else { + LOG(debug, "Next state bundle is '%s'", c->toString().c_str()); + // TODO print what's changed in distribution config? + _nextSystemState = std::move(c); + } } notifyStateListeners(); return std::nullopt; diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index d116f968731..d7a139f7ced 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -55,6 +55,7 @@ class StateManager : public NodeStateUpdater, std::mutex _listenerLock; std::shared_ptr<lib::NodeState> _nodeState; std::shared_ptr<lib::NodeState> _nextNodeState; + std::shared_ptr<const lib::Distribution> _configured_distribution; // From config system, not from CC std::shared_ptr<const ClusterStateBundle> _systemState; std::shared_ptr<const ClusterStateBundle> _nextSystemState; uint32_t _reported_host_info_cluster_state_version; @@ -78,6 +79,7 @@ class StateManager : public NodeStateUpdater, bool _noThreadTestMode; bool _grabbedExternalLock; bool _require_strictly_increasing_cluster_state_versions; + bool _receiving_distribution_config_from_cc; std::atomic<bool> _notifyingListeners; std::atomic<bool> _requested_almost_immediate_node_state_replies; @@ -102,6 +104,8 @@ public: lib::NodeState::CSP getCurrentNodeState() const override; std::shared_ptr<const ClusterStateBundle> getClusterStateBundle() const override; + void storageDistributionChanged() override; + void addStateListener(StateListener&) override; void removeStateListener(StateListener&) override; diff --git a/vbench/src/tests/app_vbench/app_vbench_test.cpp b/vbench/src/tests/app_vbench/app_vbench_test.cpp index 97ab828fc66..7353784fb96 100644 --- a/vbench/src/tests/app_vbench/app_vbench_test.cpp +++ b/vbench/src/tests/app_vbench/app_vbench_test.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vbench/test/all.h> #include <vespa/vespalib/process/process.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp b/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp index 09c2d475224..d35d4ea33da 100644 --- a/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp +++ b/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vbench/test/all.h> using namespace vbench; TEST_FF("dropped tagger", RequestReceptor(), DroppedTagger(f1)) { - Request::UP req(new Request()); + auto req = std::make_unique<Request>(); EXPECT_EQUAL(Request::STATUS_OK, req->status()); f2.handle(std::move(req)); ASSERT_TRUE(f1.request.get() != 0); diff --git a/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h b/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h index 69d94869f5b..a62c1f33e88 100644 --- a/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h +++ b/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h @@ -21,6 +21,11 @@ struct BucketSpaceDistributionConfigs { return (iter != space_configs.end()) ? iter->second : std::shared_ptr<const Distribution>(); } + [[nodiscard]] const Distribution* get_or_nullptr_raw(document::BucketSpace space) const noexcept { + auto iter = space_configs.find(space); + return (iter != space_configs.end()) ? iter->second.get() : nullptr; + } + static BucketSpaceDistributionConfigs from_default_distribution(std::shared_ptr<const Distribution>); }; diff --git a/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h b/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h index 7b68d4fb6aa..61287427274 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h +++ b/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h @@ -33,6 +33,9 @@ public: [[nodiscard]] std::shared_ptr<const Distribution> bucket_space_distribution_or_nullptr(document::BucketSpace space) const noexcept { return _bucket_space_distributions.get_or_nullptr(space); } + [[nodiscard]] const Distribution* bucket_space_distribution_or_nullptr_raw(document::BucketSpace space) const noexcept { + return _bucket_space_distributions.get_or_nullptr_raw(space); + } [[nodiscard]] const BucketSpaceDistributionConfigs& bucket_space_distributions() const noexcept { return _bucket_space_distributions; } diff --git a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp index 6ef971b4c46..536de9c895e 100644 --- a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp +++ b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp @@ -22,9 +22,8 @@ ClusterStateBundle::FeedBlock::operator==(const FeedBlock& rhs) const noexcept (_description == rhs._description); } -// TODO implement with ctor fwd -ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState) - : _baselineClusterState(std::make_shared<const ClusterState>(baselineClusterState)), +ClusterStateBundle::ClusterStateBundle(std::shared_ptr<const ClusterState> baseline_cluster_state) + : _baselineClusterState(std::move(baseline_cluster_state)), _derivedBucketSpaceStates(), _feed_block(), _distribution_bundle(), @@ -32,6 +31,11 @@ ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState) { } +ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState) + : ClusterStateBundle(std::make_shared<const ClusterState>(baselineClusterState)) +{ +} + ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState, BucketSpaceStateMapping derivedBucketSpaceStates) : _baselineClusterState(std::make_shared<const ClusterState>(baselineClusterState)), @@ -108,6 +112,14 @@ ClusterStateBundle::getDerivedClusterState(document::BucketSpace bucketSpace) co return _baselineClusterState; } +std::shared_ptr<const Distribution> +ClusterStateBundle::bucket_space_distribution_or_nullptr(document::BucketSpace space) const noexcept { + if (!_distribution_bundle) { + return {}; + } + return _distribution_bundle->bucket_space_distribution_or_nullptr(space); +} + uint32_t ClusterStateBundle::getVersion() const { diff --git a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h index 18f82176dd5..e6763a689d8 100644 --- a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h +++ b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h @@ -52,6 +52,7 @@ public: std::shared_ptr<const DistributionConfigBundle> _distribution_bundle; bool _deferredActivation; public: + explicit ClusterStateBundle(std::shared_ptr<const ClusterState> baseline_cluster_state); explicit ClusterStateBundle(const ClusterState& baselineClusterState); ClusterStateBundle(const ClusterState& baselineClusterState, BucketSpaceStateMapping derivedBucketSpaceStates); @@ -98,6 +99,7 @@ public: [[nodiscard]] const std::shared_ptr<const DistributionConfigBundle>& distribution_config_bundle() const noexcept { return _distribution_bundle; } + [[nodiscard]] std::shared_ptr<const Distribution> bucket_space_distribution_or_nullptr(document::BucketSpace space) const noexcept; [[nodiscard]] const std::optional<FeedBlock>& feed_block() const { return _feed_block; } [[nodiscard]] uint32_t getVersion() const; [[nodiscard]] bool deferredActivation() const noexcept { return _deferredActivation; } diff --git a/vespa_fsa/OWNERS b/vespa_fsa/OWNERS deleted file mode 100644 index 7ae1acb1be9..00000000000 --- a/vespa_fsa/OWNERS +++ /dev/null @@ -1 +0,0 @@ -geirst diff --git a/vespa_fsa/README b/vespa_fsa/README deleted file mode 100644 index a5437e8b651..00000000000 --- a/vespa_fsa/README +++ /dev/null @@ -1,2 +0,0 @@ -This is the FSA library and tools, an implementation of finite state -automata (FSA) and related algorithms. diff --git a/vespa_qrs/OWNERS b/vespa_qrs/OWNERS deleted file mode 100644 index 67cd2820bb8..00000000000 --- a/vespa_qrs/OWNERS +++ /dev/null @@ -1 +0,0 @@ -arnej27959 diff --git a/vespa_qrs/README b/vespa_qrs/README deleted file mode 100644 index f472f2dc860..00000000000 --- a/vespa_qrs/README +++ /dev/null @@ -1,3 +0,0 @@ -VESPA Query/Response Server - -Query processing, backend handling, and result processing for the VESPA platform. diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index 4c0c479e37a..2c08ff81354 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -20,6 +20,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/conf/default-env.txt DESTINATION conf/ install(DIRECTORY DESTINATION logs/vespa) install(DIRECTORY DESTINATION logs/vespa/access) +install(DIRECTORY DESTINATION secure) install(DIRECTORY DESTINATION var/crash) install(DIRECTORY DESTINATION var/db/vespa) install(DIRECTORY DESTINATION var/db/vespa/config_server) diff --git a/vespaclient-java/CMakeLists.txt b/vespaclient-java/CMakeLists.txt index 6db6c1c78c4..0c383a56233 100644 --- a/vespaclient-java/CMakeLists.txt +++ b/vespaclient-java/CMakeLists.txt @@ -13,4 +13,5 @@ vespa_install_script(src/main/sh/vespa-visit.sh vespa-visit bin) vespa_install_script(src/main/sh/vespa-visit-target.sh vespa-visit-target bin) vespa_install_script(src/main/sh/vespa-feed-perf vespa-feed-perf bin) vespa_install_script(src/main/sh/vespa-status-filedistribution.sh vespa-status-filedistribution bin) +vespa_install_script(src/main/sh/vespa-significance.sh vespa-significance bin) vespa_install_script(src/main/sh/vespa-curl-wrapper vespa-curl-wrapper libexec/vespa) diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java index e368eebefa5..3bd836d7a14 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java @@ -34,7 +34,6 @@ public class Main { } else { System.setProperty("vespa.replace_invalid_unicode", "true"); SignificanceModelGenerator significanceModelGenerator = createSignificanceModelGenerator(params); - significanceModelGenerator.generate(); } } catch (IllegalArgumentException e) { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java index d620820e14f..3f257fc6df0 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java @@ -44,6 +44,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.List; +import java.util.Arrays; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -56,7 +58,9 @@ public class SignificanceModelGenerator { private final Tokenizer tokenizer; private final TreeMap<String, Long> documentFrequency = new TreeMap<>(); - private final Language language; + private final List<Language> languages; + + private final Language languageTag; private final ObjectMapper objectMapper; private final static JsonFactory parserFactory = new JsonFactory(); @@ -75,11 +79,16 @@ public class SignificanceModelGenerator { throw new IllegalArgumentException("Output file must have .zst extension when using zst compression"); } - language = Language.fromLanguageTag(clientParameters.language); - if (language == Language.UNKNOWN) { - throw new IllegalArgumentException("Unknown language: " + clientParameters.language); + if (!clientParameters.zstCompression && clientParameters.outputFile.endsWith(".zst")) { + throw new IllegalArgumentException("Output file must not have .zst extension when not using zst compression"); } + this.languages = Arrays.stream(clientParameters.language.split(",")) + .map(Language::fromLanguageTag) + .collect(Collectors.toList()); + + this.languageTag = this.languages.get(0); + OpenNlpLinguistics openNlpLinguistics = new OpenNlpLinguistics(); tokenizer = openNlpLinguistics.getTokenizer(); objectMapper = new ObjectMapper(); @@ -104,7 +113,7 @@ public class SignificanceModelGenerator { while (reader.ready()) { String line = reader.readLine(); JsonReader jsonReader = new JsonReader(types, new ByteArrayInputStream(Utf8.toBytes(line)), parserFactory); - String wikimediaId = "id:wikimedia:" + language.languageCode() + "::" + i; + String wikimediaId = "id:wikimedia:" + languageTag.languageCode() + "::" + i; ParsedDocumentOperation operation = jsonReader.readSingleDocumentStreaming(DocumentOperationType.PUT, wikimediaId); DocumentPut put = (DocumentPut) operation.operation(); @@ -118,6 +127,7 @@ public class SignificanceModelGenerator { SignificanceModelFile modelFile; File outputFile = Paths.get(clientParameters.outputFile).toFile(); + String languagesKey = String.join(",", this.languages.stream().map(Language::languageCode).toList()); if (outputFile.exists()) { InputStream in = outputFile.toString().endsWith(".zst") ? @@ -126,11 +136,11 @@ public class SignificanceModelGenerator { modelFile = objectMapper.readValue(in, SignificanceModelFile.class); - modelFile.addLanguage(clientParameters.language, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); + modelFile.addLanguage(languagesKey, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); } else { HashMap<String, DocumentFrequencyFile> languages = new HashMap<>() {{ - put(clientParameters.language, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); + put(languagesKey, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); }}; modelFile = new SignificanceModelFile(VERSION, ID, SIGNIFICANCE_DESCRIPTION + clientParameters.inputFile, languages); @@ -149,7 +159,7 @@ public class SignificanceModelGenerator { } private void handleTokenization(String field) { - var tokens = tokenizer.tokenize(field, language, StemMode.ALL, false); + var tokens = tokenizer.tokenize(field, languageTag, StemMode.ALL, false); Set<String> uniqueWords = StreamSupport.stream(tokens.spliterator(), false) .filter(t -> t.getType() == TokenType.ALPHABETIC) diff --git a/vespaclient-java/src/main/sh/vespa-significance.sh b/vespaclient-java/src/main/sh/vespa-significance.sh index f26c8dfca46..dfbf5b90c2e 100755 --- a/vespaclient-java/src/main/sh/vespa-significance.sh +++ b/vespaclient-java/src/main/sh/vespa-significance.sh @@ -79,7 +79,7 @@ export ROOT export MALLOC_ARENA_MAX=1 #Does not need fast allocation exec java \ --D vespa.replace_invalid_unicode=true \ +-Dvespa.replace_invalid_unicode=true \ -server -enableassertions \ -XX:ThreadStackSize=512 \ -XX:MaxJavaStackTraceDepth=1000000 \ diff --git a/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java b/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java index 916fe05ef7b..4791d78f0a2 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java @@ -48,7 +48,7 @@ public class SignificanceModelGeneratorTest { void testGenerateSimpleFile() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params = createParameters(inputPath, outputPath, "text", "NB", "nb", "false").build(); + ClientParameters params = createParameters(inputPath, outputPath, "text", "nb", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params); generator.generate(); @@ -60,9 +60,9 @@ public class SignificanceModelGeneratorTest { HashMap<String, DocumentFrequencyFile> languages = modelFile.languages(); assertEquals(1, languages.size()); - assertTrue(languages.containsKey("NB")); + assertTrue(languages.containsKey("nb")); - DocumentFrequencyFile documentFrequencyFile = languages.get("NB"); + DocumentFrequencyFile documentFrequencyFile = languages.get("nb"); assertEquals(3, documentFrequencyFile.frequencies().get("fra")); assertEquals(3, documentFrequencyFile.frequencies().get("skriveform")); @@ -74,19 +74,17 @@ public class SignificanceModelGeneratorTest { @Test void testGenerateSimpleFileWithZST() throws IOException { String inputPath = "no.jsonl"; - ClientParameters params1 = createParameters(inputPath, "output.json", "text", "NB", "nb", "true").build(); + ClientParameters params1 = createParameters(inputPath, "output.json", "text", "nb", "nb", "true").build(); // Throws exception when outputfile does not have .zst extension when using zst compression assertThrows(IllegalArgumentException.class, () -> createSignificanceModelGenerator(params1)); String outputPath = "output.json.zst"; - ClientParameters params = createParameters(inputPath, outputPath, "text", "NB", "nb", "true").build(); + ClientParameters params = createParameters(inputPath, outputPath, "text", "nb", "nb", "true").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params); generator.generate(); - - File outputFile = new File(tempDir.resolve(outputPath ).toString()); assertTrue(outputFile.exists()); @@ -97,9 +95,9 @@ public class SignificanceModelGeneratorTest { HashMap<String, DocumentFrequencyFile> languages = modelFile.languages(); assertEquals(1, languages.size()); - assertTrue(languages.containsKey("NB")); + assertTrue(languages.containsKey("nb")); - DocumentFrequencyFile documentFrequencyFile = languages.get("NB"); + DocumentFrequencyFile documentFrequencyFile = languages.get("nb"); assertEquals(3, documentFrequencyFile.frequencies().get("fra")); assertEquals(3, documentFrequencyFile.frequencies().get("skriveform")); @@ -112,7 +110,7 @@ public class SignificanceModelGeneratorTest { void testGenerateFileWithMultipleLanguages() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params1 = createParameters(inputPath, outputPath, "text", "NB", "nb", "false").build(); + ClientParameters params1 = createParameters(inputPath, outputPath, "text", "nb", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params1); generator.generate(); @@ -120,7 +118,7 @@ public class SignificanceModelGeneratorTest { assertTrue(outputFile.exists()); String inputPath2 = "en.jsonl"; - ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "EN", "en", "false").build(); + ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "en", "en", "false").build(); generator = createSignificanceModelGenerator(params2); generator.generate(); @@ -133,11 +131,11 @@ public class SignificanceModelGeneratorTest { assertEquals(2, languages.size()); - assertTrue(languages.containsKey("NB")); - assertTrue(languages.containsKey("EN")); + assertTrue(languages.containsKey("nb")); + assertTrue(languages.containsKey("en")); - DocumentFrequencyFile nb = languages.get("NB"); - DocumentFrequencyFile en = languages.get("EN"); + DocumentFrequencyFile nb = languages.get("nb"); + DocumentFrequencyFile en = languages.get("en"); assertEquals(3, nb.documentCount()); assertEquals(3, en.documentCount()); @@ -154,7 +152,7 @@ public class SignificanceModelGeneratorTest { void testOverwriteExistingDocumentFrequencyLanguage() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params1 = createParameters(inputPath, outputPath, "text", "NB", "nb", "false").build(); + ClientParameters params1 = createParameters(inputPath, outputPath, "text", "nb", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params1); generator.generate(); @@ -166,16 +164,16 @@ public class SignificanceModelGeneratorTest { HashMap<String, DocumentFrequencyFile> oldLanguages = preUpdatedFile.languages(); assertEquals(1, oldLanguages.size()); - assertTrue(oldLanguages.containsKey("NB")); + assertTrue(oldLanguages.containsKey("nb")); - DocumentFrequencyFile oldDf = oldLanguages.get("NB"); + DocumentFrequencyFile oldDf = oldLanguages.get("nb"); assertEquals(3, oldDf.frequencies().get("fra")); assertEquals(3, oldDf.frequencies().get("skriveform")); assertFalse(oldDf.frequencies().containsKey("nytt")); String inputPath2 = "no_2.jsonl"; - ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "NB", "nb", "false").build(); + ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "nb", "nb", "false").build(); SignificanceModelGenerator generator2 = createSignificanceModelGenerator(params2); generator2.generate(); @@ -188,13 +186,39 @@ public class SignificanceModelGeneratorTest { assertEquals(1, languages.size()); - assertTrue(languages.containsKey("NB")); + assertTrue(languages.containsKey("nb")); - DocumentFrequencyFile df = languages.get("NB"); + DocumentFrequencyFile df = languages.get("nb"); assertEquals(2, df.frequencies().get("fra")); assertEquals(3, df.frequencies().get("skriveform")); assertTrue(df.frequencies().containsKey("nytt")); assertEquals(2, df.frequencies().get("nytt")); } + + @Test + void testGenerateFileWithMultipleLanguagesForSingleDocumentFrequency() throws IOException { + String inputPath = "no.jsonl"; + String outputPath = "output.json"; + ClientParameters params = createParameters(inputPath, outputPath, "text", "nb,un", "nb", "false").build(); + SignificanceModelGenerator generator = createSignificanceModelGenerator(params); + generator.generate(); + + File outputFile = new File(tempDir.resolve(outputPath).toString()); + assertTrue(outputFile.exists()); + + SignificanceModelFile modelFile = objectMapper.readValue(outputFile, SignificanceModelFile.class); + + HashMap<String, DocumentFrequencyFile> languages = modelFile.languages(); + assertEquals(1, languages.size()); + + assertTrue(languages.containsKey("nb,un")); + + DocumentFrequencyFile documentFrequencyFile = languages.get("nb,un"); + + assertEquals(3, documentFrequencyFile.frequencies().get("fra")); + assertEquals(3, documentFrequencyFile.frequencies().get("skriveform")); + assertEquals(3, documentFrequencyFile.frequencies().get("kategori")); + assertEquals(3, documentFrequencyFile.frequencies().get("eldr")); + } } diff --git a/vespalib/src/tests/array/array_test.cpp b/vespalib/src/tests/array/array_test.cpp index 5446130d5d9..75050612357 100644 --- a/vespalib/src/tests/array/array_test.cpp +++ b/vespalib/src/tests/array/array_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/memory_allocator_observer.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/util/round_up_to_page_size.h> #include <vespa/vespalib/util/size_literals.h> @@ -37,7 +38,7 @@ using AllocStats = MyMemoryAllocator::Stats; class Clever { public: Clever() : _counter(&_global) { (*_counter)++; } - Clever(std::atomic<size_t> * counter) : + explicit Clever(std::atomic<size_t> * counter) : _counter(counter) { (*_counter)++; @@ -224,7 +225,7 @@ testBeginEnd(T & v) EXPECT_EQUAL(1u, *(v.begin())); EXPECT_EQUAL(3u, *(v.end() - 1)); - typename T::iterator i(v.begin()); + auto i(v.begin()); EXPECT_EQUAL(1u, *i); EXPECT_EQUAL(2u, *(i+1)); EXPECT_EQUAL(1u, *i++); @@ -249,7 +250,7 @@ testBeginEnd(T & v) EXPECT_EQUAL(3u, *(v.rbegin())); EXPECT_EQUAL(1u, *(v.rend() - 1)); - typename T::reverse_iterator r(v.rbegin()); + auto r(v.rbegin()); EXPECT_EQUAL(3u, *r); EXPECT_EQUAL(2u, *(r+1)); EXPECT_EQUAL(3u, *r++); diff --git a/vespalib/src/tests/arrayqueue/arrayqueue.cpp b/vespalib/src/tests/arrayqueue/arrayqueue.cpp index c00cca17c78..657d379167f 100644 --- a/vespalib/src/tests/arrayqueue/arrayqueue.cpp +++ b/vespalib/src/tests/arrayqueue/arrayqueue.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/util/arrayqueue.hpp> using namespace vespalib; diff --git a/vespalib/src/tests/compression/compression_test.cpp b/vespalib/src/tests/compression/compression_test.cpp index 2257b57dc7e..3d8aeb02a81 100644 --- a/vespalib/src/tests/compression/compression_test.cpp +++ b/vespalib/src/tests/compression/compression_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> diff --git a/vespalib/src/tests/directio/directio.cpp b/vespalib/src/tests/directio/directio.cpp index 01d6be84516..7362c33c7b6 100644 --- a/vespalib/src/tests/directio/directio.cpp +++ b/vespalib/src/tests/directio/directio.cpp @@ -15,7 +15,7 @@ TEST("that DirectIOException propagates the correct information.") { EXPECT_EQUAL(3u, e.getOffset()); EXPECT_EQUAL(msg, e.getBuffer()); EXPECT_EQUAL(0u, string(e.what()).find("DirectIO failed for file 'file.a' buffer=")); - EXPECT_EQUAL(string("file.a"), e.getFileName()); + EXPECT_EQUAL("file.a", e.getFileName()); } TEST("that DirectIOException is thrown on unaligned buf.") { @@ -30,7 +30,7 @@ TEST("that DirectIOException is thrown on unaligned buf.") { EXPECT_EQUAL(4_Ki, e.getLength()); EXPECT_EQUAL(0u, e.getOffset()); EXPECT_EQUAL(buf.getFree()+1, e.getBuffer()); - EXPECT_EQUAL(string(f.GetFileName()), e.getFileName()); + EXPECT_EQUAL(f.GetFileName(), e.getFileName()); caught = true; } EXPECT_TRUE(caught); @@ -48,7 +48,7 @@ TEST("that DirectIOException is thrown on unaligned offset.") { EXPECT_EQUAL(4_Ki, e.getLength()); EXPECT_EQUAL(1u, e.getOffset()); EXPECT_EQUAL(buf.getFree(), e.getBuffer()); - EXPECT_EQUAL(string(f.GetFileName()), e.getFileName()); + EXPECT_EQUAL(f.GetFileName(), e.getFileName()); caught = true; } EXPECT_TRUE(caught); diff --git a/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp b/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp index 7e8b52e6b30..6e86197bd3a 100644 --- a/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp +++ b/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp @@ -2,6 +2,7 @@ #include <vespa/fastlib/text/unicodeutil.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> TEST("GetUTF8Char_WrongInput") { const char *testdata = "ab\xF8"; @@ -15,17 +16,4 @@ TEST("GetUTF8Char_WrongInput") { EXPECT_EQUAL(Fast_UnicodeUtil::_BadUTF8Char, the_char); } -TEST("IsTerminalPunctuationChar") { - // test a small selection - - EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar('!')); - EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(',')); - EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar('.')); - EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(':')); - EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(';')); - EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar(' ')); - EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar('a')); - EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar('A')); -} - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/io/fileutil/fileutiltest.cpp b/vespalib/src/tests/io/fileutil/fileutiltest.cpp index 8e08c8c88bb..51b755e1668 100644 --- a/vespalib/src/tests/io/fileutil/fileutiltest.cpp +++ b/vespalib/src/tests/io/fileutil/fileutiltest.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <filesystem> #include <iostream> #include <vector> diff --git a/vespalib/src/tests/memory/memory_test.cpp b/vespalib/src/tests/memory/memory_test.cpp index be413fd1a95..59d1cbe059d 100644 --- a/vespalib/src/tests/memory/memory_test.cpp +++ b/vespalib/src/tests/memory/memory_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/util/memory.h> using namespace vespalib; diff --git a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp index 4a20f907fe6..aeaff93ffe9 100644 --- a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp +++ b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/net/tls/capability_set.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib; using namespace vespalib::net::tls; diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 6d5c5fa6308..0912afde250 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/crypto/private_key.h> #include <vespa/vespalib/crypto/x509_certificate.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/net/tls/authorization_mode.h> #include <vespa/vespalib/net/tls/crypto_codec.h> @@ -14,7 +13,8 @@ #include <vespa/vespalib/test/peer_policy_utils.h> #include <vespa/vespalib/util/size_literals.h> #include <stdexcept> -#include <stdlib.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib; using namespace vespalib::crypto; diff --git a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp index 26db06e35f1..b80151e9ccf 100644 --- a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp +++ b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/net/tls/policy_checking_certificate_verifier.h> #include <vespa/vespalib/test/peer_policy_utils.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib; using namespace vespalib::net::tls; diff --git a/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp b/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp index 01e6113064a..a5fe850d85d 100644 --- a/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp +++ b/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/net/tls/protocol_snooping.h> using namespace vespalib; diff --git a/vespalib/src/tests/objects/nbostream/nbostream_test.cpp b/vespalib/src/tests/objects/nbostream/nbostream_test.cpp index a2501b836ce..91c83656353 100644 --- a/vespalib/src/tests/objects/nbostream/nbostream_test.cpp +++ b/vespalib/src/tests/objects/nbostream/nbostream_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/test/insertion_operators.h> diff --git a/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp index 62b7dc5f179..58027e35601 100644 --- a/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp +++ b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/util/polymorphicarrays.h> #include <cassert> diff --git a/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp index bcd8ddb24f5..ef13421a6ec 100644 --- a/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/adaptive_sequenced_executor.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <condition_variable> #include <unistd.h> diff --git a/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 474df9bdf3b..beed490c214 100644 --- a/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/singleexecutor.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <condition_variable> #include <unistd.h> diff --git a/vespalib/src/tests/sha1/sha1_test.cpp b/vespalib/src/tests/sha1/sha1_test.cpp index d7777189074..df7dc423b00 100644 --- a/vespalib/src/tests/sha1/sha1_test.cpp +++ b/vespalib/src/tests/sha1/sha1_test.cpp @@ -1,7 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include "rfc_sha1.h" +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/vespalib/util/sha1.h> #include <vespa/vespalib/stllike/string.h> diff --git a/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp index 224a4a95afe..e20b42601c5 100644 --- a/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp +++ b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp @@ -82,7 +82,7 @@ void verifyResizeTaskLimit(bool up) { EXPECT_EQUAL(1u, started); executor.setTaskLimit(targetTaskLimit); EXPECT_EQUAL(INITIAL_2inN, executor.getTaskLimit()); - EXPECT_EQUAL(INITIAL_2inN*waterMarkRatio, executor.get_watermark()); + EXPECT_EQUAL(INITIAL_2inN*waterMarkRatio, static_cast<double>(executor.get_watermark())); allowed = 5; while (started < 6); EXPECT_EQUAL(6u, started); @@ -101,7 +101,7 @@ void verifyResizeTaskLimit(bool up) { while (started < INITIAL + 1); EXPECT_EQUAL(INITIAL + 1, started); EXPECT_EQUAL(roundedTaskLimit, executor.getTaskLimit()); - EXPECT_EQUAL(roundedTaskLimit*waterMarkRatio, executor.get_watermark()); + EXPECT_EQUAL(roundedTaskLimit*waterMarkRatio, static_cast<double>(executor.get_watermark())); allowed = INITIAL + 1; } diff --git a/vespalib/src/tests/slime/slime_binary_format_test.cpp b/vespalib/src/tests/slime/slime_binary_format_test.cpp index 888016caf5f..31fa6037877 100644 --- a/vespalib/src/tests/slime/slime_binary_format_test.cpp +++ b/vespalib/src/tests/slime/slime_binary_format_test.cpp @@ -1,9 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> +#include "type_traits.h" #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/simple_buffer.h> -#include "type_traits.h" #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib::slime::convenience; using namespace vespalib::slime::binary_format; diff --git a/vespalib/src/tests/slime/slime_inject_test.cpp b/vespalib/src/tests/slime/slime_inject_test.cpp index a9274e3073d..be9303879bd 100644 --- a/vespalib/src/tests/slime/slime_inject_test.cpp +++ b/vespalib/src/tests/slime/slime_inject_test.cpp @@ -1,7 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/strfmt.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib::slime::convenience; diff --git a/vespalib/src/tests/slime/slime_test.cpp b/vespalib/src/tests/slime/slime_test.cpp index d2d1e368715..68c91fc900b 100644 --- a/vespalib/src/tests/slime/slime_test.cpp +++ b/vespalib/src/tests/slime/slime_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/object_value.h> #include <vespa/vespalib/data/slime/array_value.h> @@ -9,6 +8,8 @@ #include <vespa/vespalib/data/slime/symbol_table.h> #include <vespa/vespalib/data/slime/basic_value.h> #include <type_traits> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> #include <vespa/log/log.h> LOG_SETUP("slime_test"); diff --git a/vespalib/src/tests/state_server/state_server_test.cpp b/vespalib/src/tests/state_server/state_server_test.cpp index 6c248b54cc8..18dc54188ef 100644 --- a/vespalib/src/tests/state_server/state_server_test.cpp +++ b/vespalib/src/tests/state_server/state_server_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/net/connection_auth_context.h> #include <vespa/vespalib/net/http/state_server.h> #include <vespa/vespalib/net/http/simple_health_producer.h> @@ -13,6 +12,8 @@ #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/process/process.h> #include <sys/stat.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib; diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index 592fc72edeb..0f5bbf29042 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_map_equal.hpp> @@ -8,6 +7,8 @@ #include <cstddef> #include <algorithm> #include <atomic> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib; using std::make_pair; diff --git a/vespalib/src/tests/testapp-generic/testapp-generic.cpp b/vespalib/src/tests/testapp-generic/testapp-generic.cpp index e63e78040fe..9b5a5c0426a 100644 --- a/vespalib/src/tests/testapp-generic/testapp-generic.cpp +++ b/vespalib/src/tests/testapp-generic/testapp-generic.cpp @@ -1,7 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <stdexcept> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_master.hpp> using namespace vespalib; diff --git a/vespalib/src/vespa/fastlib/text/unicodeutil.h b/vespalib/src/vespa/fastlib/text/unicodeutil.h index 740cc9381b7..dd0c13738c6 100644 --- a/vespalib/src/vespa/fastlib/text/unicodeutil.h +++ b/vespalib/src/vespa/fastlib/text/unicodeutil.h @@ -41,8 +41,7 @@ private: _wordcharProp = 2, _ideographicProp = 4, _decimalDigitCharProp = 8, - _ignorableControlCharProp = 16, - _terminalPunctuationCharProp = 32 + _ignorableControlCharProp = 16 }; public: @@ -212,18 +211,6 @@ public: static int utf8cmp(const char *s1, const ucs4_t *s2) noexcept; /** - * Test for terminal punctuation. - * @param testchar the UCS4 character to test. - * @return true if testchar is a terminal punctuation character, - * i.e. if it has the terminal punctuation char property. - */ - static bool IsTerminalPunctuationChar(ucs4_t testchar) noexcept { - return (testchar < 65536 && - (_compCharProps[testchar >> 8][testchar & 255] & - _terminalPunctuationCharProp) != 0); - } - - /** * Get the next UCS4 character from an UTF-8 string buffer. * We assume that the first character in the UTF-8 string is >= 0x80 (non-ascii). * Modify the src pointer to allow future calls. diff --git a/vespalib/src/vespa/vespalib/data/simple_buffer.cpp b/vespalib/src/vespa/vespalib/data/simple_buffer.cpp index 1b366505f0e..5b761710008 100644 --- a/vespalib/src/vespa/vespalib/data/simple_buffer.cpp +++ b/vespalib/src/vespa/vespalib/data/simple_buffer.cpp @@ -5,18 +5,13 @@ namespace vespalib { -SimpleBuffer::SimpleBuffer() - : _data(), - _used(0) -{ -} SimpleBuffer::~SimpleBuffer() = default; Memory SimpleBuffer::obtain() { - return Memory(_data.data(), _used); + return {_data.data(), _used}; } Input & @@ -33,7 +28,7 @@ SimpleBuffer::reserve(size_t bytes) { assert((_used + bytes) >= _used); _data.resize(_used + bytes, char(0x55)); - return WritableMemory(&_data[_used], bytes); + return {&_data[_used], bytes}; } Output & diff --git a/vespalib/src/vespa/vespalib/data/simple_buffer.h b/vespalib/src/vespa/vespalib/data/simple_buffer.h index 9070ddbff6f..ab95b9edf42 100644 --- a/vespalib/src/vespa/vespalib/data/simple_buffer.h +++ b/vespalib/src/vespa/vespalib/data/simple_buffer.h @@ -25,8 +25,8 @@ private: size_t _used; public: - SimpleBuffer(); - ~SimpleBuffer(); + SimpleBuffer() noexcept : _data(), _used(0) {} + ~SimpleBuffer() override; Memory obtain() override; Input &evict(size_t bytes) override; WritableMemory reserve(size_t bytes) override; @@ -36,8 +36,8 @@ public: ++_used; return *this; } - Memory get() const { return Memory(_data.data(), _used); } - bool operator==(const SimpleBuffer &rhs) const { return (get() == rhs.get()); } + Memory get() const noexcept { return Memory(_data.data(), _used); } + bool operator==(const SimpleBuffer &rhs) const noexcept { return (get() == rhs.get()); } }; std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf); diff --git a/vespalib/src/vespa/vespalib/stllike/string.h b/vespalib/src/vespa/vespalib/stllike/string.h index c4b9e079a38..cbbeb94a1b6 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.h +++ b/vespalib/src/vespa/vespalib/stllike/string.h @@ -27,6 +27,7 @@ public: stringref(const char * s) noexcept : _s(s), _sz(strlen(s)) { } constexpr stringref(const char * s, size_type sz) noexcept : _s(s), _sz(sz) { } stringref(const std::string & s) noexcept : _s(s.c_str()), _sz(s.size()) { } + stringref(const std::string_view & s) noexcept : _s(s.data()), _sz(s.size()) { } stringref(const stringref &) noexcept = default; stringref & operator =(const stringref &) noexcept = default; stringref(stringref &&) noexcept = default; @@ -135,11 +136,14 @@ public: const char & operator [] (size_t i) const noexcept { return _s[i]; } operator std::string () const { return {_s, _sz}; } + operator std::string_view () const { return {_s, _sz}; } std::strong_ordering operator <=>(const char * s) const noexcept { return strong_compare(s, strlen(s)); } std::strong_ordering operator <=>(const std::string & s) const noexcept { return strong_compare(s.data(), s.size()); } + std::strong_ordering operator <=>(std::string_view s) const noexcept { return strong_compare(s.data(), s.size()); } std::strong_ordering operator <=>(stringref s) const noexcept { return strong_compare(s.data(), s.size()); } bool operator ==(const char * s) const noexcept { return strong_compare(s, strlen(s)) == std::strong_ordering::equal; } bool operator ==(const std::string & s) const noexcept { return strong_compare(s.data(), s.size()) == std::strong_ordering::equal; } + bool operator ==(std::string_view s) const noexcept { return strong_compare(s.data(), s.size()) == std::strong_ordering::equal; } bool operator ==(stringref s) const noexcept { return strong_compare(s.data(), s.size()) == std::strong_ordering::equal; } friend std::ostream & operator << (std::ostream & os, stringref v); private: @@ -187,6 +191,7 @@ public: small_string(const void * s, size_type sz) noexcept : _buf(_stack), _sz(sz) { init(s); } small_string(stringref s) noexcept : _buf(_stack), _sz(s.size()) { init(s.data()); } small_string(const std::string & s) noexcept : _buf(_stack), _sz(s.size()) { init(s.data()); } + small_string(std::string_view s) noexcept : _buf(_stack), _sz(s.size()) { init(s.data()); } small_string(small_string && rhs) noexcept : _sz(rhs.size()), _bufferSize(rhs._bufferSize) { @@ -235,10 +240,14 @@ public: small_string& operator= (const std::string &rhs) noexcept { return operator= (stringref(rhs)); } + small_string& operator= (std::string_view rhs) noexcept { + return operator= (stringref(rhs)); + } void swap(small_string & rhs) noexcept { std::swap(*this, rhs); } - operator std::string () const { return std::string(c_str(), size()); } + operator std::string () const { return {c_str(), size()}; } + operator std::string_view () const { return {c_str(), size()}; } operator stringref () const noexcept { return stringref(c_str(), size()); } [[nodiscard]] char at(size_t i) const noexcept { return buffer()[i]; } char & at(size_t i) noexcept { return buffer()[i]; } diff --git a/vespalib/src/vespa/vespalib/test/test_data.h b/vespalib/src/vespa/vespalib/test/test_data.h index 1a5c24616b0..2056466297e 100644 --- a/vespalib/src/vespa/vespalib/test/test_data.h +++ b/vespalib/src/vespa/vespalib/test/test_data.h @@ -4,6 +4,7 @@ #include "test_data_base.h" #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/objects/nbostream.h> #include <filesystem> namespace vespalib::test { diff --git a/vespalib/src/vespa/vespalib/test/test_data_base.cpp b/vespalib/src/vespa/vespalib/test/test_data_base.cpp index 12bfca50d5e..f0f1715b7df 100644 --- a/vespalib/src/vespa/vespalib/test/test_data_base.cpp +++ b/vespalib/src/vespa/vespalib/test/test_data_base.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "test_data.h" +#include "test_data_base.h" #include <vespa/vespalib/objects/nbostream.h> #include <cassert> #include <cstring> diff --git a/vespalib/src/vespa/vespalib/testkit/test_hook.cpp b/vespalib/src/vespa/vespalib/testkit/test_hook.cpp index b14b575ae93..27561983924 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_hook.cpp +++ b/vespalib/src/vespa/vespalib/testkit/test_hook.cpp @@ -81,7 +81,7 @@ bool TestHook::runMyTest(const FixtureFactory & fixture_factory, size_t num_thre FixtureUP fixture_up = fixture_factory(); fixture_up->thread_id = i; fixture_up->num_threads = num_threads; - threads.emplace_back(new TestThreadWrapper(_ignore, latch, barrier, traceStack, *fixture_up)); + threads.emplace_back(std::make_unique<TestThreadWrapper>(_ignore, latch, barrier, traceStack, *fixture_up)); fixtures.push_back(std::move(fixture_up)); } for (size_t i = 1; i < num_threads; ++i) { diff --git a/vespalib/src/vespa/vespalib/testkit/test_kit.h b/vespalib/src/vespa/vespalib/testkit/test_kit.h index 5c86b88105d..bddc2e2622c 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_kit.h +++ b/vespalib/src/vespa/vespalib/testkit/test_kit.h @@ -10,3 +10,4 @@ #include "test_hook.h" #include "test_state_guard.h" #include <vespa/vespalib/util/time.h> +#include "test_master.hpp" diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.cpp b/vespalib/src/vespa/vespalib/testkit/test_master.cpp index 4698d40ecc5..c0587ae841b 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.cpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "test_master.h" +#include "test_master.hpp" #include <vespa/vespalib/util/barrier.h> #include <vespa/vespalib/util/signalhandler.h> #include <cstring> @@ -29,8 +29,8 @@ TestMaster TestMaster::master; __thread TestMaster::ThreadState *TestMaster::_threadState = nullptr; //----------------------------------------------------------------------------- -TestMaster::TraceItem::TraceItem(const std::string &file_in, uint32_t line_in, const std::string &msg_in) - : file(file_in), line(line_in), msg(msg_in) +TestMaster::TraceItem::TraceItem(std::string file_in, uint32_t line_in, std::string msg_in) + : file(std::move(file_in)), line(line_in), msg(std::move(msg_in)) {} TestMaster::TraceItem::TraceItem(TraceItem &&) noexcept = default; TestMaster::TraceItem & TestMaster::TraceItem::operator=(TraceItem &&) noexcept = default; @@ -39,8 +39,8 @@ TestMaster::TraceItem & TestMaster::TraceItem::operator=(const TraceItem &) = de TestMaster::TraceItem::~TraceItem() = default; TestMaster::ThreadState::~ThreadState() = default; -TestMaster::ThreadState::ThreadState(const std::string &n) - : name(n), passCnt(0), failCnt(0), preIgnoreFailCnt(0), +TestMaster::ThreadState::ThreadState(std::string n) + : name(std::move(n)), passCnt(0), failCnt(0), preIgnoreFailCnt(0), ignore(false), unwind(false), traceStack(), barrier(nullptr) {} @@ -166,6 +166,27 @@ TestMaster::reportConclusion(const lock_guard &) return ok; } +void +TestMaster::report_compare(const char *file, uint32_t line, const char *aName, const char *bName, const char *opText, bool fatal, + const std::function<void(std::ostream &)> & printLhs, + const std::function<void(std::ostream &)> & printRhs) +{ + std::string str; + str += aName; + str += opText; + str += bName; + std::ostringstream lhs; + std::ostringstream rhs; + printLhs(lhs); + printRhs(rhs); + { + lock_guard guard(_lock); + checkFailed(guard, file, line, str.c_str()); + printDiff(guard, str, file, line, lhs.str(), rhs.str()); + handleFailure(guard, fatal); + } +} + //----------------------------------------------------------------------------- TestMaster::TestMaster() diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.h b/vespalib/src/vespa/vespalib/testkit/test_master.h index 232ef8009c9..85684f9458f 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.h +++ b/vespalib/src/vespa/vespalib/testkit/test_master.h @@ -6,12 +6,12 @@ #include <vector> #include <memory> #include <mutex> +#include <functional> namespace vespalib { class Barrier; -#ifndef IAM_DOXYGEN /** * The master of testing. **/ @@ -32,7 +32,7 @@ public: std::string file; uint32_t line; std::string msg; - TraceItem(const std::string &file_in, uint32_t line_in, const std::string &msg_in); + TraceItem(std::string file_in, uint32_t line_in, std::string msg_in); TraceItem(TraceItem &&) noexcept; TraceItem & operator=(TraceItem &&) noexcept; TraceItem(const TraceItem &); @@ -51,7 +51,7 @@ private: std::vector<TraceItem> traceStack; Barrier *barrier; ~ThreadState(); - ThreadState(const std::string &n); + ThreadState(std::string n); ThreadState(ThreadState &&) noexcept = default; ThreadState & operator=(ThreadState &&) noexcept = default; }; @@ -89,6 +89,9 @@ private: bool reportConclusion(const lock_guard &); + void report_compare(const char *file, uint32_t line, const char *aName, const char *bName, const char *opText, bool fatal, + const std::function<void(std::ostream &)> & printLhs, + const std::function<void(std::ostream &)> & printRhs); public: ~TestMaster(); TestMaster(const TestMaster &) = delete; @@ -109,8 +112,7 @@ public: void close_debug_files(); void pushState(const char *file, uint32_t line, const char *msg); void popState(); - bool check(bool rc, const char *file, uint32_t line, - const char *str, bool fatal); + bool check(bool rc, const char *file, uint32_t line, const char *str, bool fatal); template<class A, class B, class OP> bool compare(const char *file, uint32_t line, const char *aName, const char *bName, const char *opText, @@ -120,9 +122,5 @@ public: bool discardFailedChecks(size_t failCnt); bool fini(); }; -#endif } // namespace vespalib - -#include "test_master.hpp" - diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.hpp b/vespalib/src/vespa/vespalib/testkit/test_master.hpp index 065a64fb2ab..a5e25b7336b 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.hpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.hpp @@ -1,5 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include "test_master.h" + #include <sstream> namespace vespalib { @@ -20,24 +23,13 @@ TestMaster::compare(const char *file, uint32_t line, const char *opText, const A &a, const B &b, const OP &op, bool fatal) { - if (op(a,b)) { + if (op(a,b)) [[likely]]{ ++threadState().passCnt; return true; } - std::string str; - str += aName; - str += opText; - str += bName; - std::ostringstream lhs; - std::ostringstream rhs; - lhs << a; - rhs << b; - { - lock_guard guard(_lock); - checkFailed(guard, file, line, str.c_str()); - printDiff(guard, str, file, line, lhs.str(), rhs.str()); - handleFailure(guard, fatal); - } + report_compare(file, line, aName, bName, opText, fatal, + [&](std::ostream & os) { os << a;}, + [&](std::ostream & os) { os << b;}); return false; } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index ba597d177c4..846c7cd6d99 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -48,6 +48,8 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.logging.Level.INFO; + /** * Curator interface for Vespa. * This contains method for constructing common recipes and utilities as well as @@ -70,15 +72,14 @@ public class Curator extends AbstractComponent implements AutoCloseable { private static final Duration BASE_SLEEP_TIME = Duration.ofSeconds(1); private static final int MAX_RETRIES = 4; private static final RetryPolicy DEFAULT_RETRY_POLICY = new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES); - // Default value taken from ZookeeperServerConfig - static final long defaultJuteMaxBuffer = Long.parseLong(System.getProperty("jute.maxbuffer", "52428800")); + private static final int defaultJuteMaxBuffer = new CuratorConfig.Builder().build().juteMaxBuffer(); private final CuratorFramework curatorFramework; private final ConnectionSpec connectionSpec; private final long juteMaxBuffer; private final Duration sessionTimeout; - // All lock keys, to allow re-entrancy. This will grow forever, but this should be too slow to be a problem + // All lock keys, to allow re-entrance. This will grow forever, but this should be too slow to be a problem private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>(); /** Creates a curator instance from a comma-separated string of ZooKeeper host:port strings */ @@ -101,7 +102,7 @@ public class Curator extends AbstractComponent implements AutoCloseable { CuratorConfig.Server::port, curatorConfig.zookeeperLocalhostAffinity()), Optional.of(ZK_CLIENT_CONFIG_FILE), - defaultJuteMaxBuffer, + curatorConfig.juteMaxBuffer(), Duration.ofSeconds(curatorConfig.zookeeperSessionTimeoutSeconds())); } @@ -110,7 +111,7 @@ public class Curator extends AbstractComponent implements AutoCloseable { defaultJuteMaxBuffer, DEFAULT_ZK_SESSION_TIMEOUT); } - Curator(ConnectionSpec connectionSpec, Optional<File> clientConfigFile, long juteMaxBuffer, Duration sessionTimeout) { + Curator(ConnectionSpec connectionSpec, Optional<File> clientConfigFile, int juteMaxBuffer, Duration sessionTimeout) { this(connectionSpec, CuratorFrameworkFactory .builder() @@ -126,10 +127,11 @@ public class Curator extends AbstractComponent implements AutoCloseable { sessionTimeout); } - private Curator(ConnectionSpec connectionSpec, CuratorFramework curatorFramework, long juteMaxBuffer, Duration sessionTimeout) { + private Curator(ConnectionSpec connectionSpec, CuratorFramework curatorFramework, int juteMaxBuffer, Duration sessionTimeout) { this.connectionSpec = Objects.requireNonNull(connectionSpec); this.curatorFramework = Objects.requireNonNull(curatorFramework); this.juteMaxBuffer = juteMaxBuffer; + LOG.log(INFO, "Using juteMaxBuffer=" + this.juteMaxBuffer); this.sessionTimeout = sessionTimeout; addLoggingListener(); curatorFramework.start(); diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java index 52324e94273..aee594e4bc4 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java @@ -73,10 +73,10 @@ public class CuratorTest { } private Curator createCurator(CuratorConfig curatorConfig) { - return createCurator(curatorConfig, Curator.defaultJuteMaxBuffer); + return createCurator(curatorConfig, 12345); } - private Curator createCurator(CuratorConfig curatorConfig, long juteMaxBuffer) { + private Curator createCurator(CuratorConfig curatorConfig, int juteMaxBuffer) { return new Curator(ConnectionSpec.create(curatorConfig.server(), CuratorConfig.Server::hostname, CuratorConfig.Server::port, diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index 06e4d0da00c..1fd354fccd9 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -25,6 +25,7 @@ import java.util.stream.Collectors; import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static java.util.logging.Level.INFO; public class Configurator { @@ -47,6 +48,7 @@ public class Configurator { // Doc says that it is max size of data in a zookeeper node, but it goes for everything that // needs to be serialized, see https://issues.apache.org/jira/browse/ZOOKEEPER-1162 for details System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString()); + log.log(INFO, "Using juteMaxBuffer=" + zookeeperServerConfig.juteMaxBuffer()); // Need to set this as a system properties instead of config, config does not work System.setProperty("zookeeper.authProvider.x509", "com.yahoo.vespa.zookeeper.VespaMtlsAuthenticationProvider"); // Need to set this as a system property, otherwise it will be parsed for _every_ packet and an exception will be thrown (and handled) |