diff options
443 files changed, 3638 insertions, 2624 deletions
diff --git a/.buildkite/Makefile b/.buildkite/Makefile index b039b73ba53..98596b34aa7 100644 --- a/.buildkite/Makefile +++ b/.buildkite/Makefile @@ -12,8 +12,8 @@ export FACTORY_VESPA_VERSION := $(VESPA_VERSION) export VALGRIND_UNIT_TESTS ?= false export VESPA_USE_SANITIZER ?= null -export VESPA_MAVEN_EXTRA_OPTS ?= --show-version --batch-mode --no-snapshot-updates -Dmaven.javadoc.skip=true \ - -Dmaven.source.skip=true -DaltDeploymentRepository=local-repo::default::file:$(WORKDIR)/artifacts/$(ARCH)/maven-repo +export VESPA_MAVEN_EXTRA_OPTS ?= --show-version --batch-mode --no-snapshot-updates \ + -DaltDeploymentRepository=local-repo::default::file:$(WORKDIR)/artifacts/$(ARCH)/maven-repo export NUM_CPU_LIMIT ?= $(shell nproc) export NUM_CPP_THREADS := $(shell echo $$(( $(NUM_CPU_LIMIT)*2/3 ))) @@ -25,11 +25,12 @@ ifeq ($(BUILDKITE_PULL_REQUEST),false) export VESPA_MAVEN_TARGET ?= deploy else export VESPA_MAVEN_TARGET ?= install + export VESPA_MAVEN_EXTRA_OPTS := $(VESPA_MAVEN_EXTRA_OPTS) -Dmaven.source.skip=true -Dmaven.javadoc.skip=true endif .DEFAULT_GOAL := pr -main: build-rpms cpp-test quick-start-guide publish-container publish-artifacts +main: build-rpms cpp-test quick-start-guide publish-container publish-artifacts upload-test-results pr: build-rpms cpp-test basic-search-test check: @@ -74,6 +75,9 @@ publish-container: build-container publish-artifacts: java build-rpms @$(TOP)/execute.sh $@ +upload-test-results: java cpp-test + @$(TOP)/execute.sh $@ + .PHONY: \ main \ pr \ @@ -90,4 +94,5 @@ publish-artifacts: java build-rpms quick-start-guide \ publish-container \ publish-artifacts \ + upload-test-results \ check diff --git a/.buildkite/upload-test-results.sh b/.buildkite/upload-test-results.sh new file mode 100755 index 00000000000..e95a9448adf --- /dev/null +++ b/.buildkite/upload-test-results.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $BUILDKITE != true ]]; then + echo "Skipping artifact publishing when not executed by Buildkite." + exit 0 +fi + +if [[ $(arch) == x86_64 ]]; then + JAVA_TEST_TOKEN=$UNIT_TEST_JAVA_AMD64_TOKEN + CPP_TEST_TOKEN=$UNIT_TEST_CPP_AMD64_TOKEN +else + JAVA_TEST_TOKEN=$UNIT_TEST_JAVA_ARM64_TOKEN + CPP_TEST_TOKEN=$UNIT_TEST_CPP_ARM64_TOKEN +fi + +if [[ -z $JAVA_TEST_TOKEN ]]; then + echo "Missing JAVA_TEST_TOKEN. Exiting." + exit 1 +fi +if [[ -z $CPP_TEST_TOKEN ]]; then + echo "Missing CPP_TEST_TOKEN. Exiting." + exit 1 +fi + +upload_result() { + curl \ + -X POST \ + -H "Authorization: Token token=\"$BUILDKITE_ANALYTICS_TOKEN\"" \ + -F "data=@$1" \ + -F "format=junit" \ + -F "run_env[CI]=buildkite" \ + -F "run_env[key]=$BUILDKITE_BUILD_ID" \ + -F "run_env[url]=$BUILDKITE_BUILD_URL" \ + -F "run_env[branch]=$BUILDKITE_BRANCH" \ + -F "run_env[commit_sha]=$BUILDKITE_COMMIT" \ + -F "run_env[number]=$BUILDKITE_BUILD_NUMBER" \ + -F "run_env[job_id]=$BUILDKITE_JOB_ID" \ + -F "run_env[message]=$BUILDKITE_MESSAGE" \ + "https://analytics-api.buildkite.com/v1/uploads" +} + +export -f upload_result + +# Upload all surefire TEST-*.xml reports +cd "$WORKDIR" +export BUILDKITE_ANALYTICS_TOKEN=$JAVA_TEST_TOKEN +# shellcheck disable=2038 +find . -name "TEST-*.xml" -type f | xargs -n 1 -P 50 -I '{}' bash -c "upload_result {}" + +# Upload the cpp test report +export BUILDKITE_ANALYTICS_TOKEN=$CPP_TEST_TOKEN +upload_result "$LOG_DIR/vespa-cpptest-results.xml" + diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java index fb27327b4c5..6d28db342e6 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java @@ -16,8 +16,6 @@ public class ServiceType { public static final ServiceType HOST_ADMIN = new ServiceType("hostadmin"); public static final ServiceType CONFIG_SERVER = new ServiceType("configserver"); public static final ServiceType CONTROLLER = new ServiceType("controller"); - // TODO: Remove TRANSACTION_LOG_SERVER when oldest Vespa version in use is 8.354 - public static final ServiceType TRANSACTION_LOG_SERVER = new ServiceType("transactionlogserver"); public static final ServiceType CLUSTER_CONTROLLER = new ServiceType("container-clustercontroller"); public static final ServiceType DISTRIBUTOR = new ServiceType("distributor"); public static final ServiceType SEARCH = new ServiceType("searchnode"); diff --git a/client/go/Makefile b/client/go/Makefile index b2ffdc0feb6..fee92547e73 100644 --- a/client/go/Makefile +++ b/client/go/Makefile @@ -17,6 +17,7 @@ GO_FLAGS := -ldflags "-X github.com/vespa-engine/vespa/client/go/internal/build. PROJECT_ROOT := $(shell realpath $(CURDIR)/../..) GO_TMPDIR := $(PROJECT_ROOT)/build/go DIST_TARGETS := dist-mac dist-mac-arm64 dist-linux dist-linux-arm64 dist-win32 dist-win64 +GOTOOLCHAIN := $(shell go env GOTOOLCHAIN) all: test checkfmt install @@ -111,7 +112,9 @@ install-all: all manpages # Development targets # -ci: +setenv: +# Set GOTOOLCHAIN if its default value has been changed + @test "$(GOTOOLCHAIN)" = auto || go env -w GOTOOLCHAIN="auto" ifdef CI # Ensure that CI systems use a proxy for downloading dependencies go env -w GOPROXY="https://proxy.golang.org,direct" @@ -121,7 +124,7 @@ endif install-brew: brew install vespa-cli -install: ci +install: setenv env GOBIN=$(BIN) go install $(GO_FLAGS) ./... manpages: install @@ -133,7 +136,7 @@ clean: rm -f $(BIN)/vespa $(BIN)/vespa-wrapper $(SHARE)/man/man1/vespa.1 $(SHARE)/man/man1/vespa-*.1 rmdir -p $(BIN) $(SHARE)/man/man1 > /dev/null 2>&1 || true -test: ci +test: setenv # Why custom GOTMPDIR? go builds executables for unit tests and by default these # end up in TMPDIR/GOTMPDIR. In some environments /tmp is mounted noexec so # running test executables will fail diff --git a/client/go/go.mod b/client/go/go.mod index de626a073a0..4b6b0ceef49 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -1,13 +1,12 @@ module github.com/vespa-engine/vespa/client/go -go 1.20 +go 1.22.4 require ( github.com/alessio/shellescape v1.4.2 github.com/briandowns/spinner v1.23.1 github.com/fatih/color v1.17.0 - // This is the most recent version compatible with Go 1.20. Upgrade when we upgrade our Go version - github.com/go-json-experiment/json v0.0.0-20230324203220-04923b7a9528 + github.com/go-json-experiment/json v0.0.0-20240524174822-2d9f40f7385b github.com/klauspost/compress v1.17.9 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 @@ -30,7 +29,6 @@ require ( github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/client/go/go.sum b/client/go/go.sum index ea7b5890d95..d8b6a4cacf9 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -1,39 +1,21 @@ github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= -github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Ptu650= github.com/briandowns/spinner v1.23.1/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/go-json-experiment/json v0.0.0-20230324203220-04923b7a9528 h1:hmpF6G+rHcypt8J6jhBH/rDUx+04Th/L61Y8uCKFb7Q= -github.com/go-json-experiment/json v0.0.0-20230324203220-04923b7a9528/go.mod h1:AHV+bpNGVGD0DCHMBhhTYtT7yeBYD9Yk92XAjB7vOgo= +github.com/go-json-experiment/json v0.0.0-20240524174822-2d9f40f7385b h1:IM96IiRXFcd7l+mU8Sys9pcggoBLbH/dEgzOESrS8F8= +github.com/go-json-experiment/json v0.0.0-20240524174822-2d9f40f7385b/go.mod h1:uDEMZSTQMj7V6Lxdrx4ZwchmHEGdICbjuY+GQd7j9LM= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= -github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= -github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -46,91 +28,31 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= -github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= -github.com/zalando/go-keyring v0.2.4 h1:wi2xxTqdiwMKbM6TWwi+uJCG/Tum2UV0jqaQhCa9/68= -github.com/zalando/go-keyring v0.2.4/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= github.com/zalando/go-keyring v0.2.5 h1:Bc2HHpjALryKD62ppdEzaFG6VxL6Bc+5v0LYpN8Lba8= github.com/zalando/go-keyring v0.2.5/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/client/go/internal/admin/jvm/mem_options_test.go b/client/go/internal/admin/jvm/mem_options_test.go index 3db10153086..60cffc824e9 100644 --- a/client/go/internal/admin/jvm/mem_options_test.go +++ b/client/go/internal/admin/jvm/mem_options_test.go @@ -9,7 +9,7 @@ import ( func TestAdjustment(t *testing.T) { lastAdj := 64 - for i := 0; i < 4096; i++ { + for i := range 4096 { adj := adjustAvailableMemory(MegaBytesOfMemory(i)).ToMB() assert.True(t, int(adj) >= lastAdj) lastAdj = int(adj) diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go b/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go index ec2c53487be..180f7c84859 100644 --- a/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go +++ b/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go @@ -85,7 +85,7 @@ func (t *unixTail) openTail() { if err != nil { return } - for i := 0; i < n; i++ { + for i := range n { if t.lineBuf[i] == '\n' { sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) if err == nil { diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index 6c5df8b3e84..d6bc59f5b4f 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -136,7 +136,7 @@ func createServices(n int, timeout time.Duration, cli *CLI, waiter *Waiter) ([]h } services := make([]httputil.Client, 0, n) baseURL := "" - for i := 0; i < n; i++ { + for range n { service, err := waiter.Service(target, cli.config.cluster()) if err != nil { return nil, "", err diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index 200a0be7c5d..fc25f8e872c 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -82,13 +82,13 @@ func TestFeed(t *testing.T) { require.Nil(t, cli.Run("feed", "-")) assert.Equal(t, want, stdout.String()) - for i := 0; i < 10; i++ { + for range 10 { httpClient.NextResponseString(503, `{"message":"it's broken yo"}`) } require.Nil(t, cli.Run("feed", jsonFile1)) assert.Equal(t, "feed: got status 503 ({\"message\":\"it's broken yo\"}) for put id:ns:type::doc1: giving up after 10 attempts\n", stderr.String()) stderr.Reset() - for i := 0; i < 10; i++ { + for range 10 { httpClient.NextResponseError(fmt.Errorf("something else is broken")) } require.Nil(t, cli.Run("feed", jsonFile1)) diff --git a/client/go/internal/cli/cmd/query.go b/client/go/internal/cli/cmd/query.go index 4d5941943ca..5fa225777f0 100644 --- a/client/go/internal/cli/cmd/query.go +++ b/client/go/internal/cli/cmd/query.go @@ -84,7 +84,7 @@ func query(cli *CLI, arguments []string, timeoutSecs int, curl bool, format stri } url, _ := url.Parse(service.BaseURL + "/search/") urlQuery := url.Query() - for i := 0; i < len(arguments); i++ { + for i := range len(arguments) { key, value := splitArg(arguments[i]) urlQuery.Set(key, value) } diff --git a/client/go/internal/cli/cmd/status_test.go b/client/go/internal/cli/cmd/status_test.go index bf9b4f3493e..6db27fd2778 100644 --- a/client/go/internal/cli/cmd/status_test.go +++ b/client/go/internal/cli/cmd/status_test.go @@ -168,7 +168,7 @@ func TestStatusCloudDeployment(t *testing.T) { } func isLocalTarget(args []string) bool { - for i := 0; i < len(args)-1; i++ { + for i := range len(args) - 1 { if args[i] == "-t" { return args[i+1] == "local" } @@ -197,7 +197,7 @@ func assertStatus(expectedTarget string, args []string, t *testing.T) { t.Helper() client := &mock.HTTPClient{} clusterName := "" - for i := 0; i < 3; i++ { + for range 3 { if isLocalTarget(args) { clusterName = "foo" mockServiceStatus(client, clusterName) diff --git a/client/go/internal/cli/cmd/test_test.go b/client/go/internal/cli/cmd/test_test.go index 728e8c29691..3479e057e45 100644 --- a/client/go/internal/cli/cmd/test_test.go +++ b/client/go/internal/cli/cmd/test_test.go @@ -26,11 +26,11 @@ func TestSuite(t *testing.T) { mockServiceStatus(client, "container") client.NextStatus(200) client.NextStatus(200) - for i := 0; i < 2; i++ { + for range 2 { client.NextResponseString(200, string(searchResponse)) } mockServiceStatus(client, "container") // Some tests do not specify cluster, which is fine since we only have one, but this causes a cache miss - for i := 0; i < 9; i++ { + for range 9 { client.NextResponseString(200, string(searchResponse)) } expectedBytes, _ := os.ReadFile("testdata/tests/expected-suite.out") @@ -45,7 +45,7 @@ func TestSuite(t *testing.T) { requests = append(requests, discoveryRequest) requests = append(requests, createSearchRequest(baseUrl+"/search/")) requests = append(requests, createSearchRequest(baseUrl+"/search/?foo=%2F")) - for i := 0; i < 7; i++ { + for range 7 { requests = append(requests, createSearchRequest(baseUrl+"/search/")) } assertRequests(requests, client, t) diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index ae494059691..85594912da2 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -41,7 +41,7 @@ func TestQuoteFunc(t *testing.T) { var buf []byte = make([]byte, 3) buf[0] = 'a' buf[2] = 'z' - for i := 0; i < 256; i++ { + for i := range 256 { buf[1] = byte(i) s := string(buf) res := quoteArgForUrl(s) diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index e2a77f7b126..9c301cd7990 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" "sync" - "time" // Why do we use an experimental parser? This appears to be the only JSON library that satisfies the following @@ -19,7 +18,7 @@ import ( // - Supports parsing from a io.Reader // - Supports parsing token-by-token // - Few allocations during parsing (especially for large objects) - "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" ) type Operation int @@ -29,11 +28,11 @@ const ( OperationUpdate OperationRemove - jsonArrayStart json.Kind = '[' - jsonArrayEnd json.Kind = ']' - jsonObjectStart json.Kind = '{' - jsonObjectEnd json.Kind = '}' - jsonString json.Kind = '"' + jsonArrayStart jsontext.Kind = '[' + jsonArrayEnd jsontext.Kind = ']' + jsonObjectStart jsontext.Kind = '{' + jsonObjectEnd jsontext.Kind = '}' + jsonString jsontext.Kind = '"' ) var ( @@ -153,7 +152,7 @@ func (d *Document) Reset() { // Decoder decodes documents from a JSON structure which is either an array of objects, or objects separated by newline. type Decoder struct { - dec *json.Decoder + dec *jsontext.Decoder buf bytes.Buffer array bool @@ -202,13 +201,13 @@ func (d *Decoder) guessMode() error { return nil } -func (d *Decoder) readNext(kind json.Kind) (json.Token, error) { +func (d *Decoder) readNext(kind jsontext.Kind) (jsontext.Token, error) { t, err := d.dec.ReadToken() if err != nil { - return json.Token{}, err + return jsontext.Token{}, err } if t.Kind() != kind { - return json.Token{}, fmt.Errorf("unexpected json kind: %q: want %q", t, kind) + return jsontext.Token{}, fmt.Errorf("unexpected json kind: %q: want %q", t, kind) } return t, nil } @@ -364,7 +363,7 @@ loop: func NewDecoder(r io.Reader) *Decoder { d := &Decoder{} d.documentBuffers.New = func() any { return &bytes.Buffer{} } - d.dec = json.NewDecoder(io.TeeReader(r, &d.buf)) + d.dec = jsontext.NewDecoder(io.TeeReader(r, &d.buf)) return d } diff --git a/client/go/internal/vespa/document/document_test.go b/client/go/internal/vespa/document/document_test.go index 3fcdbd3b292..8875ad83291 100644 --- a/client/go/internal/vespa/document/document_test.go +++ b/client/go/internal/vespa/document/document_test.go @@ -176,7 +176,7 @@ func testDocumentDecoder(t *testing.T, jsonLike string) { if len(docs) != len(result) { t.Errorf("len(result) = %d, want %d", len(result), len(docs)) } - for i := 0; i < len(docs); i++ { + for i := range len(docs) { got := result[i] want := docs[i] if !got.Equal(want) { @@ -206,7 +206,7 @@ func TestDocumentDecoderInvalid(t *testing.T) { t.Errorf("unexpected error: %s", err) } _, err = dec.Decode() - wantErr := "invalid operation at byte offset 110: json: invalid character '\\n' within string (expecting non-control character)" + wantErr := "invalid operation at byte offset 110: jsontext: invalid character '\\n' within string (expecting non-control character)" if err.Error() != wantErr { t.Errorf("want error %q, got %q", wantErr, err.Error()) } diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 25f292b92f7..df4d97e2a82 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -3,6 +3,7 @@ package document import ( "bytes" + "encoding/json" "fmt" "io" "math" @@ -15,7 +16,6 @@ import ( "sync/atomic" "time" - "github.com/go-json-experiment/json" "github.com/klauspost/compress/gzip" "github.com/vespa-engine/vespa/client/go/internal/httputil" @@ -95,7 +95,7 @@ func NewClient(options ClientOptions, httpClients []httputil.Client) (*Client, e } c.gzippers.New = func() any { return gzip.NewWriter(io.Discard) } c.buffers.New = func() any { return &bytes.Buffer{} } - for i := 0; i < runtime.NumCPU(); i++ { + for range runtime.NumCPU() { go c.preparePending() } return c, nil @@ -335,7 +335,7 @@ func (c *Client) resultWithResponse(resp *http.Response, sentBytes int, result R } else { if result.Success() && c.options.TraceLevel > 0 { var jsonResponse struct { - Trace json.RawValue `json:"trace"` + Trace json.RawMessage `json:"trace"` } if err := json.Unmarshal(buf.Bytes(), &jsonResponse); err != nil { result = resultWithErr(result, fmt.Errorf("failed to decode json response: %w", err), elapsed) diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 74133fc73d8..878b7a98be3 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -34,7 +34,7 @@ type mockHTTPClient struct { func TestLeastBusyClient(t *testing.T) { httpClient := mock.HTTPClient{} var httpClients []httputil.Client - for i := 0; i < 4; i++ { + for i := range 4 { httpClients = append(httpClients, &mockHTTPClient{i, &httpClient}) } client, _ := NewClient(ClientOptions{}, httpClients) diff --git a/client/go/internal/vespa/document/throttler_test.go b/client/go/internal/vespa/document/throttler_test.go index eba8cbd2972..3cdecb22be4 100644 --- a/client/go/internal/vespa/document/throttler_test.go +++ b/client/go/internal/vespa/document/throttler_test.go @@ -13,7 +13,7 @@ func TestThrottler(t *testing.T) { if got, want := tr.TargetInflight(), int64(16); got != want { t.Errorf("got TargetInflight() = %d, but want %d", got, want) } - for i := 0; i < 65; i++ { + for range 65 { tr.Sent() tr.Success() } diff --git a/client/go/internal/vespa/load_env.go b/client/go/internal/vespa/load_env.go index 5cae03694bc..4eb1297e711 100644 --- a/client/go/internal/vespa/load_env.go +++ b/client/go/internal/vespa/load_env.go @@ -151,7 +151,7 @@ func nSpacedFields(s string, n int) []string { // pretty strict for now, can be more lenient if needed func isValidShellVariableName(s string) bool { - for i := 0; i < len(s); i++ { + for i := range len(s) { b := s[i] switch { case (b >= 'A' && b <= 'Z'): // ok diff --git a/client/go/internal/vespa/target_test.go b/client/go/internal/vespa/target_test.go index f886c9117a9..2b4cf485b83 100644 --- a/client/go/internal/vespa/target_test.go +++ b/client/go/internal/vespa/target_test.go @@ -20,7 +20,7 @@ func TestLocalTarget(t *testing.T) { client := &mock.HTTPClient{} lt := LocalTarget(client, TLSOptions{}, 0) assertServiceURL(t, "http://127.0.0.1:19071", lt, "deploy") - for i := 0; i < 2; i++ { + for range 2 { response := ` { "services": [ @@ -83,7 +83,7 @@ func TestCustomTargetWait(t *testing.T) { client.NextStatus(500) assertService(t, true, target, "", 0) // Fails multiple times - for i := 0; i < 3; i++ { + for range 3 { client.NextStatus(500) client.NextResponseError(io.EOF) } @@ -120,7 +120,7 @@ func TestCustomTargetAwaitDeployment(t *testing.T) { func TestCustomTargetCompatibleWith(t *testing.T) { client := &mock.HTTPClient{} target := CustomTarget(client, "http://192.0.2.42", TLSOptions{}, 0) - for i := 0; i < 3; i++ { + for range 3 { client.NextResponse(mock.HTTPResponse{ URI: "/state/v1/version", Status: 200, @@ -249,7 +249,7 @@ func TestLog(t *testing.T) { func TestCloudCompatibleWith(t *testing.T) { target, client := createCloudTarget(t, io.Discard) - for i := 0; i < 3; i++ { + for range 3 { client.NextResponse(mock.HTTPResponse{URI: "/cli/v1/", Status: 200, Body: []byte(`{"minVersion":"8.0.0"}`)}) } assert.Nil(t, target.CompatibleWith(version.MustParse("8.0.0"))) diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 2172bac94b8..4d557605e7b 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -1443,10 +1443,10 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.16.1": - version "1.16.1" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.16.1.tgz#73db3c48b975eeb06d0006481bde4f5f2d17d1cd" - integrity sha512-es2g3dq6Nb07iFxGk5GuHN20RwBZOsuDQN7izWIisUcv9r+d2C5jQxqmgkdebXgReWfiyUabcki6Fg77mSNrig== +"@remix-run/router@1.17.0": + version "1.17.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.17.0.tgz#fbb0add487478ef42247d5942e7a5d8a2e20095f" + integrity sha512-2D6XaHEVvkCn682XBnipbJjgZUU7xjLtA4dGJRBVUKpEaDYOZMENZoZjAOSb7qirxt5RupjzZxz4fK2FO+EFPw== "@rollup/rollup-android-arm-eabi@4.18.0": version "4.18.0" @@ -4868,19 +4868,19 @@ react-refresh@^0.14.2: integrity sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA== react-router-dom@^6: - version "6.23.1" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.23.1.tgz#30cbf266669693e9492aa4fc0dde2541ab02322f" - integrity sha512-utP+K+aSTtEdbWpC+4gxhdlPFwuEfDKq8ZrPFU65bbRJY+l706qjR7yaidBpo3MSeA/fzwbXWbKBI6ftOnP3OQ== + version "6.24.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.24.0.tgz#ec49dc38c49bb9bd25b310a8ae849268d3085e1d" + integrity sha512-960sKuau6/yEwS8e+NVEidYQb1hNjAYM327gjEyXlc6r3Skf2vtwuJ2l7lssdegD2YjoKG5l8MsVyeTDlVeY8g== dependencies: - "@remix-run/router" "1.16.1" - react-router "6.23.1" + "@remix-run/router" "1.17.0" + react-router "6.24.0" -react-router@6.23.1: - version "6.23.1" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.23.1.tgz#d08cbdbd9d6aedc13eea6e94bc6d9b29cb1c4be9" - integrity sha512-fzcOaRF69uvqbbM7OhvQyBTFDVrrGlsFdS3AL+1KfIBtGETibHzi3FkoTRyiDJnWNc2VxrfvR+657ROHjaNjqQ== +react-router@6.24.0: + version "6.24.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.24.0.tgz#aa46648f26b6525e07f908ad3e1ad2e68d131155" + integrity sha512-sQrgJ5bXk7vbcC4BxQxeNa5UmboFm35we1AFK0VvQaz9g0LzxEIuLOhHIoZ8rnu9BO21ishGeL9no1WB76W/eg== dependencies: - "@remix-run/router" "1.16.1" + "@remix-run/router" "1.17.0" react-textarea-autosize@8.3.4: version "8.3.4" diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java index 9bac6e4553d..67c45d58a95 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -295,7 +295,7 @@ public class OpenTelemetryConfigGenerator { { g.writeFieldName("exporters"); g.writeStartArray(); - g.writeString("file"); + g.writeString("otlphttp/gw"); g.writeEndArray(); } g.writeEndObject(); // metrics diff --git a/config/src/tests/failover/failover.cpp b/config/src/tests/failover/failover.cpp index 80d89f41c16..a679b2a0ecd 100644 --- a/config/src/tests/failover/failover.cpp +++ b/config/src/tests/failover/failover.cpp @@ -10,6 +10,7 @@ #include "config-my.h" #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/util/barrier.h> #include <vespa/log/log.h> LOG_SETUP("failover"); diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index d1cea49ee6f..5c6134aadbf 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -24,8 +24,8 @@ autopurge.snapRetainCount int default=15 # Vespa home is prepended if the file is relative myidFile string default="var/zookeeper/myid" -# Change from default of 1 Mb in zookeeper to 50 Mb -juteMaxBuffer int default=52428800 +# Change from default of 1 Mb in zookeeper to 100 Mb +juteMaxBuffer int default=104857600 myid int restart server[].id int diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 9bb3c86bb2c..78a4d71158f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -6,13 +6,13 @@ import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; import ai.vespa.http.HttpURL.Scheme; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; @@ -70,6 +70,7 @@ import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.deploy.Deployment; import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SecretStoreValidator; @@ -101,6 +102,7 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -111,7 +113,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -809,8 +810,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Logs ---------------------------------------------------------------- public HttpResponse getLogs(ApplicationId applicationId, Optional<DomainName> hostname, Query apiParams) { - HttpURL logServerURI = getLogServerURI(applicationId, hostname).withQuery(apiParams); - return logRetriever.getLogs(logServerURI, activationTime(applicationId)); + Exception exception = null; + for (var uri : getLogServerUris(applicationId, hostname)) { + try { + return logRetriever.getLogs(uri.withQuery(apiParams), activationTime(applicationId)); + } catch (RuntimeException e) { + exception = e; + log.log(Level.INFO, e.getMessage()); + } + } + return HttpErrorResponse.internalServerError(Exceptions.toMessageString(exception)); } // ---------------- Methods to do call against tester containers in hosted ------------------------------ @@ -1169,33 +1178,41 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - private HttpURL getLogServerURI(ApplicationId applicationId, Optional<DomainName> hostname) { + private List<HttpURL> getLogServerUris(ApplicationId applicationId, Optional<DomainName> hostname) { // Allow to get logs from a given hostname if the application is under the hosted-vespa tenant. // We make no validation that the hostname is actually allocated to the given application since // most applications under hosted-vespa are not known to the model, and it's OK for a user to get // logs for any host if they are authorized for the hosted-vespa tenant. if (hostname.isPresent() && HOSTED_VESPA_TENANT.equals(applicationId.tenant())) { int port = List.of(InfrastructureApplication.CONFIG_SERVER.id(), InfrastructureApplication.CONTROLLER.id()).contains(applicationId) ? 19071 : 8080; - return HttpURL.create(Scheme.http, hostname.get(), port).withPath(HttpURL.Path.parse("logs")); + return List.of(HttpURL.create(Scheme.http, hostname.get(), port).withPath(HttpURL.Path.parse("logs"))); } - Application application = getApplication(applicationId); - Collection<HostInfo> hostInfos = application.getModel().getHosts(); + ApplicationVersions applicationVersions = getActiveApplicationVersions(applicationId) + .orElseThrow(() -> new NotFoundException("Unable to get logs for for " + applicationId + " (application not found)")); + List<Pair<String, Integer>> hostInfo = logserverHostInfo(applicationVersions); + return hostInfo.stream() + .map(h -> HttpURL.create(Scheme.http, DomainName.of(h.getFirst()), h.getSecond(), HttpURL.Path.parse("logs"))) + .toList(); + } - HostInfo logServerHostInfo = hostInfos.stream() - .filter(host -> host.getServices().stream() + // Returns a list with hostname and port pairs for logserver container for all models/versions + private List<Pair<String, Integer>> logserverHostInfo(ApplicationVersions applicationVersions) { + return applicationVersions.applications().stream() + .peek(app -> log.log(Level.FINE, "Finding logserver host and port for version " + app.getVespaVersion())) + .map(Application::getModel) + .flatMap(model -> model.getHosts().stream()) + .filter(hostInfo -> hostInfo.getServices().stream() .anyMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver"))) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find host info for logserver")); - - ServiceInfo logService = logServerHostInfo.getServices().stream() - .filter(service -> LOGSERVER_CONTAINER.serviceName.equals(service.getServiceType())) - .findFirst() - .or(() -> logServerHostInfo.getServices().stream() - .filter(service -> CONTAINER.serviceName.equals(service.getServiceType())) - .findFirst()) - .orElseThrow(() -> new IllegalArgumentException("No container running on logserver host")); - int port = servicePort(logService); - return HttpURL.create(Scheme.http, DomainName.of(logServerHostInfo.getHostname()), port, HttpURL.Path.parse("logs")); + .map(hostInfo -> hostInfo.getServices().stream() + .filter(service -> LOGSERVER_CONTAINER.serviceName.equals(service.getServiceType())) + .findFirst() + .or(() -> hostInfo.getServices().stream() + .filter(service -> CONTAINER.serviceName.equals(service.getServiceType())) + .findFirst()) + .orElseThrow(() -> new IllegalArgumentException("No container running on logserver host"))) + .map(s -> new Pair<>(s.getHostName(), servicePort(s))) + .toList(); } private int servicePort(ServiceInfo serviceInfo) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java index fece5735ab4..553a82f11e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java @@ -66,7 +66,7 @@ public final class ApplicationVersions { if (application != null) return Optional.of(application); - // Does the latest version specify we can use it regardless? + // Does the latest version specify that we can use it regardless? Application latest = applications.get(latestVersion); if (latest.getModel().allowModelVersionMismatch(now)) return Optional.of(latest); @@ -97,7 +97,7 @@ public final class ApplicationVersions { return generation; } - List<Application> applications() { + public List<Application> applications() { return new ArrayList<>(applications.values()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index acfa8e455c0..89724f9853d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -38,7 +38,7 @@ public class LogRetriever { if (deployTime.isPresent() && Instant.now().isBefore(deployTime.get().plus(Duration.ofMinutes(2)))) return new EmptyResponse(); - return HttpErrorResponse.internalServerError("Failed to get logs: " + Exceptions.toMessageString(e)); + throw new RuntimeException("Failed to get logs from " + logServerUri, e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java index e588da8f1f9..b64b3fb05ab 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java @@ -33,7 +33,7 @@ public class ModelFactoryRegistry { } } - public Set<Version> allVersions() { return factories.keySet(); } + public Set<Version> allVersions() { return Set.copyOf(factories.keySet()); } /** * Returns the factory for the given version diff --git a/configutil/src/tests/host_filter/host_filter_test.cpp b/configutil/src/tests/host_filter/host_filter_test.cpp index eb44a5453d1..ae38119aeba 100644 --- a/configutil/src/tests/host_filter/host_filter_test.cpp +++ b/configutil/src/tests/host_filter/host_filter_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <lib/hostfilter.h> TEST("empty hostfilter includes any and all hosts") { diff --git a/configutil/src/tests/tags/tags_test.cpp b/configutil/src/tests/tags/tags_test.cpp index 6d9cc125298..b311f8dbd8f 100644 --- a/configutil/src/tests/tags/tags_test.cpp +++ b/configutil/src/tests/tags/tags_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <lib/tags.h> using namespace configdefinitions; diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 648c09d6d70..c6e3bea90a8 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1875,6 +1875,7 @@ "public void setIndexName(java.lang.String)", "public java.lang.String getIndexName()", "public int getN()", + "public boolean nIsExplicit()", "public void setN(int)", "public void disclose(com.yahoo.prelude.query.textualrepresentation.Discloser)", "public int hashCode()", @@ -5511,13 +5512,12 @@ "public com.yahoo.search.query.ranking.RankProperties getProperties()", "public void setListFeatures(boolean)", "public boolean getListFeatures()", - "public void setUseSignificance(boolean)", - "public boolean getUseSignificance()", "public com.yahoo.search.query.ranking.MatchPhase getMatchPhase()", "public com.yahoo.search.query.ranking.SecondPhase getSecondPhase()", "public com.yahoo.search.query.ranking.GlobalPhase getGlobalPhase()", "public com.yahoo.search.query.ranking.Matching getMatching()", "public com.yahoo.search.query.ranking.SoftTimeout getSoftTimeout()", + "public com.yahoo.search.query.ranking.Significance getSignificance()", "public com.yahoo.search.query.Sorting getSorting()", "public void setSorting(com.yahoo.search.query.Sorting)", "public void setSorting(java.lang.String)", @@ -5546,6 +5546,7 @@ "public static final java.lang.String SECOND_PHASE", "public static final java.lang.String GLOBAL_PHASE", "public static final java.lang.String DIVERSITY", + "public static final java.lang.String SIGNIFICANCE", "public static final java.lang.String SOFTTIMEOUT", "public static final java.lang.String MATCHING", "public static final java.lang.String FEATURES", @@ -7192,6 +7193,28 @@ ], "fields" : [ ] }, + "com.yahoo.search.query.ranking.Significance" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "java.lang.Cloneable" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public static com.yahoo.search.query.profile.types.QueryProfileType getArgumentType()", + "public void setUseModel(boolean)", + "public java.util.Optional getUseModel()", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public com.yahoo.search.query.ranking.Significance clone()", + "public bridge synthetic java.lang.Object clone()" + ], + "fields" : [ + "public static final java.lang.String USE_MODEL" + ] + }, "com.yahoo.search.query.ranking.SoftTimeout" : { "superClass" : "java.lang.Object", "interfaces" : [ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java index 8325101b4ad..ad2197cb39b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java @@ -363,12 +363,13 @@ public abstract class CompositeItem extends Item { @Override public void remove() { - owner.removing(current); wrapped.remove(); + owner.removing(current); } @Override public void set(Item newItem) { + if (newItem == current) return; owner.removing(current); owner.adding(newItem); current = newItem; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 931f9a1f1d9..778894e61b3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -20,14 +20,14 @@ import java.util.Objects; public final class WeakAndItem extends NonReducibleCompositeItem { /** The default N used if none is specified: 100 */ - public static final int defaultN = 100; + public static final int defaultN = 100; // TODO Vespa 9: Make private private int n; private String index; /** Creates a WAND item with default N */ public WeakAndItem() { - this(defaultN); + this(-1); } public WeakAndItem(int N) { @@ -72,25 +72,25 @@ public final class WeakAndItem extends NonReducibleCompositeItem { protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); - buffer.append(n); + buffer.append(getN()); buffer.append(") "); } - public int getN() { return n; } - + public int getN() { return nIsExplicit() ? n : defaultN; } + public boolean nIsExplicit() { return n > 0; } public void setN(int N) { this.n = N; } @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); - IntegerCompressor.putCompressedPositiveNumber(n, buffer); + IntegerCompressor.putCompressedPositiveNumber(getN(), buffer); putString(index, buffer); } @Override public void disclose(Discloser discloser) { super.disclose(discloser); - discloser.addProperty("N", n); + discloser.addProperty("N", getN()); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index 7dc94194f37..05a10efc12a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -118,7 +118,7 @@ public class AdvancedParser extends StructuredParser { } /** Returns whether the item is a specific word item */ - private boolean isTheWord(String word, Item item) { + private static boolean isTheWord(String word, Item item) { if (!(item instanceof WordItem)) { return false; } @@ -126,6 +126,11 @@ public class AdvancedParser extends StructuredParser { } + private static boolean needWeakAnd(Item topLevelItem, int n) { + return !(topLevelItem instanceof WeakAndItem topLevelWeakAnd) || + ((n != 0 || topLevelWeakAnd.nIsExplicit()) && (n != topLevelWeakAnd.getN())); + + } /** Returns the new top level, or null if the current item is not an operator */ private Item handleAdvancedOperator(Item topLevelItem, Item item, boolean topLevelIsClosed) { @@ -155,11 +160,11 @@ public class AdvancedParser extends StructuredParser { return topLevelItem; } else if (isTheWord("wand", item) || isTheWord("weakand", item)) { int n = consumeNumericArgument(); - if (n == 0) - n = WeakAndItem.defaultN; - if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n != ((WeakAndItem)topLevelItem).getN()) { + if (topLevelIsClosed || needWeakAnd(topLevelItem, n)) { WeakAndItem wand = new WeakAndItem(); - wand.setN(n); + if (n != 0) { + wand.setN(n); + } wand.addItem(topLevelItem); return wand; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java index 64b9b91936b..fbcd8935ebe 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java @@ -77,9 +77,8 @@ public class Match { @Override public boolean equals(Object o) { - if (! (o instanceof Match)) return false; + if (! (o instanceof Match other)) return false; - Match other = (Match)o; if (other.position != position) return false; if ( ! other.item.equals(item)) return false; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java index 284e98d4ceb..7994fe2fa41 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java @@ -32,7 +32,8 @@ public class ReplacingProductionRule extends ProductionRule { itemCount--; Match match = e.getNonreferencedMatch(i); - match.getItem().getParent().removeItem(match.getPosition()); + if (match.getItem().getParent() != null) + match.getItem().getParent().removeItem(match.getPosition()); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index a559846d5fe..f2505a47aa2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -16,6 +16,7 @@ import com.yahoo.search.query.ranking.RankFeatures; import com.yahoo.search.query.ranking.RankProperties; import com.yahoo.search.query.ranking.SecondPhase; import com.yahoo.search.query.ranking.SoftTimeout; +import com.yahoo.search.query.ranking.Significance; import com.yahoo.search.result.ErrorMessage; import java.util.Objects; @@ -49,6 +50,7 @@ public class Ranking implements Cloneable { public static final String SECOND_PHASE = "secondPhase"; public static final String GLOBAL_PHASE = "globalPhase"; public static final String DIVERSITY = "diversity"; + public static final String SIGNIFICANCE = "significance"; public static final String SOFTTIMEOUT = "softtimeout"; public static final String MATCHING = "matching"; public static final String FEATURES = "features"; @@ -75,6 +77,7 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(DIVERSITY, new QueryProfileFieldType(Diversity.getArgumentType()))); argumentType.addField(new FieldDescription(SOFTTIMEOUT, new QueryProfileFieldType(SoftTimeout.getArgumentType()))); argumentType.addField(new FieldDescription(MATCHING, new QueryProfileFieldType(Matching.getArgumentType()))); + argumentType.addField(new FieldDescription(SIGNIFICANCE, new QueryProfileFieldType(Significance.getArgumentType()))); argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature input")); // Repeated at the end of RankFeatures argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty")); argumentType.freeze(); @@ -118,7 +121,7 @@ public class Ranking implements Cloneable { private SoftTimeout softTimeout = new SoftTimeout(); - private boolean useSignificance = false; + private Significance significance = new Significance(); public Ranking(Query parent) { this.parent = parent; @@ -224,14 +227,6 @@ public class Ranking implements Cloneable { /** Returns whether rank features should be dumped with the result of this query, default false */ public boolean getListFeatures() { return listFeatures; } - /** Set whether to use significance in ranking */ - @com.yahoo.api.annotations.Beta - public void setUseSignificance(boolean useSignificance) { this.useSignificance = useSignificance; } - - /** Returns whether to use significance in ranking */ - @com.yahoo.api.annotations.Beta - public boolean getUseSignificance() { return useSignificance; } - /** Returns the match phase rank settings of this. This is never null. */ public MatchPhase getMatchPhase() { return matchPhase; } @@ -247,6 +242,10 @@ public class Ranking implements Cloneable { /** Returns the soft timeout settings of this. This is never null. */ public SoftTimeout getSoftTimeout() { return softTimeout; } + /** Returns the significance settings of this. This is never null. */ + @com.yahoo.api.annotations.Beta + public Significance getSignificance() { return significance; } + /** Returns the sorting spec of this query, or null if none is set */ public Sorting getSorting() { return sorting; } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 90d5e04d2b6..df9d95892ed 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -478,10 +478,7 @@ public class SelectParser implements Parser { if (annotations != null) { annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { - if (TARGET_HITS.equals(annotation_name)){ - weakAnd.setN((int)(annotation_value.asDouble())); - } - if (TARGET_NUM_HITS.equals(annotation_name)) { + if (TARGET_HITS.equals(annotation_name) || TARGET_NUM_HITS.equals(annotation_name)){ weakAnd.setN((int)(annotation_value.asDouble())); } }); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 29791712c9d..eed6962b14a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -22,6 +22,7 @@ import com.yahoo.search.query.ranking.Diversity; import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.query.ranking.Matching; import com.yahoo.search.query.ranking.SoftTimeout; +import com.yahoo.search.query.ranking.Significance; import com.yahoo.tensor.Tensor; import java.util.HashMap; @@ -108,6 +109,7 @@ public class QueryProperties extends Properties { map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.ENABLE), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getEnable(), (query, value) -> query.getRanking().getSoftTimeout().setEnable(asBoolean(value, true)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.FACTOR), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getFactor(), (query, value) -> query.getRanking().getSoftTimeout().setFactor(asDouble(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.TAILCOST), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getTailcost(), (query, value) -> query.getRanking().getSoftTimeout().setTailcost(asDouble(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SIGNIFICANCE, Significance.USE_MODEL), GetterSetter.of(query -> query.getRanking().getSignificance().getUseModel().orElse(false), (query, value) -> query.getRanking().getSignificance().setUseModel(asBoolean(value, false)))); map.put(CompoundName.fromComponents(Select.SELECT), GetterSetter.of(query -> query.getSelect().getGroupingExpressionString(), (query, value) -> query.getSelect().setGroupingExpressionString(asString(value, "")))); map.put(CompoundName.fromComponents(Select.SELECT, Select.WHERE), GetterSetter.of(query -> query.getSelect().getWhereString(), (query, value) -> query.getSelect().setWhereString(asString(value, "")))); map.put(CompoundName.fromComponents(Select.SELECT, Select.GROUPING), GetterSetter.of(query -> query.getSelect().getGroupingString(), (query, value) -> query.getSelect().setGroupingString(asString(value, "")))); diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/Significance.java b/container-search/src/main/java/com/yahoo/search/query/ranking/Significance.java new file mode 100644 index 00000000000..d4762e55910 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/Significance.java @@ -0,0 +1,68 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.ranking; + +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.Objects; +import java.util.Optional; + +/** + * The significance ranking settings of this query. + * + * @author MariusArhaug + */ +public class Significance implements Cloneable { + + /** The type representing the property arguments consumed by this */ + private static final QueryProfileType argumentType; + + public static final String USE_MODEL = "useModel"; + + static { + argumentType = new QueryProfileType(Ranking.SECOND_PHASE); + argumentType.setStrict(true); + argumentType.setBuiltin(true); + argumentType.addField(new FieldDescription(USE_MODEL, FieldType.booleanType)); + argumentType.freeze(); + } + public static QueryProfileType getArgumentType() { return argumentType; } + + private Boolean useModel = null; + + public void setUseModel(boolean useModel) { + this.useModel = useModel; + } + + public Optional<Boolean> getUseModel() { + return Optional.ofNullable(useModel); + } + + @Override + public int hashCode() { + return Objects.hashCode(this.useModel); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o instanceof Significance other) { + if ( ! Objects.equals(this.useModel, other.useModel)) return false; + return true; + } + return false; + } + + @Override + public Significance clone() { + try { + return (Significance) super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Won't happen", e); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 72a1a7d3430..afea39ac787 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -41,7 +41,9 @@ public class WeakAndReplacementSearcher extends Searcher { */ private void replaceOrItems(Query query) { Item root = query.getModel().getQueryTree().getRoot(); - int hits = query.properties().getInteger(WAND_HITS, WeakAndItem.defaultN); + int hits = query.getHits(); + Integer wandHits = query.properties().getInteger(WAND_HITS); + if (wandHits != null) hits = wandHits; query.getModel().getQueryTree().setRoot(replaceOrItems(root, hits)); if (root != query.getModel().getQueryTree().getRoot()) query.trace("Replaced OR by WeakAnd", true, 2); diff --git a/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java index c56b161edeb..aad892a501a 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java @@ -27,11 +27,15 @@ import com.yahoo.search.searchchain.Execution; @Beta @After(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND) public class OpportunisticWeakAndSearcher extends Searcher { + private static final CompoundName OPPORTUNISTIC_AND = CompoundName.from("weakAnd.opportunistic.and"); private static final CompoundName OPPORTUNISTIC_FACTOR = CompoundName.from("weakAnd.opportunistic.factor"); @Override public Result search(Query query, Execution execution) { + if (query.getHits() > WeakAndItem.defaultN) { + adjustWeakAndHeap(query.getModel().getQueryTree().getRoot(), query.getHits()); + } if (!query.properties().getBoolean(OPPORTUNISTIC_AND)) { return execution.search(query); } @@ -52,6 +56,18 @@ public class OpportunisticWeakAndSearcher extends Searcher { return execution.search(query); } + static Item adjustWeakAndHeap(Item item, int hits) { + if (item instanceof WeakAndItem weakAnd && hits > weakAnd.getN() && !weakAnd.nIsExplicit()) { + weakAnd.setN(hits); + } + if (item instanceof CompositeItem compositeItem) { + for (int i = 0; i < compositeItem.getItemCount(); i++) { + adjustWeakAndHeap(compositeItem.getItem(i), hits); + } + } + return item; + } + // returns targetHits for the first WeakAndItem found, -1 if none found. static int targetHits(Item item) { if (!(item instanceof CompositeItem compositeItem)) return -1; diff --git a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java index e3a559da8f9..3f72e98f18a 100644 --- a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java @@ -22,6 +22,7 @@ import com.yahoo.search.searchchain.Execution; import java.util.HashSet; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -51,8 +52,17 @@ public class SignificanceSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { + var ranking = query.getRanking(); var rankProfileName = query.getRanking().getProfile(); + Optional<Boolean> useSignificanceModelOverride = ranking.getSignificance().getUseModel(); + + if (useSignificanceModelOverride.isPresent() && !useSignificanceModelOverride.get()) { + return execution.search(query); + } + if (useSignificanceModelOverride.isPresent()) { + return calculateAndSetSignificance(query, execution); + } // Determine significance setup per schema for the given rank profile var perSchemaSetup = schemaInfo.newSession(query).schemas().stream() .collect(Collectors.toMap(Schema::name, schema -> @@ -60,6 +70,7 @@ public class SignificanceSearcher extends Searcher { // This will result in a failure later (in a "backend searcher") anyway. Optional.ofNullable(schema.rankProfiles().get(rankProfileName)) .map(RankProfile::useSignificanceModel).orElse(false))); + log.log(Level.FINE, () -> "Significance setup per schema: " + perSchemaSetup); var uniqueSetups = new HashSet<>(perSchemaSetup.values()); // Fail if the significance setup for the selected schemas are conflicting @@ -72,7 +83,7 @@ public class SignificanceSearcher extends Searcher { "(https://docs.vespa.ai/en/schemas.html#multiple-schemas). " + "Specify same 'significance' configuration for all selected schemas " + "(https://docs.vespa.ai/en/reference/schema-reference.html#significance).") - .formatted(rankProfileName, perSchemaSetup.keySet()))); + .formatted(rankProfileName, perSchemaSetup.keySet()))); return result; } @@ -80,8 +91,14 @@ public class SignificanceSearcher extends Searcher { var useSignificanceModel = uniqueSetups.iterator().next(); if (!useSignificanceModel) return execution.search(query); + return calculateAndSetSignificance(query, execution); + } + + private Result calculateAndSetSignificance(Query query, Execution execution) { Language language = query.getModel().getParsingLanguage(); Optional<SignificanceModel> model = significanceModelRegistry.getModel(language); + log.log(Level.FINE, () -> "Got model for language %s: %s" + .formatted(language, model.map(SignificanceModel::getId).orElse("<none>"))); if (model.isEmpty()) return execution.search(query); @@ -93,17 +110,17 @@ public class SignificanceSearcher extends Searcher { private void setIDF(Item root, SignificanceModel significanceModel) { if (root == null || root instanceof NullItem) return; - if (root instanceof WordItem) { - - var documentFrequency = significanceModel.documentFrequency(((WordItem) root).getWord()); + if (root instanceof WordItem wi) { + var word = wi.getWord(); + var documentFrequency = significanceModel.documentFrequency(word); long N = documentFrequency.corpusSize(); long nq_i = documentFrequency.frequency(); double idf = calculateIDF(N, nq_i); - - ((WordItem) root).setSignificance(idf); - } else if (root instanceof CompositeItem) { - for (int i = 0; i < ((CompositeItem) root).getItemCount(); i++) { - setIDF(((CompositeItem) root).getItem(i), significanceModel); + log.log(Level.FINE, () -> "Setting IDF for " + word + " to " + idf); + wi.setSignificance(idf); + } else if (root instanceof CompositeItem ci) { + for (int i = 0; i < ci.getItemCount(); i++) { + setIDF(ci.getItem(i), significanceModel); } } } @@ -111,7 +128,6 @@ public class SignificanceSearcher extends Searcher { public static double calculateIDF(long N, long nq_i) { return Math.log(1 + (N - nq_i + 0.5) / (nq_i + 0.5)); } - } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index a354006aa9b..850430484b2 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -233,7 +233,7 @@ public class VespaSerializer { boolean serialize(StringBuilder destination, EquivItem item) { String annotations = leafAnnotations(item); destination.append(getIndexName(item.getItem(0))).append(" contains "); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append("({").append(annotations).append("}"); } destination.append(EQUIV).append('('); @@ -251,7 +251,7 @@ public class VespaSerializer { destination.append('"'); } } - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } destination.append(')'); @@ -270,7 +270,7 @@ public class VespaSerializer { String annotations = nearAnnotations(item); destination.append(getIndexName(item.getItem(0))).append(" contains "); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append('(').append(annotations); } destination.append(NEAR).append('('); @@ -284,7 +284,7 @@ public class VespaSerializer { escape(close.getIndexedString(), destination).append('"'); } destination.append(')'); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } return false; @@ -415,15 +415,15 @@ public class VespaSerializer { } else if (rightOpen) { boundsAnnotation = BOUNDS + ": " + "\"" + BOUNDS_RIGHT_OPEN + "\""; } - if (annotations.length() > 0 || boundsAnnotation.length() > 0) { + if (!annotations.isEmpty() || !boundsAnnotation.isEmpty()) { destination.append("({"); } initLen = destination.length(); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(annotations); } comma(destination, initLen); - if (boundsAnnotation.length() > 0) { + if (!boundsAnnotation.isEmpty()) { destination.append(boundsAnnotation); } if (initLen != annotations.length()) { @@ -434,7 +434,7 @@ public class VespaSerializer { .append(", ").append(intItem.getFromLimit().number()) .append(", ").append(intItem.getToLimit().number()) .append(")"); - if (annotations.length() > 0 || boundsAnnotation.length() > 0) { + if (!annotations.isEmpty() || !boundsAnnotation.isEmpty()) { destination.append(")"); } } @@ -442,7 +442,7 @@ public class VespaSerializer { private void annotatedNumberImage(IntItem item, String rawNumber, StringBuilder image) { String annotations = leafAnnotations(item); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { image.append("({").append(annotations).append("}"); } if ('-' == rawNumber.charAt(0)) { @@ -453,7 +453,7 @@ public class VespaSerializer { if ('-' == rawNumber.charAt(0)) { image.append(')'); } - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { image.append(')'); } } @@ -533,7 +533,7 @@ public class VespaSerializer { destination.append(normalizeIndexName(fuzzy.getIndexName())).append(" contains "); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append('(').append(annotations); } @@ -542,7 +542,7 @@ public class VespaSerializer { escape(fuzzy.getIndexedString(), destination).append('"'); destination.append(')'); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } return false; @@ -591,7 +591,7 @@ public class VespaSerializer { String annotations = NearSerializer.nearAnnotations(item); destination.append(getIndexName(item.getItem(0))).append(" contains "); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append('(').append(annotations); } destination.append(ONEAR).append('('); @@ -605,7 +605,7 @@ public class VespaSerializer { escape(close.getIndexedString(), destination).append('"'); } destination.append(')'); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } return false; @@ -681,7 +681,7 @@ public class VespaSerializer { destination.append("({"); serializeOrigin(destination, image, offset, length); String annotations = leafAnnotations(phrase); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(", ").append(annotations); } if (phrase.getSegmentingRule() == SegmentingRule.BOOLEAN_AND) { @@ -710,7 +710,7 @@ public class VespaSerializer { if (includeField) destination.append(normalizeIndexName(phrase.getIndexName())).append(" contains "); - if (annotations.length() > 0) + if (!annotations.isEmpty()) destination.append("({").append(annotations).append("}"); destination.append(PHRASE).append('('); @@ -730,7 +730,7 @@ public class VespaSerializer { } } destination.append(')'); - if (annotations.length() > 0) + if (!annotations.isEmpty()) destination.append(')'); return false; } @@ -780,7 +780,7 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, GeoLocationItem item) { String annotations = leafAnnotations(item); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append("({").append(annotations).append("}"); } destination.append(GEO_LOCATION).append('('); @@ -896,7 +896,7 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, RangeItem range) { String annotations = leafAnnotations(range); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append("{").append(annotations).append("}"); } destination.append(RANGE).append('(') @@ -953,7 +953,7 @@ public class VespaSerializer { String annotations = leafAnnotations(alternatives); Substring origin = alternatives.getOrigin(); boolean isFromQuery = alternatives.isFromQuery(); - boolean needsAnnotations = annotations.length() > 0 || origin != null || !isFromQuery; + boolean needsAnnotations = !annotations.isEmpty() || origin != null || !isFromQuery; if (includeField) { destination.append(normalizeIndexName(alternatives.getIndexName())).append(" contains "); @@ -973,7 +973,7 @@ public class VespaSerializer { comma(destination, initLen); destination.append(IMPLICIT_TRANSFORMS).append(": false"); } - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { comma(destination, initLen); destination.append(annotations); } @@ -985,7 +985,7 @@ public class VespaSerializer { int initLen = destination.length(); List<WordAlternativesItem.Alternative> sortedAlternatives = new ArrayList<>(alternatives.getAlternatives()); // ensure most precise forms first - Collections.sort(sortedAlternatives, (x, y) -> Double.compare(y.exactness, x.exactness)); + sortedAlternatives.sort((x, y) -> Double.compare(y.exactness, x.exactness)); for (WordAlternativesItem.Alternative alternative : sortedAlternatives) { comma(destination, initLen); destination.append('"'); @@ -1049,7 +1049,7 @@ public class VespaSerializer { } private boolean needsAnnotationBlock(WeakAndItem item) { - return nonDefaultTargetNumHits(item); + return item.nIsExplicit(); } @Override @@ -1057,7 +1057,7 @@ public class VespaSerializer { if (needsAnnotationBlock(item)) { destination.append("({"); } - if (nonDefaultTargetNumHits(item)) { + if (item.nIsExplicit()) { destination.append(TARGET_NUM_HITS).append(": ").append(item.getN()); } if (needsAnnotationBlock(item)) { @@ -1067,9 +1067,6 @@ public class VespaSerializer { return true; } - private boolean nonDefaultTargetNumHits(WeakAndItem w) { - return w.getN() != WeakAndItem.defaultN; - } } private static class WeightedSetSerializer extends Serializer<WeightedSetItem> { @@ -1157,7 +1154,7 @@ public class VespaSerializer { StringBuilder wordAnnotations = new StringBuilder(WordSerializer.wordAnnotations(w)); String leafAnnotations = leafAnnotations(w); - if (leafAnnotations.length() > 0) { + if (!leafAnnotations.isEmpty()) { comma(wordAnnotations, 0); wordAnnotations.append(leafAnnotations(w)); } @@ -1472,7 +1469,7 @@ public class VespaSerializer { for (Iterator<Entry<Object, Integer>> i = weightedSet.getTokens(); i.hasNext();) { tokens.add(i.next()); } - Collections.sort(tokens, tokenComparator); + tokens.sort(tokenComparator); for (Entry<Object, Integer> entry : tokens) { comma(destination, initLen); destination.append('"'); @@ -1491,14 +1488,14 @@ public class VespaSerializer { int incomingLen = destination.length(); String annotations = leafAnnotations(weightedSet); - if (optionalAnnotations.length() > 0 || annotations.length() > 0) { + if (!optionalAnnotations.isEmpty() || !annotations.isEmpty()) { destination.append("({"); } preAnnotationValueLen = destination.length(); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(annotations); } - if (optionalAnnotations.length() > 0) { + if (!optionalAnnotations.isEmpty()) { comma(destination, preAnnotationValueLen); destination.append(optionalAnnotations); } @@ -1592,7 +1589,7 @@ public class VespaSerializer { } private static String normalizeIndexName(String indexName) { - if (indexName.length() == 0) { + if (indexName.isEmpty()) { return "default"; } else { return indexName; @@ -1600,12 +1597,12 @@ public class VespaSerializer { } private static void annotatedTerm(StringBuilder destination, IndexedItem w, String annotations) { - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append("({").append(annotations).append("}"); } destination.append('"'); escape(w.getIndexedString(), destination).append('"'); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index fb4ec5ba872..1f377afdb5e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -434,18 +434,12 @@ public class YqlParser implements Parser { } private ParsedDegree degreesFromArg(OperatorNode<ExpressionOperator> ast, boolean first) { - Object arg = null; - switch (ast.getOperator()) { - case LITERAL: - arg = ast.getArgument(0); - break; - case READ_FIELD: - arg = ast.getArgument(1); - break; - default: - throw newUnexpectedArgumentException(ast.getOperator(), - ExpressionOperator.READ_FIELD, ExpressionOperator.PROPREF); - } + Object arg = switch (ast.getOperator()) { + case LITERAL -> ast.getArgument(0); + case READ_FIELD -> ast.getArgument(1); + default -> throw newUnexpectedArgumentException(ast.getOperator(), + ExpressionOperator.READ_FIELD, ExpressionOperator.PROPREF); + }; if (arg instanceof Number n) { return new ParsedDegree(n.doubleValue(), first, !first); } @@ -822,8 +816,10 @@ public class YqlParser implements Parser { // Set grammar-specific annotations if (WEAKAND_GRAMMARS.contains(grammar) && item instanceof WeakAndItem weakAndItem) { - weakAndItem.setN(getAnnotation(ast, TARGET_HITS, Integer.class, WeakAndItem.defaultN, - "'targetHits' (N) for weak and")); + Integer targetNumHits = getAnnotation(ast, TARGET_HITS, Integer.class, null, "'targetHits' (N) for weak and"); + if (targetNumHits != null) { + weakAndItem.setN(targetNumHits); + } } return item; } @@ -845,17 +841,16 @@ public class YqlParser implements Parser { } private String getStringContents(OperatorNode<ExpressionOperator> operator) { - switch (operator.getOperator()) { - case LITERAL: - return operator.getArgument(0, String.class); - case VARREF: + return switch (operator.getOperator()) { + case LITERAL -> operator.getArgument(0, String.class); + case VARREF -> { Preconditions.checkState(userQuery != null, - "properties must be available when trying to fetch user input"); - return userQuery.properties().getString(operator.getArgument(0, String.class)); - default: - throw newUnexpectedArgumentException(operator.getOperator(), - ExpressionOperator.LITERAL, ExpressionOperator.VARREF); - } + "properties must be available when trying to fetch user input"); + yield userQuery.properties().getString(operator.getArgument(0, String.class)); + } + default -> throw newUnexpectedArgumentException(operator.getOperator(), + ExpressionOperator.LITERAL, ExpressionOperator.VARREF); + }; } private void propagateUserInputAnnotationsRecursively(OperatorNode<ExpressionOperator> ast, Item item) { @@ -1139,16 +1134,15 @@ public class YqlParser implements Parser { negative = "-"; ast = ast.getArgument(0); } - switch (ast.getOperator()) { - case VARREF: + return switch (ast.getOperator()) { + case VARREF -> { Preconditions.checkState(userQuery != null, - "properties must be available when trying to fetch user input"); - return negative + userQuery.properties().getString(ast.getArgument(0, String.class)); - case LITERAL: - return negative + ast.getArgument(0).toString(); - default: - throw new IllegalArgumentException("Expected VARREF or LITERAL, got " + ast.getOperator()); - } + "properties must be available when trying to fetch user input"); + yield negative + userQuery.properties().getString(ast.getArgument(0, String.class)); + } + case LITERAL -> negative + ast.getArgument(0).toString(); + default -> throw new IllegalArgumentException("Expected VARREF or LITERAL, got " + ast.getOperator()); + }; } private String fetchConditionWord(OperatorNode<ExpressionOperator> ast) { @@ -1301,17 +1295,21 @@ public class YqlParser implements Parser { } else { Limit from; Limit to; - if (BOUNDS_OPEN.equals(bounds)) { - from = new Limit(lowerArg, false); - to = new Limit(upperArg, false); - } else if (BOUNDS_LEFT_OPEN.equals(bounds)) { - from = new Limit(lowerArg, false); - to = new Limit(upperArg, true); - } else if (BOUNDS_RIGHT_OPEN.equals(bounds)) { - from = new Limit(lowerArg, true); - to = new Limit(upperArg, false); - } else { - throw newUnexpectedArgumentException(bounds, BOUNDS_OPEN, BOUNDS_LEFT_OPEN, BOUNDS_RIGHT_OPEN); + switch (bounds) { + case BOUNDS_OPEN -> { + from = new Limit(lowerArg, false); + to = new Limit(upperArg, false); + } + case BOUNDS_LEFT_OPEN -> { + from = new Limit(lowerArg, false); + to = new Limit(upperArg, true); + } + case BOUNDS_RIGHT_OPEN -> { + from = new Limit(lowerArg, true); + to = new Limit(upperArg, false); + } + default -> + throw newUnexpectedArgumentException(bounds, BOUNDS_OPEN, BOUNDS_LEFT_OPEN, BOUNDS_RIGHT_OPEN); } return new IntItem(from, to, getIndex(args.get(0))); } @@ -1418,7 +1416,7 @@ public class YqlParser implements Parser { private Item instantiateWordAlternativesItem(String field, OperatorNode<ExpressionOperator> ast) { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); - Preconditions.checkArgument(args.size() >= 1, "Expected 1 or more arguments, got %s.", args.size()); + Preconditions.checkArgument(!args.isEmpty(), "Expected 1 or more arguments, got %s.", args.size()); Preconditions.checkArgument(args.get(0).getOperator() == ExpressionOperator.MAP, "Expected MAP, got %s.", args.get(0).getOperator()); @@ -1566,7 +1564,7 @@ public class YqlParser implements Parser { List<String> words = segmenter.segment(toSegment, usedLanguage); TaggableItem wordItem; - if (words.size() == 0) { + if (words.isEmpty()) { wordItem = new WordItem(wordData, fromQuery); } else if (words.size() == 1 || !phraseArgumentSupported(parent)) { wordItem = new WordItem(words.get(0), fromQuery); @@ -1909,18 +1907,7 @@ public class YqlParser implements Parser { return new IllegalArgumentException(out.toString()); } - private static final class ConnectedItem { - - final double weight; - final int toId; - final TaggableItem fromItem; - - ConnectedItem(TaggableItem fromItem, int toId, double weight) { - this.weight = weight; - this.toId = toId; - this.fromItem = fromItem; - } - } + private record ConnectedItem(TaggableItem fromItem, int toId, double weight) { } private class AnnotationPropagator extends QueryVisitor { diff --git a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java index 01f8e86ba4c..eb17a1560e0 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java @@ -5,9 +5,10 @@ import com.yahoo.search.Query; import com.yahoo.search.query.Sorting; import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Arne Bergene Fossaa @@ -96,4 +97,18 @@ public class RankingTestCase { assertEquals("17.5", ranking.getProperties().get("vespa.hitcollector.secondphase.rankscoredroplimit").get(0)); } + @Test + void testSignificanceUseModel() { + var query = new Query("?query=test"); + var ranking = query.getRanking(); + assertTrue(ranking.getSignificance().getUseModel().isEmpty()); + + var query1 = new Query("?query=test&ranking.significance.useModel=true"); + var ranking1 = query1.getRanking(); + assertEquals(true, ranking1.getSignificance().getUseModel().get()); + + var query2 = new Query("?query=test&ranking.significance.useModel=false"); + var ranking2 = query2.getRanking(); + assertEquals(false, ranking2.getSignificance().getUseModel().get()); + } } diff --git a/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java index f41c74a4b8e..cdfbdd62765 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java @@ -11,7 +11,12 @@ import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WordItem; import org.junit.jupiter.api.Test; +import static com.yahoo.search.searchers.OpportunisticWeakAndSearcher.targetHits; +import static com.yahoo.search.searchers.OpportunisticWeakAndSearcher.weakAnd2AndRecurse; +import static com.yahoo.search.searchers.OpportunisticWeakAndSearcher.adjustWeakAndHeap; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; public class OpportunisticWeakAndSearcherTestCase { @@ -30,21 +35,47 @@ public class OpportunisticWeakAndSearcherTestCase { @Test public void requireThatWeakAndIsDetected() { - assertEquals(-1, OpportunisticWeakAndSearcher.targetHits(new OrItem())); - assertEquals(-1, OpportunisticWeakAndSearcher.targetHits(new WeakAndItem(33))); - assertEquals(-1, OpportunisticWeakAndSearcher.targetHits(addItem(new WeakAndItem(33), new TrueItem()))); - assertEquals(33, OpportunisticWeakAndSearcher.targetHits(addItem(addItem(new WeakAndItem(33), new TrueItem()), new TrueItem()))); - assertEquals(77, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new OrItem(), new WeakAndItem(77)))); - assertEquals(77, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new AndItem(), new WeakAndItem(77)))); - assertEquals(-1, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new OrItem(), new AndItem()))); + assertEquals(-1, targetHits(new OrItem())); + assertEquals(-1, targetHits(new WeakAndItem(33))); + assertEquals(-1, targetHits(addItem(new WeakAndItem(33), new TrueItem()))); + assertEquals(33, targetHits(addItem(addItem(new WeakAndItem(33), new TrueItem()), new TrueItem()))); + assertEquals(77, targetHits(buildQueryItem(new OrItem(), new WeakAndItem(77)))); + assertEquals(77, targetHits(buildQueryItem(new AndItem(), new WeakAndItem(77)))); + assertEquals(-1, targetHits(buildQueryItem(new OrItem(), new AndItem()))); } @Test public void requireThatWeakAndIsReplacedWithAnd() { assertEquals(buildQueryItem(new OrItem(), new AndItem()), - OpportunisticWeakAndSearcher.weakAnd2AndRecurse(buildQueryItem(new OrItem(), new WeakAndItem()))); + weakAnd2AndRecurse(buildQueryItem(new OrItem(), new WeakAndItem()))); assertEquals(buildQueryItem(new AndItem(), new AndItem()), - OpportunisticWeakAndSearcher.weakAnd2AndRecurse(buildQueryItem(new AndItem(), new WeakAndItem()))); + weakAnd2AndRecurse(buildQueryItem(new AndItem(), new WeakAndItem()))); + } + + private static WeakAndItem try2Adjust(WeakAndItem item, int hits) { + adjustWeakAndHeap(item, hits); + return item; + } + + private static WeakAndItem try2Adjust(WeakAndItem item, CompositeItem parent, int hits) { + parent.addItem(item); + adjustWeakAndHeap(parent, hits); + return item; + } + + @Test + public void requireThatDefaultWeakAndHeapIsAdjustedUpToHits() { + assertEquals(1000, try2Adjust(new WeakAndItem(), 1000).getN()); + assertFalse(try2Adjust(new WeakAndItem(), 10).nIsExplicit()); + + assertEquals(1000, try2Adjust(new WeakAndItem(), new OrItem(), 1000).getN()); + assertFalse(try2Adjust(new WeakAndItem(), new OrItem(), 10).nIsExplicit()); + } + @Test + public void requireThatNonDefaultWeakAndHeapIsNotAdjustedUpToHits() { + assertEquals(33, try2Adjust(new WeakAndItem(33), 1000).getN()); + assertEquals(33, try2Adjust(new WeakAndItem(33), 11).getN()); + assertEquals(WeakAndItem.defaultN, try2Adjust(new WeakAndItem(WeakAndItem.defaultN), 1000).getN()); } } diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java index eaa66755608..be68c87efb3 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -79,6 +79,40 @@ public class SignificanceSearcherTest { WordItem w0 = (WordItem) root.getItem(0); assertEquals(helloSignificanceValue, w0.getSignificance()); } + + @Test + void testSignificanceValueOnSimpleQueryWithRankingOverride() { + Query q1 = new Query("?query=hello&ranking.significance.useModel=true"); + q1.getRanking().setProfile("significance-ranking"); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("hello", true); + root.addItem(tmp); + + q1.getModel().getQueryTree().setRoot(root); + + SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get(); + var helloFrequency = model.documentFrequency("hello"); + var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency()); + Result r = createExecution(searcher).search(q1); + + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + assertEquals(helloSignificanceValue, w0.getSignificance()); + + Query q2 = new Query("?query=hello&ranking.significance.useModel=false"); + q2.getRanking().setProfile("significance-ranking"); + root = new AndItem(); + tmp = new WordItem("hello", true); + root.addItem(tmp); + + q2.getModel().getQueryTree().setRoot(root); + Result r2 = createExecution(searcher).search(q2); + root = (AndItem) r2.getQuery().getModel().getQueryTree().getRoot(); + WordItem w1 = (WordItem) root.getItem(0); + assertEquals(0.0, w1.getSignificance()); + } + @Test void testSignificanceValueOnSimpleANDQuery() { diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 3a61c03a9f9..c7c535c84c4 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -68,8 +68,8 @@ <assertj.vespa.version>3.26.0</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.747</aws-sdk.vespa.version> - <athenz.vespa.version>1.11.59</athenz.vespa.version> + <aws-sdk.vespa.version>1.12.749</aws-sdk.vespa.version> + <athenz.vespa.version>1.11.60</athenz.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -147,8 +147,8 @@ <surefire.vespa.version>3.3.0</surefire.vespa.version> <velocity.vespa.version>2.3</velocity.vespa.version> <velocity.tools.vespa.version>3.1</velocity.tools.vespa.version> - <wiremock.vespa.version>3.6.0</wiremock.vespa.version> - <woodstox.vespa.version>6.6.2</woodstox.vespa.version> + <wiremock.vespa.version>3.7.0</wiremock.vespa.version> + <woodstox.vespa.version>7.0.0</woodstox.vespa.version> <stax2-api.vespa.version>4.2.2</stax2-api.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> <zero-allocation-hashing.vespa.version>0.16</zero-allocation-hashing.vespa.version> @@ -171,7 +171,7 @@ <maven-bundle-plugin.vespa.version>5.1.9</maven-bundle-plugin.vespa.version> <maven-compiler-plugin.vespa.version>3.13.0</maven-compiler-plugin.vespa.version> <maven-core.vespa.version>3.9.8</maven-core.vespa.version> - <maven-dependency-plugin.vespa.version>3.7.0</maven-dependency-plugin.vespa.version> + <maven-dependency-plugin.vespa.version>3.7.1</maven-dependency-plugin.vespa.version> <maven-deploy-plugin.vespa.version>3.1.2</maven-deploy-plugin.vespa.version> <maven-enforcer-plugin.vespa.version>3.5.0</maven-enforcer-plugin.vespa.version> <maven-failsafe-plugin.vespa.version>3.3.0</maven-failsafe-plugin.vespa.version> diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java index c6eccdacf26..465d7da5e8e 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -6,7 +6,9 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.PositionDataType; import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.TokenBuffer; import com.yahoo.document.update.ValueUpdate; @@ -41,6 +43,11 @@ public class SingleValueReader { } public static FieldValue readSingleValue(TokenBuffer buffer, DataType expectedType, boolean ignoreUndefinedFields) { + if (expectedType instanceof TensorDataType) { + FieldValue fieldValue = expectedType.createFieldValue(); + TensorReader.fillTensor(buffer, (TensorFieldValue) fieldValue); + return fieldValue; + } if (buffer.current().isScalarValue()) { return readAtomic(buffer.currentText(), expectedType); } else { diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index 3aa6dc96e56..a8166e9fa5e 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -37,6 +37,18 @@ public class TensorReader { // MUST be kept in sync with com.yahoo.tensor.serialization.JsonFormat.decode in vespajlib static void fillTensor(TokenBuffer buffer, TensorFieldValue tensorFieldValue) { Tensor.Builder builder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); + if (buffer.current() == JsonToken.VALUE_STRING + && builder instanceof IndexedTensor.BoundBuilder indexedBuilder) + { + double[] decoded = decodeHexString(buffer.currentText(), builder.type().valueType()); + if (decoded.length == 0) + throw new IllegalArgumentException("Bad string input for tensor with type " + builder.type()); + for (int i = 0; i < decoded.length; i++) { + indexedBuilder.cellByDirectIndex(i, decoded[i]); + } + tensorFieldValue.assign(builder.build()); + return; + } expectOneOf(buffer.current(), JsonToken.START_OBJECT, JsonToken.START_ARRAY); int initNesting = buffer.nesting(); while (true) { diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index e72d3720024..a61ad87ca8d 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -175,6 +175,8 @@ public class JsonReaderTestCase { new TensorDataType(new TensorType.Builder().indexed("x", 2).indexed("y", 3).build()))); x.addField(new Field("dense_int8_tensor", new TensorDataType(TensorType.fromSpec("tensor<int8>(x[2],y[3])")))); + x.addField(new Field("dense_float_tensor", + new TensorDataType(TensorType.fromSpec("tensor<float>(y[3])")))); x.addField(new Field("dense_unbound_tensor", new TensorDataType(new TensorType.Builder().indexed("x").indexed("y").build()))); x.addField(new Field("mixed_tensor", @@ -1780,7 +1782,7 @@ public class JsonReaderTestCase { "remove": "id:unittest:smoke::whee", "what is love": "baby, do not hurt me... much } - ]"""; + ]"""; // " new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); } @@ -1996,6 +1998,30 @@ public class JsonReaderTestCase { "values": "020304050607" }""", "dense_int8_tensor"), "dense_int8_tensor"); assertTrue(tensor instanceof IndexedTensor); // this matters for performance + tensor = assertTensorField(expected, + createPutWithTensor(""" + "020304050607" + """, "dense_int8_tensor"), "dense_int8_tensor"); + assertTrue(tensor instanceof IndexedTensor); // this matters for performance + builder = Tensor.Builder.of(TensorType.fromSpec("tensor<float>(y[3])")); + builder.cell().label("y", 0).value(42.0); + builder.cell().label("y", 1).value(-0.125); + builder.cell().label("y", 2).value(Double.POSITIVE_INFINITY); + expected = builder.build(); + tensor = assertTensorField(expected, + createPutWithTensor(""" + "42280000be0000007f800000" + """, "dense_float_tensor"), "dense_float_tensor"); + try { + assertTensorField(expected, + createPutWithTensor(""" + "" + """, "dense_int8_tensor"), "dense_int8_tensor"); + } + catch (IllegalArgumentException e) { + assertTrue(Exceptions.toMessageString(e).contains( + "Bad string input for tensor with type tensor<int8>(x[2],y[3])")); + } } @Test @@ -2018,6 +2044,13 @@ public class JsonReaderTestCase { """; var put = createPutWithTensor(inputJson(mixedJson), "mixed_bfloat16_tensor"); Tensor tensor = assertTensorField(expected, put, "mixed_bfloat16_tensor"); + mixedJson = """ + { + "blocks":{"foo":"400040404080", "bar":"40A040C040E0"} + } + """; + put = createPutWithTensor(inputJson(mixedJson), "mixed_bfloat16_tensor"); + tensor = assertTensorField(expected, put, "mixed_bfloat16_tensor"); } /** Tests parsing of various tensor values set at the root, i.e. no 'cells', 'blocks' or 'values' */ diff --git a/document/src/tests/annotation/annotation_test.cpp b/document/src/tests/annotation/annotation_test.cpp index d95e5c7d1b9..c5635b9f4f9 100644 --- a/document/src/tests/annotation/annotation_test.cpp +++ b/document/src/tests/annotation/annotation_test.cpp @@ -17,7 +17,7 @@ #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/doublefieldvalue.h> #include <vespa/document/fieldvalue/structfieldvalue.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <memory> using std::unique_ptr; diff --git a/document/src/tests/base/documentid_test.cpp b/document/src/tests/base/documentid_test.cpp index bf61440af52..66632fc1bf6 100644 --- a/document/src/tests/base/documentid_test.cpp +++ b/document/src/tests/base/documentid_test.cpp @@ -3,7 +3,7 @@ #include <vespa/document/base/documentid.h> #include <vespa/document/base/idstringexception.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using vespalib::string; diff --git a/document/src/tests/datatype/datatype_test.cpp b/document/src/tests/datatype/datatype_test.cpp index f81a6be6768..1362afe8dbc 100644 --- a/document/src/tests/datatype/datatype_test.cpp +++ b/document/src/tests/datatype/datatype_test.cpp @@ -7,7 +7,7 @@ #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/eval/eval/value_type.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> using namespace document; diff --git a/document/src/tests/datatype/referencedatatype_test.cpp b/document/src/tests/datatype/referencedatatype_test.cpp index 2442e51e7ec..54489feb2f2 100644 --- a/document/src/tests/datatype/referencedatatype_test.cpp +++ b/document/src/tests/datatype/referencedatatype_test.cpp @@ -3,7 +3,7 @@ #include <vespa/document/base/field.h> #include <vespa/document/datatype/referencedatatype.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <ostream> #include <sstream> diff --git a/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp b/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp index c56b9ca55a6..edb7f8e4f45 100644 --- a/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp +++ b/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/document_type_repo_factory.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using vespalib::string; using namespace document::config_builder; diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 9da03d6001b..7c571df257c 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -33,8 +33,11 @@ #include <vespa/eval/eval/value.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/test/test_data.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/exceptions.h> +#include <filesystem> #include <fstream> #include <unistd.h> @@ -44,6 +47,7 @@ using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; using vespalib::nbostream; +using vespalib::test::TestDataBase; namespace document { @@ -87,31 +91,38 @@ void testRoundtripSerialize(const UpdateType& update, const DataType &type) { } } -void -writeBufferToFile(const nbostream &buf, const vespalib::string &fileName) +} + +class DocumentUpdateTest : public ::testing::Test, public vespalib::test::TestData<DocumentUpdateTest> { +protected: + DocumentUpdateTest(); + ~DocumentUpdateTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +DocumentUpdateTest::DocumentUpdateTest() + : ::testing::Test(), + vespalib::test::TestData<DocumentUpdateTest>() { - auto file = std::fstream(fileName, std::ios::out | std::ios::binary); - file.write(buf.data(), buf.size()); - assert(file.good()); - file.close(); } -nbostream -readBufferFromFile(const vespalib::string &fileName) +DocumentUpdateTest::~DocumentUpdateTest() = default; + +void +DocumentUpdateTest::SetUpTestSuite() { - auto file = std::fstream(fileName, std::ios::in | std::ios::binary | std::ios::ate); - auto size = file.tellg(); - file.seekg(0); - vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(size); - file.read(static_cast<char *>(buf.get()), size); - assert(file.good()); - file.close(); - return nbostream(std::move(buf), size); + setup_test_data(TEST_PATH("data"), "documentupdate-build-data"); + std::filesystem::create_directory(build_testdata()); } +void +DocumentUpdateTest::TearDownTestSuite() +{ + tear_down_test_data(); } -TEST(DocumentUpdateTest, testSimpleUsage) +TEST_F(DocumentUpdateTest, testSimpleUsage) { DocumenttypesConfigBuilderHelper builder; builder.document(42, "test", @@ -205,7 +216,7 @@ TEST(DocumentUpdateTest, testSimpleUsage) } } -TEST(DocumentUpdateTest, testClearField) +TEST_F(DocumentUpdateTest, testClearField) { // Create a document. TestDocMan docMan; @@ -220,7 +231,7 @@ TEST(DocumentUpdateTest, testClearField) EXPECT_FALSE(doc->getValue("headerval")); } -TEST(DocumentUpdateTest, testUpdateApplySingleValue) +TEST_F(DocumentUpdateTest, testUpdateApplySingleValue) { // Create a document. TestDocMan docMan; @@ -235,7 +246,7 @@ TEST(DocumentUpdateTest, testUpdateApplySingleValue) EXPECT_EQ(9, doc->getValue("headerval")->getAsInt()); } -TEST(DocumentUpdateTest, testUpdateArray) +TEST_F(DocumentUpdateTest, testUpdateArray) { // Create a document. TestDocMan docMan; @@ -314,7 +325,7 @@ createAddUpdate(int key, int weight) { return upd; } -TEST(DocumentUpdateTest, testUpdateWeightedSet) +TEST_F(DocumentUpdateTest, testUpdateWeightedSet) { // Create a test document TestDocMan docMan; @@ -427,7 +438,7 @@ WeightedSetAutoCreateFixture::WeightedSetAutoCreateFixture() } } // anon ns -TEST(DocumentUpdateTest, testIncrementNonExistingAutoCreateWSetField) +TEST_F(DocumentUpdateTest, testIncrementNonExistingAutoCreateWSetField) { WeightedSetAutoCreateFixture fixture; @@ -440,7 +451,7 @@ TEST(DocumentUpdateTest, testIncrementNonExistingAutoCreateWSetField) EXPECT_EQ(1, ws->get(StringFieldValue("foo"), 0)); } -TEST(DocumentUpdateTest, testIncrementExistingWSetField) +TEST_F(DocumentUpdateTest, testIncrementExistingWSetField) { WeightedSetAutoCreateFixture fixture; { @@ -456,7 +467,7 @@ TEST(DocumentUpdateTest, testIncrementExistingWSetField) EXPECT_EQ(1, ws->get(StringFieldValue("foo"), 0)); } -TEST(DocumentUpdateTest, testIncrementWithZeroResultWeightIsRemoved) +TEST_F(DocumentUpdateTest, testIncrementWithZeroResultWeightIsRemoved) { WeightedSetAutoCreateFixture fixture; fixture.update.addUpdate(FieldUpdate(fixture.field) @@ -471,13 +482,13 @@ TEST(DocumentUpdateTest, testIncrementWithZeroResultWeightIsRemoved) EXPECT_FALSE(ws->contains(StringFieldValue("baz"))); } -TEST(DocumentUpdateTest, testReadSerializedFile) +TEST_F(DocumentUpdateTest, testReadSerializedFile) { // Reads a file serialized from java - const std::string file_name = "data/crossplatform-java-cpp-doctypes.cfg"; + const std::string file_name = source_testdata() + "/crossplatform-java-cpp-doctypes.cfg"; DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); - auto is = readBufferFromFile("data/serializeupdatejava.dat"); + auto is = read_buffer_from_file(source_testdata() + "/serializeupdatejava.dat"); DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, is)); DocumentUpdate& upd(*updp); @@ -533,11 +544,11 @@ TEST(DocumentUpdateTest, testReadSerializedFile) } -TEST(DocumentUpdateTest, testGenerateSerializedFile) +TEST_F(DocumentUpdateTest, testGenerateSerializedFile) { // Tests nothing, only generates a file for java test - const std::string file_name = "data/crossplatform-java-cpp-doctypes.cfg"; - DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); + const std::string cfg_file_name = source_testdata() + "/crossplatform-java-cpp-doctypes.cfg"; + DocumentTypeRepo repo(readDocumenttypesConfig(cfg_file_name)); const DocumentType *type(repo.getDocumentType("serializetest")); DocumentUpdate upd(repo, *type, DocumentId("id:ns:serializetest::update")); @@ -557,11 +568,13 @@ TEST(DocumentUpdateTest, testGenerateSerializedFile) .addUpdate(std::make_unique<MapValueUpdate>(StringFieldValue::make("foo"), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Mul, 2)))); nbostream buf(serializeHEAD(upd)); - writeBufferToFile(buf, "data/serializeupdatecpp.dat"); + std::string file_name("serializeupdatecpp.dat"); + write_buffer_to_file(buf, build_testdata() + "/" + file_name); + ASSERT_NO_FATAL_FAILURE(remove_unchanged_build_testdata_file_or_fail(buf, file_name)); } -TEST(DocumentUpdateTest, testSetBadFieldTypes) +TEST_F(DocumentUpdateTest, testSetBadFieldTypes) { // Create a test document TestDocMan docMan; @@ -582,7 +595,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) doc->getValue(doc->getField("headerval")).get()); } -TEST(DocumentUpdateTest, testUpdateApplyNoParams) +TEST_F(DocumentUpdateTest, testUpdateApplyNoParams) { TestDocMan docMan; Document::UP doc(docMan.createDocument()); @@ -597,7 +610,7 @@ TEST(DocumentUpdateTest, testUpdateApplyNoParams) EXPECT_FALSE(doc->hasValue(doc->getField("tags"))); } -TEST(DocumentUpdateTest, testUpdateApplyNoArrayValues) +TEST_F(DocumentUpdateTest, testUpdateApplyNoArrayValues) { TestDocMan docMan; Document::UP doc(docMan.createDocument()); @@ -617,7 +630,7 @@ TEST(DocumentUpdateTest, testUpdateApplyNoArrayValues) EXPECT_EQ((size_t) 0, fval->size()); } -TEST(DocumentUpdateTest, testUpdateArrayEmptyParamValue) +TEST_F(DocumentUpdateTest, testUpdateArrayEmptyParamValue) { // Create a test document. TestDocMan docMan; @@ -645,7 +658,7 @@ TEST(DocumentUpdateTest, testUpdateArrayEmptyParamValue) EXPECT_FALSE(fval2); } -TEST(DocumentUpdateTest, testUpdateWeightedSetEmptyParamValue) +TEST_F(DocumentUpdateTest, testUpdateWeightedSetEmptyParamValue) { // Create a test document TestDocMan docMan; @@ -673,7 +686,7 @@ TEST(DocumentUpdateTest, testUpdateWeightedSetEmptyParamValue) EXPECT_FALSE(fval2); } -TEST(DocumentUpdateTest, testUpdateArrayWrongSubtype) +TEST_F(DocumentUpdateTest, testUpdateArrayWrongSubtype) { // Create a test document TestDocMan docMan; @@ -697,7 +710,7 @@ TEST(DocumentUpdateTest, testUpdateArrayWrongSubtype) EXPECT_EQ((document::FieldValue*) 0, fval.get()); } -TEST(DocumentUpdateTest, testUpdateWeightedSetWrongSubtype) +TEST_F(DocumentUpdateTest, testUpdateWeightedSetWrongSubtype) { // Create a test document TestDocMan docMan; @@ -721,7 +734,7 @@ TEST(DocumentUpdateTest, testUpdateWeightedSetWrongSubtype) EXPECT_EQ((document::FieldValue*) 0, fval.get()); } -TEST(DocumentUpdateTest, testMapValueUpdate) +TEST_F(DocumentUpdateTest, testMapValueUpdate) { // Create a test document TestDocMan docMan; @@ -940,7 +953,7 @@ struct TensorUpdateFixture { }; -TEST(DocumentUpdateTest, tensor_assign_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_assign_update_can_be_applied) { TensorUpdateFixture f; f.applyUpdate(std::make_unique<AssignValueUpdate>(f.makeBaselineTensor())); @@ -948,7 +961,7 @@ TEST(DocumentUpdateTest, tensor_assign_update_can_be_applied) f.assertTensor(*f.makeBaselineTensor()); } -TEST(DocumentUpdateTest, tensor_clear_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_clear_update_can_be_applied) { TensorUpdateFixture f; f.setTensor(*f.makeBaselineTensor()); @@ -957,7 +970,7 @@ TEST(DocumentUpdateTest, tensor_clear_update_can_be_applied) EXPECT_FALSE(f.getTensor()); } -TEST(DocumentUpdateTest, tensor_add_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_add_update_can_be_applied) { TensorUpdateFixture f; f.assertApplyUpdate(f.spec().add({{"x", "a"}}, 2) @@ -971,7 +984,7 @@ TEST(DocumentUpdateTest, tensor_add_update_can_be_applied) .add({{"x", "c"}}, 7)); } -TEST(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor) +TEST_F(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor) { TensorUpdateFixture f; f.assertApplyUpdateNonExisting(std::make_unique<TensorAddUpdate>(f.makeTensor(f.spec().add({{"x", "b"}}, 5) @@ -981,7 +994,7 @@ TEST(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor) .add({{"x", "c"}}, 7)); } -TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_remove_update_can_be_applied) { TensorUpdateFixture f; f.assertApplyUpdate(f.spec().add({{"x", "a"}}, 2) @@ -992,13 +1005,13 @@ TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied) f.spec().add({{"x", "a"}}, 2)); } -TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied_to_nonexisting_tensor) +TEST_F(DocumentUpdateTest, tensor_remove_update_can_be_applied_to_nonexisting_tensor) { TensorUpdateFixture f; f.assertApplyUpdateNonExisting(std::make_unique<TensorRemoveUpdate>(f.makeTensor(f.spec().add({{"x", "b"}}, 1)))); } -TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_modify_update_can_be_applied) { TensorUpdateFixture f; auto baseLine = f.spec().add({{"x", "a"}}, 2) @@ -1024,7 +1037,7 @@ TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied) .add({{"x", "b"}}, 15)); } -TEST(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_can_be_applied) { TensorUpdateFixture f; auto baseLine = f.spec().add({{"x", "a"}}, 2) @@ -1038,14 +1051,14 @@ TEST(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_can .add({{"x", "c"}}, 6)); } -TEST(DocumentUpdateTest, tensor_modify_update_is_ignored_when_applied_to_nonexisting_tensor) +TEST_F(DocumentUpdateTest, tensor_modify_update_is_ignored_when_applied_to_nonexisting_tensor) { TensorUpdateFixture f; f.assertApplyUpdateNonExisting(std::make_unique<TensorModifyUpdate>(TensorModifyUpdate::Operation::ADD, f.makeTensor(f.spec().add({{"x", "b"}}, 5)))); } -TEST(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_is_applied_to_nonexisting_tensor) +TEST_F(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_is_applied_to_nonexisting_tensor) { TensorUpdateFixture f; f.assertApplyUpdateNonExisting(std::make_unique<TensorModifyUpdate>(TensorModifyUpdate::Operation::ADD, @@ -1055,25 +1068,25 @@ TEST(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_is_ .add({{"x", "c"}}, 6)); } -TEST(DocumentUpdateTest, tensor_assign_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_assign_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; f.assertRoundtripSerialize(AssignValueUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_add_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_add_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; f.assertRoundtripSerialize(TensorAddUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_remove_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_remove_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; f.assertRoundtripSerialize(TensorRemoveUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_remove_update_with_not_fully_specified_address_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_remove_update_with_not_fully_specified_address_can_be_roundtrip_serialized) { TensorUpdateFixture f("sparse_xy_tensor"); TensorDataType type(ValueType::from_spec("tensor(y{})")); @@ -1081,13 +1094,13 @@ TEST(DocumentUpdateTest, tensor_remove_update_with_not_fully_specified_address_c makeTensorFieldValue(TensorSpec("tensor(y{})").add({{"y", "a"}}, 1), type))); } -TEST(DocumentUpdateTest, tensor_remove_update_on_float_tensor_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_remove_update_on_float_tensor_can_be_roundtrip_serialized) { TensorUpdateFixture f("sparse_float_tensor"); f.assertRoundtripSerialize(TensorRemoveUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_modify_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_modify_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, f.makeBaselineTensor())); @@ -1098,7 +1111,7 @@ TEST(DocumentUpdateTest, tensor_modify_update_can_be_roundtrip_serialized) f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::MULTIPLY, f.makeBaselineTensor(), 1.0)); } -TEST(DocumentUpdateTest, tensor_modify_update_on_float_tensor_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_modify_update_on_float_tensor_can_be_roundtrip_serialized) { TensorUpdateFixture f("sparse_float_tensor"); f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, f.makeBaselineTensor())); @@ -1109,7 +1122,7 @@ TEST(DocumentUpdateTest, tensor_modify_update_on_float_tensor_can_be_roundtrip_s f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::MULTIPLY, f.makeBaselineTensor(), 1.0)); } -TEST(DocumentUpdateTest, tensor_modify_update_on_dense_tensor_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_modify_update_on_dense_tensor_can_be_roundtrip_serialized) { TensorUpdateFixture f("dense_tensor"); vespalib::string sparseType("tensor(x{})"); @@ -1118,25 +1131,25 @@ TEST(DocumentUpdateTest, tensor_modify_update_on_dense_tensor_can_be_roundtrip_s f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, std::move(sparseTensor))); } -TEST(DocumentUpdateTest, tensor_add_update_throws_on_non_tensor_field) +TEST_F(DocumentUpdateTest, tensor_add_update_throws_on_non_tensor_field) { TensorUpdateFixture f; f.assertThrowOnNonTensorField(TensorAddUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_remove_update_throws_on_non_tensor_field) +TEST_F(DocumentUpdateTest, tensor_remove_update_throws_on_non_tensor_field) { TensorUpdateFixture f; f.assertThrowOnNonTensorField(TensorRemoveUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_modify_update_throws_on_non_tensor_field) +TEST_F(DocumentUpdateTest, tensor_modify_update_throws_on_non_tensor_field) { TensorUpdateFixture f; f.assertThrowOnNonTensorField(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_remove_update_throws_if_address_tensor_is_not_sparse) +TEST_F(DocumentUpdateTest, tensor_remove_update_throws_if_address_tensor_is_not_sparse) { TensorUpdateFixture f("dense_tensor"); auto addressTensor = f.makeTensor(f.spec().add({{"x", 0}}, 2)); // creates a dense address tensor @@ -1145,7 +1158,7 @@ TEST(DocumentUpdateTest, tensor_remove_update_throws_if_address_tensor_is_not_sp vespalib::IllegalStateException); } -TEST(DocumentUpdateTest, tensor_modify_update_throws_if_cells_tensor_is_not_sparse) +TEST_F(DocumentUpdateTest, tensor_modify_update_throws_if_cells_tensor_is_not_sparse) { TensorUpdateFixture f("dense_tensor"); auto cellsTensor = f.makeTensor(f.spec().add({{"x", 0}}, 2)); // creates a dense cells tensor @@ -1209,31 +1222,34 @@ struct TensorUpdateSerializeFixture { void serializeUpdateToFile(const DocumentUpdate &update, const vespalib::string &fileName) { nbostream buf = serializeHEAD(update); - writeBufferToFile(buf, fileName); + TestDataBase::write_buffer_to_file(buf, fileName); } DocumentUpdate::UP deserializeUpdateFromFile(const vespalib::string &fileName) { - auto stream = readBufferFromFile(fileName); + auto stream = TestDataBase::read_buffer_from_file(fileName); return DocumentUpdate::createHEAD(*repo, stream); } }; -TEST(DocumentUpdateTest, tensor_update_file_java_can_be_deserialized) +TEST_F(DocumentUpdateTest, tensor_update_file_java_can_be_deserialized) { TensorUpdateSerializeFixture f; - auto update = f.deserializeUpdateFromFile("data/serialize-tensor-update-java.dat"); + auto update = f.deserializeUpdateFromFile(source_testdata() + "/serialize-tensor-update-java.dat"); EXPECT_EQ(*f.makeUpdate(), *update); } -TEST(DocumentUpdateTest, generate_serialized_tensor_update_file_cpp) +TEST_F(DocumentUpdateTest, generate_serialized_tensor_update_file_cpp) { TensorUpdateSerializeFixture f; auto update = f.makeUpdate(); - f.serializeUpdateToFile(*update, "data/serialize-tensor-update-cpp.dat"); + std::string file_name("serialize-tensor-update-cpp.dat"); + auto act_path = build_testdata() + "/" + file_name; + f.serializeUpdateToFile(*update, act_path); + auto buf = read_buffer_from_file(act_path); + ASSERT_NO_FATAL_FAILURE(remove_unchanged_build_testdata_file_or_fail(buf, file_name)); } - void assertDocumentUpdateFlag(bool createIfNonExistent, int value) { @@ -1249,7 +1265,7 @@ assertDocumentUpdateFlag(bool createIfNonExistent, int value) EXPECT_EQ(value, extractedValue); } -TEST(DocumentUpdateTest, testThatDocumentUpdateFlagsIsWorking) +TEST_F(DocumentUpdateTest, testThatDocumentUpdateFlagsIsWorking) { { // create-if-non-existent = true assertDocumentUpdateFlag(true, 0); @@ -1289,7 +1305,7 @@ CreateIfNonExistentFixture::CreateIfNonExistentFixture() update->setCreateIfNonExistent(true); } -TEST(DocumentUpdateTest, testThatCreateIfNonExistentFlagIsSerializedAndDeserialized) +TEST_F(DocumentUpdateTest, testThatCreateIfNonExistentFlagIsSerializedAndDeserialized) { CreateIfNonExistentFixture f; @@ -1322,7 +1338,7 @@ ArrayUpdateFixture::ArrayUpdateFixture() } ArrayUpdateFixture::~ArrayUpdateFixture() = default; -TEST(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) { ArrayUpdateFixture f; @@ -1332,7 +1348,7 @@ TEST(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) EXPECT_EQ(*f.update, *deserialized); } -TEST(DocumentUpdateTest, array_element_update_applies_to_specified_element) +TEST_F(DocumentUpdateTest, array_element_update_applies_to_specified_element) { ArrayUpdateFixture f; @@ -1351,7 +1367,7 @@ TEST(DocumentUpdateTest, array_element_update_applies_to_specified_element) EXPECT_EQ(vespalib::string("blarg"), (*result_array)[2].getAsString()); } -TEST(DocumentUpdateTest, array_element_update_for_invalid_index_is_ignored) +TEST_F(DocumentUpdateTest, array_element_update_for_invalid_index_is_ignored) { ArrayUpdateFixture f; @@ -1414,7 +1430,7 @@ struct UpdateToEmptyDocumentFixture { } }; -TEST(DocumentUpdateTest, string_field_annotations_can_be_deserialized_after_assign_update_to_empty_document) +TEST_F(DocumentUpdateTest, string_field_annotations_can_be_deserialized_after_assign_update_to_empty_document) { UpdateToEmptyDocumentFixture f; auto doc = f.make_empty_doc(); diff --git a/document/src/tests/fieldvalue/document_test.cpp b/document/src/tests/fieldvalue/document_test.cpp index ef338c3e97f..0f2d6b30462 100644 --- a/document/src/tests/fieldvalue/document_test.cpp +++ b/document/src/tests/fieldvalue/document_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/base/documentid.h> #include <vespa/document/base/testdocrepo.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> using namespace document; diff --git a/document/src/tests/fieldvalue/fieldvalue_test.cpp b/document/src/tests/fieldvalue/fieldvalue_test.cpp index e78509143ad..6cb5fb2dcb8 100644 --- a/document/src/tests/fieldvalue/fieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/fieldvalue_test.cpp @@ -5,7 +5,7 @@ #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/document/fieldvalue/intfieldvalue.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("fieldvalue_test"); diff --git a/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp b/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp index 93505d71c96..67d5b163463 100644 --- a/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicatefieldvalue. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/predicatefieldvalue.h> diff --git a/document/src/tests/fieldvalue/referencefieldvalue_test.cpp b/document/src/tests/fieldvalue/referencefieldvalue_test.cpp index 8dc5778055a..57b313b2237 100644 --- a/document/src/tests/fieldvalue/referencefieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/referencefieldvalue_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/datatype/referencedatatype.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <ostream> diff --git a/document/src/tests/predicate/predicate_builder_test.cpp b/document/src/tests/predicate/predicate_builder_test.cpp index 81809846a9a..ea5544d5d98 100644 --- a/document/src/tests/predicate/predicate_builder_test.cpp +++ b/document/src/tests/predicate/predicate_builder_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("predicate_builder_test"); #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_builder.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using std::string; using vespalib::Slime; diff --git a/document/src/tests/predicate/predicate_printer_test.cpp b/document/src/tests/predicate/predicate_printer_test.cpp index 49a0a5abc81..06c9b025ee2 100644 --- a/document/src/tests/predicate/predicate_printer_test.cpp +++ b/document/src/tests/predicate/predicate_printer_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("predicate_printer_test"); #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_printer.h> #include <vespa/document/predicate/predicate_slime_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using vespalib::Slime; using vespalib::slime::Cursor; diff --git a/document/src/tests/predicate/predicate_test.cpp b/document/src/tests/predicate/predicate_test.cpp index 9835bbc765b..327d05743a0 100644 --- a/document/src/tests/predicate/predicate_test.cpp +++ b/document/src/tests/predicate/predicate_test.cpp @@ -3,7 +3,7 @@ #include <vespa/document/predicate/predicate.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/predicate/predicate_slime_builder.h> #include <string> diff --git a/document/src/tests/repo/doctype_config_test.cpp b/document/src/tests/repo/doctype_config_test.cpp index 044293d90c6..0705b53c2df 100644 --- a/document/src/tests/repo/doctype_config_test.cpp +++ b/document/src/tests/repo/doctype_config_test.cpp @@ -14,7 +14,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <set> diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index e8017ae166a..b2d79209c12 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -14,7 +14,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <set> diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index ddc633e42f3..65a77b1f872 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -43,7 +43,7 @@ #include <vespa/eval/eval/test/value_compare.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/exceptions.h> #include <vespa/vespalib/util/compressionconfig.h> #include <filesystem> diff --git a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp index 13f2a58e880..4cc4e5e6215 100644 --- a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp +++ b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp @@ -11,7 +11,7 @@ LOG_SETUP("fieldvalue_test"); #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using vespalib::eval::SimpleValue; diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index cd637e50093..a3b7b463c3f 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -9,7 +9,7 @@ vespa_define_module( vespa_slobrok vespa_messagebus vespa_configdefinitions - vdslib + vespa_vdslib LIBS src/vespa/documentapi diff --git a/fnet/src/tests/locking/lockspeed.cpp b/fnet/src/tests/locking/lockspeed.cpp index 78fe0869e22..eaf79ff38d4 100644 --- a/fnet/src/tests/locking/lockspeed.cpp +++ b/fnet/src/tests/locking/lockspeed.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include "dummy.h" #include <chrono> +#include <condition_variable> class SpinLock { private: @@ -202,16 +203,16 @@ TEST("lock speed") { start = clock::now(); for (i = 0; i < 1000000; i++) { - DummyObj *dummy0 = new DummyObj(); - DummyObj *dummy1 = new DummyObj(); - DummyObj *dummy2 = new DummyObj(); - DummyObj *dummy3 = new DummyObj(); - DummyObj *dummy4 = new DummyObj(); - DummyObj *dummy5 = new DummyObj(); - DummyObj *dummy6 = new DummyObj(); - DummyObj *dummy7 = new DummyObj(); - DummyObj *dummy8 = new DummyObj(); - DummyObj *dummy9 = new DummyObj(); + auto *dummy0 = new DummyObj(); + auto *dummy1 = new DummyObj(); + auto *dummy2 = new DummyObj(); + auto *dummy3 = new DummyObj(); + auto *dummy4 = new DummyObj(); + auto *dummy5 = new DummyObj(); + auto *dummy6 = new DummyObj(); + auto *dummy7 = new DummyObj(); + auto *dummy8 = new DummyObj(); + auto *dummy9 = new DummyObj(); delete dummy9; delete dummy8; delete dummy7; diff --git a/fnet/src/tests/scheduling/schedule.cpp b/fnet/src/tests/scheduling/schedule.cpp index 88b84bec67a..2890cc1d5f0 100644 --- a/fnet/src/tests/scheduling/schedule.cpp +++ b/fnet/src/tests/scheduling/schedule.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/fnet/scheduler.h> #include <vespa/fnet/task.h> +#include <cassert> using vespalib::steady_clock; using vespalib::steady_time; diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java index b54e71993c9..f03354f6938 100644 --- a/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java +++ b/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java @@ -34,28 +34,35 @@ import static org.junit.jupiter.api.Assertions.assertTrue; */ class SignaturesTest { - private static final String ecPemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + - "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + - "-----END PUBLIC KEY-----\n"; - - private static final String ecPemPrivateKey = "-----BEGIN EC PRIVATE KEY-----\n" + - "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" + - "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" + - "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + - "-----END EC PRIVATE KEY-----\n"; - - private static final String otherEcPemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + - "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + - "-----END PUBLIC KEY-----\n"; - - private static final byte[] message = ("Hello,\n" + - "\n" + - "this is a secret message.\n" + - "\n" + - "Yours truly,\n" + - "∠( ᐛ 」∠)＿").getBytes(UTF_8); + private static final String ecPemPublicKey = """ + -----BEGIN PUBLIC KEY----- + MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9 + z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw== + -----END PUBLIC KEY----- + """; + + private static final String ecPemPrivateKey = """ + -----BEGIN EC PRIVATE KEY----- + MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49 + AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU + PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw== + -----END EC PRIVATE KEY----- + """; + + private static final String otherEcPemPublicKey = """ + -----BEGIN PUBLIC KEY----- + MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE + pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw== + -----END PUBLIC KEY----- + """; + + private static final byte[] message = (""" + Hello, + + this is a secret message. + + Yours truly, + ∠( ᐛ 」∠)＿""").getBytes(UTF_8); @Test void testHashing() throws Exception { diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index e83d5dd4ce9..60fa96d1fd6 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -54,6 +54,11 @@ <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> <build> <plugins> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java index 05ac73618e8..c2d41edf1b6 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java @@ -187,8 +187,8 @@ public class EmbedExpression extends Expression { if (targetType.rank() == 2 && targetType.mappedSubtype().rank() == 2) { if (embedderArguments.size() != 1) throw new VerificationException(this, "When the embedding target field is a 2d mapped tensor " + - "the name of the tensor dimension that corresponds to the input array elements must " + - "be given as a second argument to embed, e.g: ... | embed splade paragraph | ..."); + "the name of the tensor dimension that corresponds to the input array elements must " + + "be given as a second argument to embed, e.g: ... | embed splade paragraph | ..."); if ( ! targetType.mappedSubtype().dimensionNames().contains(embedderArguments.get(0))) { throw new VerificationException(this, "The dimension '" + embedderArguments.get(0) + "' given to embed " + "is not a sparse dimension of the target type " + targetType); @@ -254,7 +254,7 @@ public class EmbedExpression extends Expression { List<String> embedderIds = new ArrayList<>(); embedders.forEach((key, value) -> embedderIds.add(key)); embedderIds.sort(null); - return String.join(",", embedderIds); + return String.join(", ", embedderIds); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java index f6995ac5a72..dd0ec255c35 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -202,9 +202,9 @@ public class ScriptTestCase { "my input", "[110.0, 122.0, 33.0, 106.0]"); assertThrows(() -> testEmbedStatement("input myText | embed | attribute 'myTensor'", embedders, "input text", "[105, 110, 112, 117]"), - "Multiple embedders are provided but no embedder id is given. Valid embedders are emb1,emb2"); + "Multiple embedders are provided but no embedder id is given. Valid embedders are emb1, emb2"); assertThrows(() -> testEmbedStatement("input myText | embed emb3 | attribute 'myTensor'", embedders, "input text", "[105, 110, 112, 117]"), - "Can't find embedder 'emb3'. Valid embedders are emb1,emb2"); + "Can't find embedder 'emb3'. Valid embedders are emb1, emb2"); } private void testEmbedStatement(String expressionString, Map<String, Embedder> embedders, String input, String expected) { @@ -562,12 +562,12 @@ public class ScriptTestCase { } - private void assertThrows(Runnable r, String msg) { + private void assertThrows(Runnable r, String expectedMessage) { try { r.run(); fail(); } catch (IllegalStateException e) { - assertEquals(e.getMessage(), msg); + assertEquals(expectedMessage, e.getMessage()); } } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java index 6a106c6aa3f..515297ad20c 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java @@ -2,6 +2,7 @@ package com.yahoo.jdisc.http.filter.security.cloud; import com.yahoo.component.annotation.Inject; +import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; @@ -31,6 +32,7 @@ import static com.yahoo.jdisc.http.filter.security.cloud.Permission.WRITE; * * @author bjorncs */ +@Provides("Vespa Cloud mTLS Authorization Filter") public class CloudDataPlaneFilter extends JsonSecurityRequestFilterBase { private static final Logger log = Logger.getLogger(CloudDataPlaneFilter.class.getName()); diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java index 699aa5c9187..d8263301049 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java @@ -2,6 +2,7 @@ package com.yahoo.jdisc.http.filter.security.cloud; import com.yahoo.component.annotation.Inject; +import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; @@ -32,6 +33,7 @@ import static com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler.CONT * * @author bjorncs */ +@Provides("Vespa Cloud Token Authorization Filter") public class CloudTokenDataPlaneFilter extends JsonSecurityRequestFilterBase { private static final Logger log = Logger.getLogger(CloudTokenDataPlaneFilter.class.getName()); diff --git a/linguistics/pom.xml b/linguistics/pom.xml index d07ff5d9fdb..40767bfff26 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -14,15 +14,19 @@ <packaging>container-plugin</packaging> <version>8-SNAPSHOT</version> <dependencies> + <!-- Compile Scope Dependencies --> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> + <scope>compile</scope> </dependency> + + <!-- Provided Scope Dependencies --> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -38,19 +42,19 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>annotations</artifactId> + <artifactId>configdefinitions</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>configdefinitions</artifactId> - <version>${project.version}</version> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>com.google.inject</groupId> @@ -58,14 +62,22 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> <scope>provided</scope> </dependency> + + <!-- Test Scope Dependencies --> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <scope>provided</scope> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.junit.vintage</groupId> @@ -73,8 +85,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter</artifactId> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> </dependencies> diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java index 72874c15d9e..45c6054f748 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java @@ -7,14 +7,20 @@ import com.yahoo.language.Language; import com.yahoo.language.significance.SignificanceModel; import com.yahoo.language.significance.SignificanceModelRegistry; import com.yahoo.search.significance.config.SignificanceConfig; +import io.airlift.compress.zstd.ZstdInputStream; import java.io.IOException; import java.io.UncheckedIOException; +import java.io.FileInputStream; +import java.io.InputStream; import java.nio.file.Path; import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.ArrayList; +import java.util.Objects; +import java.util.logging.Logger; /** * Default implementation of {@link SignificanceModelRegistry}. @@ -24,6 +30,8 @@ import java.util.Optional; */ public class DefaultSignificanceModelRegistry implements SignificanceModelRegistry { + private static final Logger log = Logger.getLogger(DefaultSignificanceModelRegistry.class.getName()); + private final Map<Language, SignificanceModel> models; @Inject @@ -42,13 +50,25 @@ public class DefaultSignificanceModelRegistry implements SignificanceModelRegist } public void addModel(Path path) { + log.fine(() -> "Loading model from " + path); ObjectMapper objectMapper = new ObjectMapper(); try { - SignificanceModelFile file = objectMapper.readValue(path.toFile(), SignificanceModelFile.class); + InputStream in = path.toString().endsWith(".zst") ? + new ZstdInputStream(new FileInputStream(path.toFile())) : + new FileInputStream(path.toFile()); + + SignificanceModelFile file = objectMapper.readValue(in, SignificanceModelFile.class); for (var pair : file.languages().entrySet()) { - this.models.put( - Language.fromLanguageTag(pair.getKey()), - new DefaultSignificanceModel(pair.getValue(), file.id())); + + var languagesStr = pair.getKey(); + log.fine(() -> "Found model for languages '%s'".formatted(languagesStr)); + String[] languageTags = languagesStr.split(","); + + for (var languageTag : languageTags) { + var language = Language.fromLanguageTag(languageTag); + log.fine(() -> "Adding model for language %s with id %s".formatted(language, file.id())); + this.models.put(language, new DefaultSignificanceModel(pair.getValue(), file.id())); + } } } catch (IOException e) { throw new UncheckedIOException("Failed to load model from " + path, e); diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java index 94030108671..fd9b206b334 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java @@ -3,11 +3,9 @@ package com.yahoo.language.significance.impl; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.HashMap; -import java.util.List; /** * diff --git a/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java b/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java index e8594885b9e..ead0f2c15c3 100644 --- a/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java +++ b/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java @@ -35,7 +35,7 @@ public class DefaultSignificanceModelRegistryTest { var englishModel = optionalEnglishModel.get(); var norwegianModel = optionalNorwegianModel.get(); - assertTrue( defaultSignificanceModelRegistry.getModel(Language.FRENCH).isEmpty()); + assertTrue( defaultSignificanceModelRegistry.getModel(Language.GERMAN).isEmpty()); assertNotNull(englishModel); assertNotNull(norwegianModel); @@ -55,6 +55,58 @@ public class DefaultSignificanceModelRegistryTest { } @Test + public void testDefaultSignificanceModelRegistryWithUnknownLanguage() { + List<Path> models = new ArrayList<>(); + + models.add(Path.of("src/test/models/docv2.json")); + + DefaultSignificanceModelRegistry defaultSignificanceModelRegistry = new DefaultSignificanceModelRegistry(models); + + assertTrue(defaultSignificanceModelRegistry.getModel(Language.ENGLISH).isPresent()); + assertTrue(defaultSignificanceModelRegistry.getModel(Language.FRENCH).isPresent()); + assertTrue(defaultSignificanceModelRegistry.getModel(Language.UNKNOWN).isPresent()); + + var frenchModel = defaultSignificanceModelRegistry.getModel(Language.FRENCH).get(); + var unknownModel = defaultSignificanceModelRegistry.getModel(Language.UNKNOWN).get(); + + assertEquals("test::2", frenchModel.getId()); + assertEquals("test::2", unknownModel.getId()); + + assertEquals(1, frenchModel.documentFrequency("non-existent-word").frequency()); + assertEquals(20, frenchModel.documentFrequency("non-existent-word").corpusSize()); + + assertEquals(1, unknownModel.documentFrequency("non-existent-word").frequency()); + assertEquals(20, unknownModel.documentFrequency("non-existent-word").corpusSize()); + + assertEquals(6, frenchModel.documentFrequency("bonjour").frequency()); + assertEquals(20, frenchModel.documentFrequency("bonjour").corpusSize()); + assertEquals(6, unknownModel.documentFrequency("bonjour").frequency()); + assertEquals(20, unknownModel.documentFrequency("bonjour").corpusSize()); + + } + + @Test + public void testDefaultSignificanceModelRegistryWithZSTDecompressing() { + List<Path> models = new ArrayList<>(); + + models.add(Path.of("src/test/models/docv1.json.zst")); + + DefaultSignificanceModelRegistry defaultSignificanceModelRegistry = new DefaultSignificanceModelRegistry(models); + + var optionalEnglishModel = defaultSignificanceModelRegistry.getModel(Language.ENGLISH); + assertTrue(optionalEnglishModel.isPresent()); + + var englishModel = optionalEnglishModel.get(); + + assertTrue( defaultSignificanceModelRegistry.getModel(Language.FRENCH).isEmpty()); + assertNotNull(englishModel); + assertEquals("test::1", englishModel.getId()); + assertEquals(2, englishModel.documentFrequency("test").frequency()); + assertEquals(10, englishModel.documentFrequency("test").corpusSize()); + + } + + @Test public void testDefaultSignificanceModelRegistryInOppsiteOrder() { List<Path> models = new ArrayList<>(); diff --git a/linguistics/src/test/models/docv1.json.zst b/linguistics/src/test/models/docv1.json.zst Binary files differnew file mode 100644 index 00000000000..61de6c7cb72 --- /dev/null +++ b/linguistics/src/test/models/docv1.json.zst diff --git a/linguistics/src/test/models/docv2.json b/linguistics/src/test/models/docv2.json index c00d02fb744..1f9d3779f26 100644 --- a/linguistics/src/test/models/docv2.json +++ b/linguistics/src/test/models/docv2.json @@ -26,6 +26,19 @@ "ja": 12, "nei": 3 } + }, + "fr,un": { + "description" : "french / unknown model", + "document-count" : 20, + "document-frequencies" : { + "usa" : 2, + "bonjour": 6, + "monde": 5, + "test": 2, + "france": 11, + "oui": 12, + "no": 3 + } } } } diff --git a/messagebus/src/tests/bucketsequence/bucketsequence.cpp b/messagebus/src/tests/bucketsequence/bucketsequence.cpp index 7a58fe3d861..832f22ece54 100644 --- a/messagebus/src/tests/bucketsequence/bucketsequence.cpp +++ b/messagebus/src/tests/bucketsequence/bucketsequence.cpp @@ -7,7 +7,7 @@ #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/error/error.cpp b/messagebus/src/tests/error/error.cpp index bfc4d4fc988..e7be09c3fcb 100644 --- a/messagebus/src/tests/error/error.cpp +++ b/messagebus/src/tests/error/error.cpp @@ -11,7 +11,7 @@ #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/messenger/messenger.cpp b/messagebus/src/tests/messenger/messenger.cpp index 92fd189aa51..ed080833882 100644 --- a/messagebus/src/tests/messenger/messenger.cpp +++ b/messagebus/src/tests/messenger/messenger.cpp @@ -2,7 +2,7 @@ #include <vespa/messagebus/messenger.h> #include <vespa/vespalib/util/barrier.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/replygate/replygate.cpp b/messagebus/src/tests/replygate/replygate.cpp index 104c2f82518..e1b1a5ed1dd 100644 --- a/messagebus/src/tests/replygate/replygate.cpp +++ b/messagebus/src/tests/replygate/replygate.cpp @@ -5,7 +5,7 @@ #include <vespa/messagebus/replygate.h> #include <vespa/messagebus/routablequeue.h> #include <vespa/messagebus/testlib/simplemessage.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/sendadapter/sendadapter.cpp b/messagebus/src/tests/sendadapter/sendadapter.cpp index 48618ab3061..633cc6de314 100644 --- a/messagebus/src/tests/sendadapter/sendadapter.cpp +++ b/messagebus/src/tests/sendadapter/sendadapter.cpp @@ -6,7 +6,7 @@ #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/messagebus/network/rpcsendv2.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("sendadapter_test"); diff --git a/messagebus/src/tests/serviceaddress/serviceaddress.cpp b/messagebus/src/tests/serviceaddress/serviceaddress.cpp index 376ee5fa38a..e2440cd917f 100644 --- a/messagebus/src/tests/serviceaddress/serviceaddress.cpp +++ b/messagebus/src/tests/serviceaddress/serviceaddress.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/messagebus/network/rpcservice.h> diff --git a/messagebus/src/tests/servicepool/servicepool.cpp b/messagebus/src/tests/servicepool/servicepool.cpp index 4a21f40e202..7439127e883 100644 --- a/messagebus/src/tests/servicepool/servicepool.cpp +++ b/messagebus/src/tests/servicepool/servicepool.cpp @@ -5,7 +5,7 @@ #include <vespa/messagebus/network/rpcservicepool.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/shutdown/shutdown.cpp b/messagebus/src/tests/shutdown/shutdown.cpp index 8dc4aa555b4..424bd86e203 100644 --- a/messagebus/src/tests/shutdown/shutdown.cpp +++ b/messagebus/src/tests/shutdown/shutdown.cpp @@ -7,7 +7,7 @@ #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> using namespace mbus; diff --git a/messagebus/src/tests/targetpool/targetpool.cpp b/messagebus/src/tests/targetpool/targetpool.cpp index dd86b279838..4265f9a946d 100644 --- a/messagebus/src/tests/targetpool/targetpool.cpp +++ b/messagebus/src/tests/targetpool/targetpool.cpp @@ -2,7 +2,7 @@ #include <vespa/messagebus/network/rpctargetpool.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("targetpool_test"); diff --git a/messagebus_test/src/tests/error/error.cpp b/messagebus_test/src/tests/error/error.cpp index 6e1c6b8ee70..d360849e8af 100644 --- a/messagebus_test/src/tests/error/error.cpp +++ b/messagebus_test/src/tests/error/error.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 64fd34c155b..e2931b70dec 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -239,7 +239,9 @@ public class MetricsReporter extends NodeRepositoryMaintainer { .orElse(0L); metric.add(ConfigServerMetrics.SUSPENDED_SECONDS.baseName(), suspendedSeconds, context); }); - if (nodeRepository().zone().environment().isTest() && node.state() == State.active) { + if (nodeRepository().zone().environment().isTest() && + node.state() == State.active && + node.type() == NodeType.tenant) { node.history() .event(History.Event.Type.activated) .ifPresent(event -> { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index 0ff8acdcf1f..7f13e3a5bca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -75,9 +76,10 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { MutableInteger failures, ApplicationId application) { if (exception != null) { - if (failures.get() < maxWarningsPerInvocation) + if (nodeRepository().zone().environment() == Environment.prod + && failures.get() < maxWarningsPerInvocation) log.log(Level.WARNING, "Could not update metrics for " + application + ": " + - Exceptions.toMessageString(exception)); + Exceptions.toMessageString(exception)); failures.add(1); } else if (response != null) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index bdd16bc1c11..ddaf5679cf3 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -131,7 +131,7 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { return SuspensionLimit.fromAllowedDown(1); } - if (Set.of(ServiceType.STORAGE, ServiceType.SEARCH, ServiceType.DISTRIBUTOR, ServiceType.TRANSACTION_LOG_SERVER) + if (Set.of(ServiceType.STORAGE, ServiceType.SEARCH, ServiceType.DISTRIBUTOR) .contains(clusterApi.serviceType())) { // Delegate to the cluster controller return SuspensionLimit.fromAllowedDownRatio(1); diff --git a/parent/pom.xml b/parent/pom.xml index effe9ace34b..8bded0b0430 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -724,6 +724,21 @@ </dependency> <dependency> <groupId>io.netty</groupId> + <artifactId>netty-buffer</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-codec</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-common</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> <artifactId>netty-handler</artifactId> <version>${netty.vespa.version}</version> </dependency> @@ -734,6 +749,16 @@ </dependency> <dependency> <groupId>io.netty</groupId> + <artifactId>netty-transport</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-transport-classes-epoll</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> <artifactId>netty-transport-native-epoll</artifactId> <version>${netty.vespa.version}</version> <classifier>linux-x86_64</classifier> @@ -745,6 +770,11 @@ </dependency> <dependency> <groupId>io.netty</groupId> + <artifactId>netty-transport-native-unix-common</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> <artifactId>netty-tcnative</artifactId> <version>${netty-tcnative.vespa.version}</version> </dependency> diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index ca1a3f73057..db34bc6d02b 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_define_module( vespalog vespalib vespa_document - vdslib + vespa_vdslib LIBS src/vespa/persistence diff --git a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp index fa0669a994a..835ac26d5c1 100644 --- a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp +++ b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp @@ -2,7 +2,7 @@ // Unit tests for dummypersistence. #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/documentid.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/vdslib/distribution/distribution.h> diff --git a/renovate.json b/renovate.json index b815ae4bf88..6e992909ba0 100644 --- a/renovate.json +++ b/renovate.json @@ -21,7 +21,6 @@ { "description": "Disable automatic PRs for artifacts, e.g. fixed version required like ZK dependencies or released to frequently. PRs can still be created manually from dependency dashboard.", "matchPackageNames": [ - "github.com/go-json-experiment/json", "javax.servlet:javax.servlet-api", "io.dropwizard.metrics:metrics-core", "org.apache.zookeeper:zookeeper" diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 1d4019a170e..6a5d3915d6a 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -68,11 +68,8 @@ vespa_define_module( src/tests/proton/bucketdb/bucketdb src/tests/proton/common src/tests/proton/common/alloc_config - src/tests/proton/common/attribute_updater - src/tests/proton/common/document_type_inspector src/tests/proton/common/hw_info_sampler src/tests/proton/common/operation_rate_tracker - src/tests/proton/common/state_reporter_utils src/tests/proton/common/timer src/tests/proton/docsummary src/tests/proton/document_iterator @@ -101,7 +98,6 @@ vespa_define_module( src/tests/proton/documentmetastore/lid_allocator src/tests/proton/documentmetastore/lid_state_vector src/tests/proton/feed_and_search - src/tests/proton/feedoperation src/tests/proton/feedtoken src/tests/proton/flushengine src/tests/proton/flushengine/prepare_restart_flush_strategy @@ -121,9 +117,6 @@ vespa_define_module( src/tests/proton/matching/request_context src/tests/proton/matching/same_element_builder src/tests/proton/matching/unpacking_iterators_optimizer - src/tests/proton/metrics/documentdb_job_trackers - src/tests/proton/metrics/job_load_sampler - src/tests/proton/metrics/job_tracked_flush src/tests/proton/metrics/metrics_engine src/tests/proton/persistenceconformance src/tests/proton/persistenceengine @@ -152,7 +145,6 @@ vespa_define_module( src/tests/proton/server/memory_flush_config_updater src/tests/proton/server/memoryflush src/tests/proton/server/shared_threading_service - src/tests/proton/statusreport src/tests/proton/summaryengine src/tests/proton/verify_ranksetup src/tests/index/disk_indexes diff --git a/searchcore/src/tests/grouping/grouping_test.cpp b/searchcore/src/tests/grouping/grouping_test.cpp index e77bc4e9e22..4bc4ee7755f 100644 --- a/searchcore/src/tests/grouping/grouping_test.cpp +++ b/searchcore/src/tests/grouping/grouping_test.cpp @@ -12,6 +12,7 @@ #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/searchlib/test/mock_attribute_context.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/util/testclock.h> #include <iostream> #include <vespa/vespalib/testkit/test_kit.h> @@ -36,57 +37,64 @@ const uint32_t NUM_DOCS = 1000; struct MyWorld { MockAttributeContext attributeContext; + document::DocumentType documentType; search::AllocatedBitVector bv; - MyWorld() - : attributeContext(), - bv(NUM_DOCS+1) + MyWorld(); + ~MyWorld(); +}; + +MyWorld::~MyWorld() = default; + +MyWorld::MyWorld() + : attributeContext(), + documentType("test"), + bv(NUM_DOCS+1) +{ + bv.setInterval(0, NUM_DOCS); + // attribute context { - bv.setInterval(0, NUM_DOCS); - // attribute context - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("attr0"); - AttributeVector::DocId docid; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i, docid); // value = docid - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); + auto attr = std::make_shared<SingleInt32ExtAttribute>("attr0"); + AttributeVector::DocId docid; + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i, docid); // value = docid } - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("attr1"); - AttributeVector::DocId docid; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i * 2, docid); // value = docid * 2 - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); + assert(docid + 1 == NUM_DOCS); + attributeContext.add(attr); + } + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("attr1"); + AttributeVector::DocId docid; + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i * 2, docid); // value = docid * 2 } - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("attr2"); - AttributeVector::DocId docid; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i * 3, docid); // value = docid * 3 - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); + assert(docid + 1 == NUM_DOCS); + attributeContext.add(attr); + } + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("attr2"); + AttributeVector::DocId docid; + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i * 3, docid); // value = docid * 3 } - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("attr3"); - AttributeVector::DocId docid; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i * 4, docid); // value = docid * 4 - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); + assert(docid + 1 == NUM_DOCS); + attributeContext.add(attr); + } + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("attr3"); + AttributeVector::DocId docid; + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i * 4, docid); // value = docid * 4 } - + assert(docid + 1 == NUM_DOCS); + attributeContext.add(attr); } -}; + +} //----------------------------------------------------------------------------- @@ -291,7 +299,7 @@ TEST_F("testGroupingSession", DoomFixture()) { SessionId id("foo"); // Test initialization phase - GroupingSession session(id, initContext, world.attributeContext); + GroupingSession session(id, initContext, world.attributeContext, &world.documentType); CheckAttributeReferences attrCheck2; EXPECT_EQUAL(2u, initContext.getGroupingList().size()); for (const auto & g : initContext.getGroupingList()) { @@ -356,7 +364,7 @@ TEST_F("testEmptySessionId", DoomFixture()) { SessionId id; // Test initialization phase - GroupingSession session(id, initContext, world.attributeContext); + GroupingSession session(id, initContext, world.attributeContext, &world.documentType); RankedHit hit; hit._docId = 0; GroupingManager &manager(session.getGroupingManager()); @@ -390,9 +398,9 @@ TEST_F("testSessionManager", DoomFixture()) { SessionId id1("foo"); SessionId id2("bar"); SessionId id3("baz"); - auto s1 = std::make_unique<GroupingSession>(id1, initContext, world.attributeContext); - auto s2 = std::make_unique<GroupingSession>(id2, initContext, world.attributeContext); - auto s3 = std::make_unique<GroupingSession>(id3, initContext, world.attributeContext); + auto s1 = std::make_unique<GroupingSession>(id1, initContext, world.attributeContext, &world.documentType); + auto s2 = std::make_unique<GroupingSession>(id2, initContext, world.attributeContext, &world.documentType); + auto s3 = std::make_unique<GroupingSession>(id3, initContext, world.attributeContext, &world.documentType); ASSERT_EQUAL(f1.timeOfDoom, s1->getTimeOfDoom()); mgr.insert(std::move(s1)); @@ -442,13 +450,12 @@ TEST_F("test grouping fork/join", DoomFixture()) { auto g1 = std::make_shared<Grouping>(request); GroupingContext context(world.bv, f1.clock.nowRef(), f1.timeOfDoom); context.addGrouping(g1); - GroupingSession session(SessionId(), context, world.attributeContext); + GroupingSession session(SessionId(), context, world.attributeContext, &world.documentType); session.prepareThreadContextCreation(4); - - GroupingContext::UP ctx0 = session.createThreadContext(0, world.attributeContext); - GroupingContext::UP ctx1 = session.createThreadContext(1, world.attributeContext); - GroupingContext::UP ctx2 = session.createThreadContext(2, world.attributeContext); - GroupingContext::UP ctx3 = session.createThreadContext(3, world.attributeContext); + GroupingContext::UP ctx0 = session.createThreadContext(0, world.attributeContext, &world.documentType); + GroupingContext::UP ctx1 = session.createThreadContext(1, world.attributeContext, &world.documentType); + GroupingContext::UP ctx2 = session.createThreadContext(2, world.attributeContext, &world.documentType); + GroupingContext::UP ctx3 = session.createThreadContext(3, world.attributeContext, &world.documentType); doGrouping(*ctx0, 12, 30.0, 11, 20.0, 10, 10.0); doGrouping(*ctx1, 22, 150.0, 21, 40.0, 20, 25.0); doGrouping(*ctx2, 32, 100.0, 31, 15.0, 30, 5.0); @@ -483,8 +490,8 @@ TEST_F("test session timeout", DoomFixture()) { GroupingContext initContext1(world.bv, f1.clock.nowRef(), steady_time(duration(10))); GroupingContext initContext2(world.bv, f1.clock.nowRef(), steady_time(duration(20))); - auto s1 = std::make_unique<GroupingSession>(id1, initContext1, world.attributeContext); - auto s2 = std::make_unique<GroupingSession>(id2, initContext2, world.attributeContext); + auto s1 = std::make_unique<GroupingSession>(id1, initContext1, world.attributeContext, &world.documentType); + auto s2 = std::make_unique<GroupingSession>(id2, initContext2, world.attributeContext, &world.documentType); mgr.insert(std::move(s1)); mgr.insert(std::move(s2)); mgr.pruneTimedOutSessions(steady_time(5ns)); diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp index e25c675ff64..af37aca4702 100644 --- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -13,7 +13,7 @@ #include <vespa/searchcommon/attribute/i_multi_value_attribute.h> #include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("attribute_initializer_test"); diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 634bc2c93b5..df27c4d7cc2 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -32,7 +32,7 @@ #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/hw_info.h> diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp index 2cdfd908bb1..0d011797395 100644 --- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp @@ -13,7 +13,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/hw_info.h> diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp index 15c26797f15..a20e08d5be2 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -3,7 +3,7 @@ #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchlib/attribute/address_space_components.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp index 6615a0e583a..6f84b793608 100644 --- a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp @@ -14,7 +14,7 @@ #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchcore/proton/attribute/document_field_extractor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::Field; using document::DataType; diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp index 08f9bfdb52d..6d522f254b0 100644 --- a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp @@ -2,7 +2,7 @@ #include <vespa/document/datatype/datatype.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> #include <vespa/searchlib/attribute/attributefactory.h> diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp index d97b2c2a3e7..6d2f375e410 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/attribute/imported_attributes_context.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp index d0b7ac8e688..0447db9c89e 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("imported_attributes_repo_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/basictype.h> #include <vespa/searchcommon/attribute/iattributevector.h> diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp index 4f6d09e6ffa..213d9c38290 100644 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp +++ b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp @@ -6,7 +6,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("bucketdb_test"); diff --git a/searchcore/src/tests/proton/common/.gitignore b/searchcore/src/tests/proton/common/.gitignore index 9ce51ef2178..e69de29bb2d 100644 --- a/searchcore/src/tests/proton/common/.gitignore +++ b/searchcore/src/tests/proton/common/.gitignore @@ -1,2 +0,0 @@ -searchcore_cachedselect_test_app -searchcore_selectpruner_test_app diff --git a/searchcore/src/tests/proton/common/CMakeLists.txt b/searchcore/src/tests/proton/common/CMakeLists.txt index 658afa38247..7eec733214b 100644 --- a/searchcore/src/tests/proton/common/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/CMakeLists.txt @@ -1,24 +1,23 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_selectpruner_test_app TEST +vespa_add_executable(searchcore_proton_common_vespa_test_app TEST SOURCES + vespa_testrunner.cpp selectpruner_test.cpp - DEPENDS - searchcore_pcommon - searchlib_test -) -vespa_add_test(NAME searchcore_selectpruner_test_app COMMAND searchcore_selectpruner_test_app) -vespa_add_executable(searchcore_cachedselect_test_app TEST - SOURCES cachedselect_test.cpp - DEPENDS - searchcore_pcommon - searchlib_test -) -vespa_add_test(NAME searchcore_cachedselect_test_app COMMAND searchcore_cachedselect_test_app) -vespa_add_executable(pendinglidtracker_test_app TEST - SOURCES pendinglidtracker_test.cpp + attribute_updater_test.cpp + state_reporter_utils_test.cpp + document_type_inspector_test.cpp + feedoperation_test.cpp + documentdb_job_trackers_test.cpp + job_load_sampler_test.cpp + job_tracked_flush_test.cpp + statusreport_test.cpp DEPENDS + searchcore_proton_metrics + searchcore_feedoperation searchcore_pcommon + searchcore_test + searchlib_test ) -vespa_add_test(NAME pendinglidtracker_test_app COMMAND pendinglidtracker_test_app) +vespa_add_test(NAME searchcore_proton_common_vespa_test_app COMMAND searchcore_proton_common_vespa_test_app) diff --git a/searchcore/src/tests/proton/common/attribute_updater/.gitignore b/searchcore/src/tests/proton/common/attribute_updater/.gitignore deleted file mode 100644 index 3c6e15d6808..00000000000 --- a/searchcore/src/tests/proton/common/attribute_updater/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -attribute_updater_test -searchcore_attribute_updater_test_app diff --git a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt deleted file mode 100644 index be0da1012d0..00000000000 --- a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_updater_test_app TEST - SOURCES - attribute_updater_test.cpp - DEPENDS - searchcore_pcommon - searchlib_test -) -vespa_add_test(NAME searchcore_attribute_updater_test_app COMMAND searchcore_attribute_updater_test_app) diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater_test.cpp index 0efeaf18c65..432386af0e6 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater_test.cpp @@ -38,10 +38,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("attribute_updater_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using document::config_builder::Array; @@ -472,6 +469,3 @@ TEST_F("require that tensor remove update is applied", } } - -TEST_MAIN() { TEST_RUN_ALL(); } - diff --git a/searchcore/src/tests/proton/common/cachedselect_test.cpp b/searchcore/src/tests/proton/common/cachedselect_test.cpp index a0c8fef3b83..70cec30392b 100644 --- a/searchcore/src/tests/proton/common/cachedselect_test.cpp +++ b/searchcore/src/tests/proton/common/cachedselect_test.cpp @@ -21,10 +21,10 @@ #include <vespa/searchlib/attribute/singlenumericenumattribute.hpp> #include <vespa/searchlib/attribute/singlenumericpostattribute.h> #include <vespa/searchlib/test/mock_attribute_manager.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> -LOG_SETUP("cachedselect_test"); +LOG_SETUP(".cachedselect_test"); using document::DataType; using document::Document; @@ -145,9 +145,7 @@ checkSelect(const NodeUP &sel, } std::ostringstream os; EXPECT_TRUE(sel->trace(ctx, os) == exp); - LOG(info, - "trace output: '%s'", - os.str().c_str()); + LOG(info, "trace output: '%s'", os.str().c_str()); return false; } @@ -679,7 +677,4 @@ TEST_F("Test performance when using attributes", TestFixture) } - } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/document_type_inspector/.gitignore b/searchcore/src/tests/proton/common/document_type_inspector/.gitignore deleted file mode 100644 index 49db4ae7746..00000000000 --- a/searchcore/src/tests/proton/common/document_type_inspector/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_document_type_inspector_test_app diff --git a/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt b/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt deleted file mode 100644 index 339574dc906..00000000000 --- a/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_type_inspector_test_app TEST - SOURCES - document_type_inspector_test.cpp - DEPENDS - searchcore_pcommon -) -vespa_add_test(NAME searchcore_document_type_inspector_test_app COMMAND searchcore_document_type_inspector_test_app) diff --git a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp b/searchcore/src/tests/proton/common/document_type_inspector_test.cpp index 83106747623..46022ed9273 100644 --- a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp +++ b/searchcore/src/tests/proton/common/document_type_inspector_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/common/document_type_inspector.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/field.h> #include <vespa/document/datatype/datatypes.h> @@ -127,8 +127,3 @@ TEST_F("require that struct addition is detected", Fixture(false, false)) EXPECT_FALSE(inspector.hasUnchangedField("map.key")); EXPECT_FALSE(inspector.hasUnchangedField("map.value")); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp b/searchcore/src/tests/proton/common/documentdb_job_trackers_test.cpp index c32b3439c5d..89c3b164e96 100644 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp +++ b/searchcore/src/tests/proton/common/documentdb_job_trackers_test.cpp @@ -3,12 +3,9 @@ #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> #include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h> #include <vespa/searchcore/proton/test/dummy_flush_target.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <thread> -#include <vespa/log/log.h> -LOG_SETUP("documentdb_job_trackers_test"); - using namespace proton; using namespace searchcorespi; @@ -115,5 +112,3 @@ TEST_F("require that un-known flush targets are not tracked", Fixture) EXPECT_EQUAL(1u, output.size()); EXPECT_EQUAL(&*output[0].get(), &*input[0]); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/common/feedoperation_test.cpp index b4cf29f67e0..48893aa7da3 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/common/feedoperation_test.cpp @@ -24,7 +24,7 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::BucketId; using document::DataType; @@ -357,5 +357,3 @@ TEST_F("require that we can serialize and deserialize remove by gid operations", } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp index d23505dae9c..ad74039a5ee 100644 --- a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp +++ b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using proton::HwInfoSampler; using search::test::DirectoryHandler; diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp b/searchcore/src/tests/proton/common/job_load_sampler_test.cpp index 2b74d1425a1..b6fcd3fe092 100644 --- a/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp +++ b/searchcore/src/tests/proton/common/job_load_sampler_test.cpp @@ -1,9 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("job_load_sampler_test"); #include <vespa/searchcore/proton/metrics/job_load_sampler.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <chrono> using namespace proton; @@ -101,5 +99,3 @@ TEST_F("require that multiple jobs that starts and ends in several intervals get f.end(45); EXPECT_APPROX(0.5, f.sample(50), EPS); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp b/searchcore/src/tests/proton/common/job_tracked_flush_test.cpp index fa6b158136f..608fbb60a70 100644 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp +++ b/searchcore/src/tests/proton/common/job_tracked_flush_test.cpp @@ -5,14 +5,11 @@ #include <vespa/searchcore/proton/test/dummy_flush_target.h> #include <vespa/searchcore/proton/test/simple_job_tracker.h> #include <vespa/searchlib/common/flush_token.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/gate.h> -#include <vespa/log/log.h> -LOG_SETUP("job_tracked_flush_test"); - using namespace proton; using namespace searchcorespi; using search::SerialNum; @@ -134,5 +131,3 @@ TEST_F("require that nullptr flush task is not tracked", Fixture) FlushTask::UP task = f._trackedFlush.initFlush(0, std::make_shared<search::FlushToken>()); EXPECT_TRUE(task.get() == nullptr); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp b/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp index 1aac149b7e7..f8d0d218670 100644 --- a/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp +++ b/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp @@ -1,11 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/common/pendinglidtracker.h> -#include <vespa/log/log.h> -LOG_SETUP("pendinglidtracker_test"); - using namespace proton; constexpr uint32_t LID_1 = 1u; @@ -76,5 +73,3 @@ TEST("test pendinglidtracker for needcommit") { EXPECT_EQUAL(ILidCommitState::State::COMPLETED, tracker.getState(LID_1)); EXPECT_EQUAL(ILidCommitState::State::COMPLETED, tracker.getState(LIDV_2_1_3)); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp index 1f71da5aeda..09854af99e1 100644 --- a/searchcore/src/tests/proton/common/selectpruner_test.cpp +++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp @@ -10,10 +10,10 @@ #include <vespa/document/select/parser.h> #include <vespa/document/select/cloningvisitor.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> -LOG_SETUP("selectpruner_test"); +LOG_SETUP(".selectpruner_test"); using document::DataType; using document::Document; @@ -824,5 +824,3 @@ TEST_F("Complex imported field references return Invalid", TestFixture) } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore b/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore deleted file mode 100644 index bb0963e5ec3..00000000000 --- a/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_state_reporter_utils_test_app diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt b/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt deleted file mode 100644 index 1bdb0b613cf..00000000000 --- a/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_state_reporter_utils_test_app TEST - SOURCES - state_reporter_utils_test.cpp - DEPENDS - searchcore_pcommon -) -vespa_add_test(NAME searchcore_state_reporter_utils_test_app COMMAND searchcore_state_reporter_utils_test_app) diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp b/searchcore/src/tests/proton/common/state_reporter_utils_test.cpp index 749f8b147ac..6c9025d276f 100644 --- a/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp +++ b/searchcore/src/tests/proton/common/state_reporter_utils_test.cpp @@ -1,10 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("state_reporter_utils_test"); #include <vespa/searchcore/proton/common/state_reporter_utils.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace proton; using namespace vespalib::slime; @@ -44,4 +42,3 @@ TEST("require that advanced status report is correctly converted to slime") message("foo")))); } -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/statusreport/statusreport_test.cpp b/searchcore/src/tests/proton/common/statusreport_test.cpp index d1ef6c3af29..052eb795529 100644 --- a/searchcore/src/tests/proton/statusreport/statusreport_test.cpp +++ b/searchcore/src/tests/proton/common/statusreport_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/common/statusreport.h> namespace proton { @@ -37,5 +37,3 @@ TEST("require that custom status report works") } } // namespace proton - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/vespa_testrunner.cpp b/searchcore/src/tests/proton/common/vespa_testrunner.cpp new file mode 100644 index 00000000000..1e4e79047c3 --- /dev/null +++ b/searchcore/src/tests/proton/common/vespa_testrunner.cpp @@ -0,0 +1,8 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Unit tests for predicate_index. +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/log/log.h> +LOG_SETUP("proton_common_test"); + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp index 173cfbbf052..361e37278da 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp @@ -65,7 +65,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/encoding/base64.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp index c428f350d1a..e09a0d2ea2b 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchcore/proton/test/test.h> #include <vespa/persistence/spi/test.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("buckethandler_test"); diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp index 3904156170d..c713108dcc0 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchcore/proton/test/test.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("combiningfeedview_test"); diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 2504bca17e4..2eeae63dc5f 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-summary.h> #include <vespa/document/datatype/documenttype.h> @@ -146,7 +146,7 @@ ViewSet::~ViewSet() = default; struct EmptyConstantValueFactory : public vespalib::eval::ConstantValueFactory { vespalib::eval::ConstantValue::UP create(const vespalib::string &, const vespalib::string &) const override { - return vespalib::eval::ConstantValue::UP(nullptr); + return {}; } }; @@ -178,13 +178,13 @@ struct Fixture const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, IDocumentDBReferenceResolver& resolver, - SerialNum serial_num); + SerialNum serial_num) const; IReprocessingInitializer::UP reconfigure(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, IDocumentDBReferenceResolver& resolver, uint32_t docid_limit, - SerialNum serial_num); + SerialNum serial_num) const; }; Fixture::Fixture() @@ -255,7 +255,7 @@ Fixture::reconfigure(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, IDocumentDBReferenceResolver& resolver, - SerialNum serial_num) + SerialNum serial_num) const { EXPECT_FALSE(reconfig_params.shouldAttributeManagerChange()); uint32_t docid_limit = 1; @@ -271,7 +271,7 @@ Fixture::reconfigure(const DocumentDBConfig& new_config_snapshot, const ReconfigParams& reconfig_params, IDocumentDBReferenceResolver& resolver, uint32_t docid_limit, - SerialNum serial_num) + SerialNum serial_num) const { AttributeCollectionSpecFactory attr_spec_factory(AllocStrategy(), false); auto prepared_reconfig = _configurer->prepare_reconfig(new_config_snapshot, attr_spec_factory, reconfig_params, docid_limit, serial_num); @@ -279,7 +279,7 @@ Fixture::reconfigure(const DocumentDBConfig& new_config_snapshot, return _configurer->reconfigure(new_config_snapshot, old_config_snapshot, reconfig_params, resolver, *prepared_reconfig, serial_num); } -using MySummaryAdapter = test::MockSummaryAdapter; +using MySummaryAdapter = proton::test::MockSummaryAdapter; struct MyFastAccessFeedView { @@ -367,14 +367,14 @@ FastAccessFixture::reconfigure(const DocumentDBConfig& new_config_snapshot, DocumentDBConfig::SP createConfig() { - return test::DocumentDBConfigBuilder(0, make_shared<Schema>(), "client", DOC_TYPE). + return proton::test::DocumentDBConfigBuilder(0, make_shared<Schema>(), "client", DOC_TYPE). repo(createRepo()).build(); } DocumentDBConfig::SP createConfig(const Schema::SP &schema) { - return test::DocumentDBConfigBuilder(0, schema, "client", DOC_TYPE). + return proton::test::DocumentDBConfigBuilder(0, schema, "client", DOC_TYPE). repo(createRepo()).build(); } diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp index 56fb1ab3914..e3ab2737fe5 100644 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp @@ -3,7 +3,7 @@ #include <vespa/searchcore/proton/server/document_scan_iterator.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/documentid.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index 75ef56299d9..73d27672380 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -5,7 +5,7 @@ #include <vespa/config-rank-profiles.h> #include <vespa/searchcore/proton/server/documentdbconfig.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-summary.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp index 95ad4bd143b..862792bf274 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/server/documentdbconfig.h> #include <vespa/searchcore/proton/server/documentdbconfigscout.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-attributes.h> #include <ostream> diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 808c5743538..cdd275e898d 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -38,7 +38,7 @@ #include <vespa/searchlib/test/doc_builder.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp index b92560c57e9..e1ec3d2bb29 100644 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp +++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/server/i_blockable_maintenance_job.h> #include <vespa/searchcore/proton/server/job_tracked_maintenance_job.h> #include <vespa/searchcore/proton/test/simple_job_tracker.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/threadstackexecutor.h> diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 509210679da..6c67848ae51 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -35,7 +35,7 @@ #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/lambdatask.h> diff --git a/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp b/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp index 62530e9de7b..766158bd178 100644 --- a/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp +++ b/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp @@ -2,7 +2,7 @@ #include <vespa/searchcore/proton/server/i_blockable_maintenance_job.h> #include <vespa/searchcore/proton/server/move_operation_limiter.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <queue> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index e6923674584..db30c8e03fb 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -17,7 +17,7 @@ #include <vespa/searchlib/test/doc_builder.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("storeonlyfeedview_test"); diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index 40b1904cd1a..7a16e6b17cb 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -2,7 +2,7 @@ #include <vespa/config-proton.h> #include <vespa/searchcore/proton/server/threading_service_config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/hw_info.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index 4bcfb2eedd9..335119927f7 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/documentmetastore/i_store.h> #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h> #include <vespa/searchcore/proton/test/thread_utils.h> diff --git a/searchcore/src/tests/proton/feedoperation/.gitignore b/searchcore/src/tests/proton/feedoperation/.gitignore deleted file mode 100644 index cfdeb9049b2..00000000000 --- a/searchcore/src/tests/proton/feedoperation/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*_test -.depend -Makefile -searchcore_feedoperation_test_app diff --git a/searchcore/src/tests/proton/feedoperation/CMakeLists.txt b/searchcore/src/tests/proton/feedoperation/CMakeLists.txt deleted file mode 100644 index fe9d3bad302..00000000000 --- a/searchcore/src/tests/proton/feedoperation/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_feedoperation_test_app TEST - SOURCES - feedoperation_test.cpp - DEPENDS - searchcore_feedoperation - searchcore_pcommon -) -vespa_add_test(NAME searchcore_feedoperation_test_app COMMAND searchcore_feedoperation_test_app) diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index 9a0dc47771f..dc9dcd3e0b0 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -13,7 +13,7 @@ #include <vespa/searchlib/common/flush_token.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <mutex> #include <thread> diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 06a2ebec958..7abac088011 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/flushengine/active_flush_stats.h> #include <vespa/searchcore/proton/flushengine/flush_target_candidate.h> diff --git a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp index 39175e8c27c..70a8887b9bb 100644 --- a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp +++ b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/common/i_compactable_lid_space.h> diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp index 51465c59ac7..c52eb940817 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/test/mock_index_manager.h> #include <vespa/searchlib/test/doc_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/initializer/task_runner_test.cpp b/searchcore/src/tests/proton/initializer/task_runner_test.cpp index 82cce924832..a8b9969299f 100644 --- a/searchcore/src/tests/proton/initializer/task_runner_test.cpp +++ b/searchcore/src/tests/proton/initializer/task_runner_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("task_runner_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> #include <vespa/searchcore/proton/initializer/task_runner.h> #include <vespa/vespalib/stllike/string.h> diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 83b7e10c7a8..b917e69ce7b 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -30,7 +30,7 @@ #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/thread_bundle.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/query/tree/querytreecreator.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index 64c6870499c..a1c73d0fa76 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -29,7 +29,7 @@ #include <vespa/searchlib/queryeval/sourceblendersearch.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("querynodes_test"); diff --git a/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp b/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp index e85bef44f3e..d73038ddcb5 100644 --- a/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp +++ b/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp @@ -10,7 +10,7 @@ LOG_SETUP("resolveviewvisitor_test"); #include <vespa/searchcore/proton/matching/viewresolver.h> #include <vespa/searchlib/query/tree/node.h> #include <vespa/searchlib/query/tree/querybuilder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <string> namespace fef_test = search::fef::test; diff --git a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp index 8bfbcacbf23..3153bcccb42 100644 --- a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp +++ b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchcore/proton/matching/match_tools.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp index 43d2c54fd03..f440d573859 100644 --- a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp +++ b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp @@ -15,7 +15,7 @@ LOG_SETUP("termdataextractor_test"); #include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/querybuilder.h> #include <vespa/searchlib/query/weight.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <string> #include <vector> diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore deleted file mode 100644 index 84c97c63aca..00000000000 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_documentdb_job_trackers_test_app diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt deleted file mode 100644 index 81d054e2242..00000000000 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentdb_job_trackers_test_app TEST - SOURCES - documentdb_job_trackers_test.cpp - DEPENDS - searchcore_proton_metrics - searchcore_test -) -vespa_add_test(NAME searchcore_documentdb_job_trackers_test_app COMMAND searchcore_documentdb_job_trackers_test_app) diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore b/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore deleted file mode 100644 index 2e02ec8191b..00000000000 --- a/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_job_load_sampler_test_app diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt deleted file mode 100644 index 955b1e14028..00000000000 --- a/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_job_load_sampler_test_app TEST - SOURCES - job_load_sampler_test.cpp - DEPENDS - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_job_load_sampler_test_app COMMAND searchcore_job_load_sampler_test_app) diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore b/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore deleted file mode 100644 index 85e6097878b..00000000000 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_job_tracked_flush_test_app diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt deleted file mode 100644 index a4f0ff0bcec..00000000000 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_job_tracked_flush_test_app TEST - SOURCES - job_tracked_flush_test.cpp - DEPENDS - searchcore_proton_metrics - searchcore_test -) -vespa_add_test(NAME searchcore_job_tracked_flush_test_app COMMAND searchcore_job_tracked_flush_test_app) diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp index 523ffccd2f0..dd622a4cac1 100644 --- a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp +++ b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp @@ -3,7 +3,7 @@ #include <vespa/metrics/metricset.h> #include <vespa/searchcore/proton/metrics/attribute_metrics.h> #include <vespa/searchcore/proton/metrics/metrics_engine.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("metrics_engine_test"); diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp index 528067aeeb1..f13b9e9a9db 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -5,7 +5,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using namespace proton; diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 21ac6893356..0f1e2e9487a 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -17,7 +17,7 @@ #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <set> using document::BucketId; diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 06264e3e642..1dfc6c481b6 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -24,7 +24,7 @@ #include <vespa/fileacquirer/config-filedistributorrpc.h> #include <vespa/vespalib/util/hw_info.h> #include <vespa/vespalib/util/varholder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config.h> #include <map> #include <thread> diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp index 411dd88f995..881c547df1e 100644 --- a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp +++ b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/stringfmt.h> #include <filesystem> diff --git a/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp index a2f85ce991b..dee0b6a2448 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <cassert> #include <vespa/log/log.h> LOG_SETUP("document_db_reference_test"); diff --git a/searchcore/src/tests/proton/reference/document_db_reference_registry/document_db_reference_registry_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_registry/document_db_reference_registry_test.cpp index 0aa0e31442e..5ec785496e2 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_registry/document_db_reference_registry_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_registry/document_db_reference_registry_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchcore/proton/reference/document_db_reference_registry.h> #include <vespa/searchcore/proton/test/mock_document_db_reference.h> diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp index 80f4f826c4a..57bae7cfc7c 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp @@ -20,7 +20,7 @@ #include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-imported-fields.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp index 48df346317a..22c6c936465 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/base/documentid.h> #include <vespa/vespalib/util/threadstackexecutor.h> diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp index da3449b91aa..4c70c1830db 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h> diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp index 2c218534545..6e08c8397d0 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/base/globalid.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp index 36429c79a26..70211909296 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp index 97a319ecfd4..97c5b458867 100644 --- a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp @@ -4,7 +4,7 @@ LOG_SETUP("document_reprocessing_handler_test"); #include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h> #include <vespa/searchlib/test/doc_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using namespace proton; diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp index bb62b239cd2..5e7b11e09a7 100644 --- a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp @@ -4,7 +4,7 @@ LOG_SETUP("reprocessing_runner_test"); #include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h> #include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace proton; diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index 142a2b8693f..b05964fe7e0 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -40,7 +40,7 @@ #include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> diff --git a/searchcore/src/tests/proton/server/feeddebugger_test.cpp b/searchcore/src/tests/proton/server/feeddebugger_test.cpp index cedaf2be12e..4d1bbdb2f03 100644 --- a/searchcore/src/tests/proton/server/feeddebugger_test.cpp +++ b/searchcore/src/tests/proton/server/feeddebugger_test.cpp @@ -6,7 +6,7 @@ LOG_SETUP("feeddebugger_test"); #include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/common/feeddebugger.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::DocumentId; using std::string; diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index 8ecdfba63f5..c46510c5181 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -16,7 +16,7 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/foreground_thread_executor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp index 79b4b4a3627..0dc8390d7d9 100644 --- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp +++ b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/server/memory_flush_config_updater.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp index e07886f9576..c0245cc1a9d 100644 --- a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp +++ b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("memoryconfigstore_test"); #include <vespa/searchcommon/common/schema.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using search::SerialNum; using search::index::Schema; diff --git a/searchcore/src/tests/proton/statusreport/.gitignore b/searchcore/src/tests/proton/statusreport/.gitignore deleted file mode 100644 index 68753df292a..00000000000 --- a/searchcore/src/tests/proton/statusreport/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_statusreport_test_app diff --git a/searchcore/src/tests/proton/statusreport/CMakeLists.txt b/searchcore/src/tests/proton/statusreport/CMakeLists.txt deleted file mode 100644 index 1f2a3cfbec1..00000000000 --- a/searchcore/src/tests/proton/statusreport/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_statusreport_test_app TEST - SOURCES - statusreport_test.cpp - DEPENDS - searchcore_pcommon -) -vespa_add_test(NAME searchcore_statusreport_test_app COMMAND searchcore_statusreport_test_app) diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp index bb1d06acb4c..2ab81058e43 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> #include <vespa/searchlib/engine/searchreply.h> diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp index e21cb4c1282..88939958aa0 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp @@ -53,11 +53,7 @@ GroupingContext::setDistributionKey(uint32_t distributionKey) GroupingContext::GroupingContext(const BitVector & validLids, const std::atomic<steady_time> & now_ref, vespalib::steady_time timeOfDoom, const char *groupSpec, uint32_t groupSpecLen) - : _validLids(validLids), - _now_ref(now_ref), - _timeOfDoom(timeOfDoom), - _os(), - _groupingList() + : GroupingContext(validLids, now_ref, timeOfDoom) { deserialize(groupSpec, groupSpecLen); } @@ -71,11 +67,7 @@ GroupingContext::GroupingContext(const BitVector & validLids, const std::atomic< { } GroupingContext::GroupingContext(const GroupingContext & rhs) - : _validLids(rhs._validLids), - _now_ref(rhs._now_ref), - _timeOfDoom(rhs._timeOfDoom), - _os(), - _groupingList() + : GroupingContext(rhs._validLids, rhs._now_ref, rhs._timeOfDoom) { } void diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp index 9ece37665cd..18c942b80a0 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp @@ -4,6 +4,7 @@ #include "groupingcontext.h" #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/expression/attributenode.h> +#include <vespa/searchlib/aggregation/modifiers.h> #include <vespa/vespalib/util/issue.h> #include <vespa/vespalib/util/stringfmt.h> @@ -30,7 +31,7 @@ bool GroupingManager::empty() const { } void -GroupingManager::init(const IAttributeContext &attrCtx) +GroupingManager::init(const IAttributeContext &attrCtx, const document::DocumentType * documentType) { GroupingContext::GroupingList list; GroupingContext::GroupingList &groupingList(_groupingContext.getGroupingList()); @@ -48,7 +49,8 @@ GroupingManager::init(const IAttributeContext &attrCtx) an.enableEnumOptimization(true); } } - ConfigureStaticParams stuff(&attrCtx, nullptr); + aggregation::NonAttribute2DocumentAccessor nonAttributes2DocumentAccess(attrCtx); + ConfigureStaticParams stuff(&attrCtx, documentType); grouping.configureStaticStuff(stuff); list.push_back(groupingList[i]); } catch (const std::exception & e) { diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h index b0856ae7189..05f0e1886aa 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h @@ -8,6 +8,7 @@ namespace search { struct RankedHit; class BitVector; } +namespace document { class DocumentType; } namespace search::grouping { @@ -21,6 +22,7 @@ class GroupingManager { private: GroupingContext &_groupingContext; + const document::DocumentType *_documentType; public: GroupingManager(const GroupingManager &) = delete; GroupingManager &operator=(const GroupingManager &) = delete; @@ -30,7 +32,8 @@ public: * @param groupingContext Context to use for grouping **/ GroupingManager(GroupingContext & groupingContext) noexcept - : _groupingContext(groupingContext) + : _groupingContext(groupingContext), + _documentType(nullptr) {} /** @@ -43,7 +46,7 @@ public: * * @param attrCtx attribute context **/ - void init(const attribute::IAttributeContext &attrCtx); + void init(const attribute::IAttributeContext &attrCtx, const document::DocumentType * documentType); /** * Perform actual grouping on the given results. diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp index ef45b36b551..8d845766966 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp @@ -16,29 +16,24 @@ using search::attribute::IAttributeContext; GroupingSession::GroupingSession(const SessionId &sessionId, GroupingContext & groupingContext, - const IAttributeContext &attrCtx) + const IAttributeContext &attrCtx, + const document::DocumentType * documentType) : _sessionId(sessionId), _mgrContext(std::make_unique<GroupingContext>(groupingContext)), _groupingManager(std::make_unique<GroupingManager>(*_mgrContext)), _timeOfDoom(groupingContext.getTimeOfDoom()) { - init(groupingContext, attrCtx); + init(groupingContext, attrCtx, documentType); } GroupingSession::~GroupingSession() = default; -using search::expression::ExpressionNode; -using search::expression::AttributeNode; -using search::expression::ConfigureStaticParams; -using search::aggregation::Grouping; -using search::aggregation::GroupingLevel; - void -GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext &attrCtx) +GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext &attrCtx, + const document::DocumentType * documentType) { GroupingList & sessionList(groupingContext.getGroupingList()); - for (size_t i = 0; i < sessionList.size(); ++i) { - GroupingPtr g(sessionList[i]); + for (auto g : sessionList) { // Make internal copy of those we want to keep for another pass if (!_sessionId.empty() && g->getLastLevel() < g->levels().size()) { auto gp = std::make_shared<Grouping>(*g); @@ -48,7 +43,7 @@ GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext } _mgrContext->addGrouping(std::move(g)); } - _groupingManager->init(attrCtx); + _groupingManager->init(attrCtx, documentType); } void @@ -60,7 +55,8 @@ GroupingSession::prepareThreadContextCreation(size_t num_threads) } GroupingContext::UP -GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext &attrCtx) +GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext &attrCtx, + const document::DocumentType * documentType) { auto ctx = std::make_unique<GroupingContext>(*_mgrContext); if (thread_id == 0) { @@ -70,7 +66,7 @@ GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext & } else { ctx->deserialize(_mgrContext->getResult().peek(), _mgrContext->getResult().size()); GroupingManager man(*ctx); - man.init(attrCtx); + man.init(attrCtx, documentType); } return ctx; } diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.h b/searchcore/src/vespa/searchcore/grouping/groupingsession.h index e3e600161d0..d24f0b4ef3d 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingsession.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.h @@ -7,6 +7,7 @@ #include <vector> #include <map> +namespace document { class DocumentType; } namespace search::aggregation { class Grouping; } namespace search::grouping { @@ -44,7 +45,8 @@ public: **/ GroupingSession(const SessionId & sessionId, GroupingContext & groupingContext, - const attribute::IAttributeContext &attrCtx); + const attribute::IAttributeContext &attrCtx, + const document::DocumentType * documentType); GroupingSession(const GroupingSession &) = delete; GroupingSession &operator=(const GroupingSession &) = delete; @@ -63,7 +65,7 @@ public: * @param groupingContext The current grouping context. * @param attrCtx attribute context. **/ - void init(GroupingContext & groupingContext, const attribute::IAttributeContext &attrCtx); + void init(GroupingContext & groupingContext, const attribute::IAttributeContext &attrCtx, const document::DocumentType * documentType); /** * This function is called to prepare for creation of individual @@ -85,7 +87,8 @@ public: * @param thread_id thread id * @param attrCtx attribute context. **/ - std::unique_ptr<GroupingContext> createThreadContext(size_t thread_id, const attribute::IAttributeContext &attrCtx); + std::unique_ptr<GroupingContext> createThreadContext(size_t thread_id, const attribute::IAttributeContext &attrCtx, + const document::DocumentType * documentType); /** * Return the GroupingManager to use when performing grouping. diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp index 3337c1adb9b..1aad609963d 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp @@ -16,12 +16,6 @@ using namespace search::docsummary; namespace proton { -namespace { - -const vespalib::string DOCUMENT_ID_FIELD("documentid"); - -} - DocumentStoreAdapter:: DocumentStoreAdapter(const search::IDocumentStore & docStore, const DocumentTypeRepo &repo) diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h index 308a85baf96..757ce863245 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h @@ -16,7 +16,7 @@ private: public: DocumentStoreAdapter(const search::IDocumentStore &docStore, const document::DocumentTypeRepo &repo); - ~DocumentStoreAdapter(); + ~DocumentStoreAdapter() override; std::unique_ptr<const search::docsummary::IDocsumStoreDocument> get_document(uint32_t docId) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp index e96e231d46a..623a17b89f4 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp @@ -74,7 +74,7 @@ ResultProcessor::ResultProcessor(IAttributeContext &attrContext, _wasMerged(false) { if (!_groupingContext.empty()) { - _groupingSession = std::make_unique<GroupingSession>(sessionId, _groupingContext, attrContext); + _groupingSession = std::make_unique<GroupingSession>(sessionId, _groupingContext, attrContext, nullptr); } } @@ -98,7 +98,7 @@ ResultProcessor::createThreadContext(const vespalib::Doom & hardDoom, size_t thr auto result = std::make_unique<PartialResult>((_offset + _hits), sort->hasSortData()); search::grouping::GroupingContext::UP groupingContext; if (_groupingSession) { - groupingContext = _groupingSession->createThreadContext(thread_id, _attrContext); + groupingContext = _groupingSession->createThreadContext(thread_id, _attrContext, nullptr); } return std::make_unique<Context>(_metaStore.getValidLids(), std::move(sort), std::move(result), std::move(groupingContext)); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h index e11e09ef338..1cf6284295f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h @@ -41,7 +41,7 @@ public: Sort(const Sort &) = delete; Sort & operator = (const Sort &) = delete; Sort(uint32_t partitionId, const vespalib::Doom & doom, IAttributeContext &ac, const vespalib::string &ss); - bool hasSortData() const { + bool hasSortData() const noexcept { return (sorter == (const FastS_IResultSorter *) &sortSpec); } }; @@ -51,7 +51,7 @@ public: **/ struct GroupingSource : vespalib::DualMergeDirector::Source { GroupingContext *ctx; - GroupingSource(GroupingContext *g) : ctx(g) {} + explicit GroupingSource(GroupingContext *g) noexcept : ctx(g) {} void merge(Source &s) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h index afeacfe6a4f..a95c7b7a8a8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h +++ b/searchcore/src/vespa/searchcore/proton/matching/sessionmanager.h @@ -46,7 +46,7 @@ private: std::unique_ptr<SearchSessionCache> _search_map; public: - SessionManager(uint32_t maxSizeGrouping); + explicit SessionManager(uint32_t maxSizeGrouping); ~SessionManager(); void insert(search::grouping::GroupingSession::UP session); diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h index 5ac170cd6f3..a98e98bc66a 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h @@ -14,7 +14,7 @@ class ImportedAttributesRepo; * Interface used by a given document db to resolve all references to parent document dbs. */ struct IDocumentDBReferenceResolver { - virtual ~IDocumentDBReferenceResolver() {} + virtual ~IDocumentDBReferenceResolver() = default; virtual std::unique_ptr<ImportedAttributesRepo> resolve(const search::IAttributeManager &newAttrMgr, const search::IAttributeManager &oldAttrMgr, const std::shared_ptr<search::IDocumentMetaStoreContext> &documentMetaStore, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index b073fb2133d..3fc3202ebd7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -202,9 +202,9 @@ fillInPositionFields(Document &doc, DocumentIdT lid, const DocumentRetriever::Po class PopulateVisitor : public search::IDocumentVisitor { public: - PopulateVisitor(const DocumentRetriever & retriever, search::IDocumentVisitor & visitor) : - _retriever(retriever), - _visitor(visitor) + PopulateVisitor(const DocumentRetriever & retriever, search::IDocumentVisitor & visitor) noexcept + : _retriever(retriever), + _visitor(visitor) { } void visit(uint32_t lid, document::Document::UP doc) override { if (doc) { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 4bf21b814fe..78101ca6afe 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -108,7 +108,7 @@ DocumentSubDBCollection::~DocumentSubDBCollection() void DocumentSubDBCollection::createRetrievers() { - RetrieversSP retrievers(std::make_shared<std::vector<IDocumentRetriever::SP>>()); + RetrieversSP retrievers(std::make_shared<std::vector<std::shared_ptr<IDocumentRetriever>>>()); retrievers->reserve(_subDBs.size()); for (auto subDb : _subDBs) { retrievers->emplace_back(subDb->getDocumentRetriever()); diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp index 27913eca62b..02fd222b9ad 100644 --- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp @@ -18,9 +18,6 @@ using namespace vespalib::slime; namespace proton { -EmptySearchView::EmptySearchView() = default; - - DocsumReply::UP EmptySearchView::getDocsums(const DocsumRequest &req) { diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h index 8dc5f10b717..cda593a9600 100644 --- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h @@ -11,12 +11,9 @@ class EmptySearchView : public ISearchHandler public: using SP = std::shared_ptr<EmptySearchView>; - EmptySearchView(); - std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override; - - std::unique_ptr<SearchReply> - match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override; + std::unique_ptr<SearchReply> match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override; +private: }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 4972cc790c5..1e96f604530 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -201,8 +201,7 @@ FastAccessDocSubDB::FastAccessDocSubDB(const Config &cfg, const Context &ctx) _fastAccessAttributesOnly(cfg._fastAccessAttributesOnly), _initAttrMgr(), _fastAccessFeedView(), - _configurer(_fastAccessFeedView, - getSubDbName()), + _configurer(_fastAccessFeedView, getSubDbName()), _subAttributeMetrics(ctx._subAttributeMetrics), _addMetrics(cfg._addMetrics), _metricsWireService(ctx._metricsWireService), @@ -306,12 +305,12 @@ FastAccessDocSubDB::getAttributeManager() const return extractAttributeManager(_fastAccessFeedView.get()); } -IDocumentRetriever::UP +std::shared_ptr<IDocumentRetriever> FastAccessDocSubDB::getDocumentRetriever() { FastAccessFeedView::SP feedView = _fastAccessFeedView.get(); proton::IAttributeManager::SP attrMgr = extractAttributeManager(feedView); - return std::make_unique<FastAccessDocumentRetriever>(feedView, attrMgr); + return std::make_shared<FastAccessDocumentRetriever>(feedView, attrMgr); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index 4337185a8d3..692667ff2b4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -104,7 +104,7 @@ protected: public: FastAccessDocSubDB(const Config &cfg, const Context &ctx); - ~FastAccessDocSubDB(); + ~FastAccessDocSubDB() override; std::unique_ptr<DocumentSubDbInitializer> createInitializer(const DocumentDBConfig &configSnapshot, SerialNum configSerialNum, @@ -121,7 +121,7 @@ public: std::shared_ptr<IAttributeWriter> get_attribute_writer() const override; std::shared_ptr<IAttributeManager> getAttributeManager() const override; - IDocumentRetriever::UP getDocumentRetriever() override; + std::shared_ptr<IDocumentRetriever> getDocumentRetriever() override; void onReplayDone() override; void onReprocessDone(SerialNum serialNum) override; SerialNum getOldestFlushedSerial() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.cpp index c2a190cc494..323b91d017c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.cpp @@ -4,7 +4,7 @@ namespace proton { -FastAccessDocumentRetriever::FastAccessDocumentRetriever(FastAccessFeedView::SP feedView, IAttributeManager::SP attrMgr) +FastAccessDocumentRetriever::FastAccessDocumentRetriever(FastAccessFeedView::SP feedView, IAttributeManager::SP attrMgr) noexcept : DocumentRetriever(feedView->getPersistentParams()._docTypeName, *feedView->getDocumentTypeRepo(), *feedView->getSchema(), diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h index 6a9a385ce5a..83884f8f327 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h @@ -20,7 +20,7 @@ private: IAttributeManager::SP _attrMgr; public: - FastAccessDocumentRetriever(FastAccessFeedView::SP feedView, IAttributeManager::SP attrMgr); + FastAccessDocumentRetriever(FastAccessFeedView::SP feedView, IAttributeManager::SP attrMgr) noexcept; ~FastAccessDocumentRetriever() override; uint32_t getDocIdLimit() const override { return _feedView->getDocIdLimit().get(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 60e844dad60..2249cb5392a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -68,8 +68,8 @@ public: using OnDone = std::shared_ptr<vespalib::IDestructorCallback>; using SessionManager = matching::SessionManager; public: - IDocumentSubDB() { } - virtual ~IDocumentSubDB() { } + IDocumentSubDB() noexcept { } + virtual ~IDocumentSubDB() = default; virtual uint32_t getSubDbId() const = 0; virtual vespalib::string getName() const = 0; @@ -124,7 +124,7 @@ public: virtual void pruneRemovedFields(SerialNum serialNum) = 0; virtual void setIndexSchema(const SchemaSP &schema, SerialNum serialNum) = 0; virtual search::SearchableStats getSearchableStats() const = 0; - virtual std::unique_ptr<IDocumentRetriever> getDocumentRetriever() = 0; + virtual std::shared_ptr<IDocumentRetriever> getDocumentRetriever() = 0; virtual matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const = 0; virtual void close() = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 5e9d9585ca9..fd62baa5bcb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -162,8 +162,8 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const if (initializer && initializer->hasReprocessors()) { tasks.emplace_back(createReprocessingTask(*initializer, newConfigSnapshot.getDocumentTypeRepoSP())); } - proton::IAttributeManager::SP newMgr = getAttributeManager(); if (_addMetrics) { + proton::IAttributeManager::SP newMgr = getAttributeManager(); reconfigureAttributeMetrics(*newMgr, *oldMgr); } } else { @@ -309,10 +309,10 @@ SearchableDocSubDB::getSearchableStats() const return _indexMgr ? _indexMgr->getSearchableStats() : search::SearchableStats(); } -IDocumentRetriever::UP +std::shared_ptr<IDocumentRetriever> SearchableDocSubDB::getDocumentRetriever() { - return std::make_unique<FastAccessDocumentRetriever>(_rFeedView.get(), _rSearchView.get()->getAttributeManager()); + return std::make_shared<FastAccessDocumentRetriever>(_rFeedView.get(), _rSearchView.get()->getAttributeManager()); } MatchingStats diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index eb27406e228..01b78c6bb4d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -129,7 +129,7 @@ public: void setIndexSchema(const Schema::SP &schema, SerialNum serialNum) override; size_t getNumActiveDocs() const override; search::SearchableStats getSearchableStats() const override ; - IDocumentRetriever::UP getDocumentRetriever() override; + std::shared_ptr<IDocumentRetriever> getDocumentRetriever() override; matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const override; void close() override; std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h index 41b84edc887..15990f8a55d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h @@ -15,9 +15,9 @@ private: std::shared_ptr<DocumentDB> _documentDB; vespalib::RetainGuard _retainGuard; public: - SearchHandlerProxy(std::shared_ptr<DocumentDB> documentDB); - + explicit SearchHandlerProxy(std::shared_ptr<DocumentDB> documentDB); ~SearchHandlerProxy() override; + std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & request) override; std::unique_ptr<SearchReply> match(const SearchRequest &req, ThreadBundle &threadBundle) const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 0c217eeeeba..0e383c47e9b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -548,10 +548,10 @@ StoreOnlyDocSubDB::getSearchableStats() const return {}; } -IDocumentRetriever::UP +std::shared_ptr<IDocumentRetriever> StoreOnlyDocSubDB::getDocumentRetriever() { - return std::make_unique<MinimalDocumentRetriever>(_docTypeName, _iFeedView.get()->getDocumentTypeRepo(), + return std::make_shared<MinimalDocumentRetriever>(_docTypeName, _iFeedView.get()->getDocumentTypeRepo(), *_metaStoreCtx, _iSummaryMgr->getBackingStore(), _subDbType != SubDbType::REMOVED); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index d530e44755c..db7b59611d6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -232,7 +232,7 @@ public: void pruneRemovedFields(SerialNum serialNum) override; void setIndexSchema(const Schema::SP &schema, SerialNum serialNum) override; search::SearchableStats getSearchableStats() const override; - IDocumentRetriever::UP getDocumentRetriever() override; + std::shared_ptr<IDocumentRetriever> getDocumentRetriever() override; matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const override; void close() override; std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override; diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt index 5474f1ac157..b7cd9d35b1e 100644 --- a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt @@ -8,6 +8,7 @@ vespa_add_library(searchcore_test STATIC bucketstatecalculator.cpp clusterstatehandler.cpp documentdb_config_builder.cpp + dummy_document_sub_db.cpp dummy_feed_view.cpp dummy_flush_target.cpp dummydbowner.cpp diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.cpp b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.cpp new file mode 100644 index 00000000000..f70bdb0bff6 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.cpp @@ -0,0 +1,18 @@ +#include "dummy_document_sub_db.h" + +namespace proton::test { + +DummyDocumentSubDb::DummyDocumentSubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, uint32_t subDbId) + : _subDbId(subDbId), + _metaStoreCtx(std::move(bucketDB)), + _summaryManager(), + _indexManager(), + _summaryAdapter(), + _indexWriter(), + _service(1), + _pendingLidTracker() +{ } + +DummyDocumentSubDb::~DummyDocumentSubDb() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 7d6ea39a65a..35ada75b001 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -30,18 +30,8 @@ struct DummyDocumentSubDb : public IDocumentSubDB mutable TransportAndExecutorService _service; PendingLidTracker _pendingLidTracker; - DummyDocumentSubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, uint32_t subDbId) - : _subDbId(subDbId), - _metaStoreCtx(std::move(bucketDB)), - _summaryManager(), - _indexManager(), - _summaryAdapter(), - _indexWriter(), - _service(1), - _pendingLidTracker() - { - } - ~DummyDocumentSubDb() override { } + DummyDocumentSubDb(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, uint32_t subDbId); + ~DummyDocumentSubDb() override; void close() override { } uint32_t getSubDbId() const override { return _subDbId; } vespalib::string getName() const override { return "dummysubdb"; } @@ -59,11 +49,11 @@ struct DummyDocumentSubDb : public IDocumentSubDB IReprocessingTask::List applyConfig(const DocumentDBConfig &, const DocumentDBConfig &, SerialNum, const ReconfigParams &, IDocumentDBReferenceResolver &, const DocumentSubDBReconfig&) override { - return IReprocessingTask::List(); + return {}; } void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &, OnDone) override { } - ISearchHandler::SP getSearchView() const override { return ISearchHandler::SP(); } - IFeedView::SP getFeedView() const override { return IFeedView::SP(); } + ISearchHandler::SP getSearchView() const override { return {}; } + IFeedView::SP getFeedView() const override { return {}; } void clearViews() override {} const ISummaryManager::SP &getSummaryManager() const override { return _summaryManager; } std::shared_ptr<IAttributeWriter> get_attribute_writer() const override { return {}; } @@ -74,7 +64,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB const IIndexWriter::SP &getIndexWriter() const override { return _indexWriter; } IDocumentMetaStoreContext &getDocumentMetaStoreContext() override { return _metaStoreCtx; } const IDocumentMetaStoreContext &getDocumentMetaStoreContext() const override { return _metaStoreCtx; } - IFlushTargetList getFlushTargets() override { return IFlushTargetList(); } + IFlushTargetList getFlushTargets() override { return {}; } size_t getNumDocs() const override { return 0; } size_t getNumActiveDocs() const override { return 0; } bool hasDocument(const document::DocumentId &) override { return false; } @@ -85,16 +75,16 @@ struct DummyDocumentSubDb : public IDocumentSubDB void pruneRemovedFields(SerialNum) override { } void setIndexSchema(const Schema::SP &, SerialNum) override { } search::SearchableStats getSearchableStats() const override { - return search::SearchableStats(); + return {}; } - IDocumentRetriever::UP getDocumentRetriever() override { - return IDocumentRetriever::UP(); + std::shared_ptr<IDocumentRetriever> getDocumentRetriever() override { + return {}; } matching::MatchingStats getMatcherStats(const vespalib::string &) const override { - return matching::MatchingStats(); + return {}; } std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override { - return std::shared_ptr<IDocumentDBReference>(); + return {}; } PendingLidTrackerBase &getUncommittedLidsTracker() override { diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h index ce67099dc3e..288f6ebfebd 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> #include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/document/base/globalid.h> diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 763ba860fe6..b9c80a3c04d 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -21,7 +21,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/vespalib/util/rand48.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchlib/src/tests/aggregator/perdocexpr_test.cpp b/searchlib/src/tests/aggregator/perdocexpr_test.cpp index e9f0981739c..61c5a4f8de9 100644 --- a/searchlib/src/tests/aggregator/perdocexpr_test.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/objects/objectdumper.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> diff --git a/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp b/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp index 733c9e1a574..7589663073f 100644 --- a/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp +++ b/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("attribute_operation_test"); diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index e8d95eab7e2..56117fdd17f 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/attributefilewriter.h> #include <vespa/searchlib/attribute/attributefilebufferwriter.h> #include <vespa/searchlib/attribute/attribute_header.h> diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index c3254cf930d..2494b514c56 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/attribute/configconverter.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multinumericattribute.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("attribute_test"); diff --git a/searchlib/src/tests/attribute/changevector/changevector_test.cpp b/searchlib/src/tests/attribute/changevector/changevector_test.cpp index ff0b358ea2e..9dd96b1b72c 100644 --- a/searchlib/src/tests/attribute/changevector/changevector_test.cpp +++ b/searchlib/src/tests/attribute/changevector/changevector_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/changevector.hpp> #include <vespa/vespalib/stllike/hash_set.h> diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 0795d85e4a2..431d09a8665 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -19,7 +19,7 @@ #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp index fb733db5f71..868c85c7962 100644 --- a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp +++ b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/posting_list_merger.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index 7a794795cce..0278c4f32ef 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/attribute_blueprint_factory.h> #include <vespa/searchlib/attribute/attribute_weighted_set_blueprint.h> #include <vespa/searchlib/attribute/attributecontext.h> diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index 1bfb9fb41f9..b558c50488a 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/enumstore.h> #include <vespa/searchlib/attribute/singlestringattribute.h> diff --git a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp index 03cef8c8079..f76476dfda2 100644 --- a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp +++ b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vector> #include <algorithm> #include <cinttypes> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP("expglomb_test"); diff --git a/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp b/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp index d111a392270..2600ebedbee 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("bitvector_benchmark"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/bitvector.h> using namespace search; diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 758f44cdba2..ef9e801cda3 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/searchlib/common/growablebitvector.h> #include <vespa/searchlib/common/partialbitvector.h> diff --git a/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp b/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp index f4bbb0fe7ca..4f7a0dfa736 100644 --- a/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/condensedbitvectors.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/common/resultset/resultset_test.cpp b/searchlib/src/tests/common/resultset/resultset_test.cpp index d8cba35a96c..d757785206a 100644 --- a/searchlib/src/tests/common/resultset/resultset_test.cpp +++ b/searchlib/src/tests/common/resultset/resultset_test.cpp @@ -6,7 +6,7 @@ LOG_SETUP("resultset_test"); #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/resultset.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/arraysize.h> using namespace search; diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4_hugeword_cornercase_test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4_hugeword_cornercase_test.cpp index 76efb2ec788..c17d78b25f3 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4_hugeword_cornercase_test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4_hugeword_cornercase_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/countcompression.h> diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index 62fee24f972..d2b07c6c552 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -14,7 +14,7 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/test/make_attribute_map_lookup_node.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("attribute_node_test"); diff --git a/searchlib/src/tests/features/prod_features_test.cpp b/searchlib/src/tests/features/prod_features_test.cpp index fb00b4ff5e6..45682458dfa 100644 --- a/searchlib/src/tests/features/prod_features_test.cpp +++ b/searchlib/src/tests/features/prod_features_test.cpp @@ -2121,10 +2121,10 @@ TEST_F(ProdFeaturesTest, test_term) ASSERT_TRUE(ft.setup()); RankResult exp; - exp.addScore("term(1).significance", util::getSignificance(0.50)). + exp.addScore("term(1).significance", util::calculate_legacy_significance(0.50)). addScore("term(1).weight", 200.0f). addScore("term(1).connectedness", 0.7f). - addScore("term(2).significance", util::getSignificance(0.25)). + addScore("term(2).significance", util::calculate_legacy_significance(0.25)). addScore("term(2).weight", 400.0f). addScore("term(2).connectedness", 0.1f). // default connectedness setEpsilon(10e-6); @@ -2216,20 +2216,20 @@ Test::assertTermDistance(const TermDistanceCalculator::Result & exp, TEST_F(ProdFeaturesTest, test_utils) { { // getSignificance - EXPECT_NEAR(util::getSignificance(0.0), 1, EPS); - EXPECT_NEAR(util::getSignificance(0.0 + 1.0e-7), 1, EPS); - EXPECT_NEAR(util::getSignificance(1.0), 0.5, EPS); - EXPECT_NEAR(util::getSignificance(1.0 + 1.0e-7), 0.5, EPS); + EXPECT_NEAR(util::calculate_legacy_significance(0.0), 1, EPS); + EXPECT_NEAR(util::calculate_legacy_significance(0.0 + 1.0e-7), 1, EPS); + EXPECT_NEAR(util::calculate_legacy_significance(1.0), 0.5, EPS); + EXPECT_NEAR(util::calculate_legacy_significance(1.0 + 1.0e-7), 0.5, EPS); feature_t last = 1; for (uint32_t i = 2; i <= 100; i = i + 1) { - feature_t s = util::getSignificance(i * 1.0e-6); + feature_t s = util::calculate_legacy_significance(i * 1.0e-6); EXPECT_GT(s, 0); EXPECT_LT(s, 1); EXPECT_LT(s, last); last = s; } for (uint32_t i = 999900; i <= 1000000; i = i + 1) { - feature_t s = util::getSignificance(i * 1.0e-6); + feature_t s = util::calculate_legacy_significance(i * 1.0e-6); EXPECT_GT(s, 0); EXPECT_LT(s, 1); EXPECT_LT(s, last); diff --git a/searchlib/src/tests/features/util/CMakeLists.txt b/searchlib/src/tests/features/util/CMakeLists.txt index 0eee4d3b7ac..1315734eee7 100644 --- a/searchlib/src/tests/features/util/CMakeLists.txt +++ b/searchlib/src/tests/features/util/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchlib_util_test_app TEST util_test.cpp DEPENDS vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_util_test_app COMMAND searchlib_util_test_app) diff --git a/searchlib/src/tests/features/util/util_test.cpp b/searchlib/src/tests/features/util/util_test.cpp index 7f3d8ad209f..e51eb8e77b8 100644 --- a/searchlib/src/tests/features/util/util_test.cpp +++ b/searchlib/src/tests/features/util/util_test.cpp @@ -1,8 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/utils.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace search; using namespace search::fef; @@ -10,6 +10,14 @@ using namespace search::fef::test; using namespace search::features; using namespace search::features::util; +namespace search::features::util { + +void PrintTo(const DocumentFrequency& document_frequency, std::ostream* os) { + *os << "{" << document_frequency.frequency << "," << document_frequency.count << "}"; +} + +} + SimpleTermData make_term(uint32_t uid) { SimpleTermData term; term.setUniqueId(uid); @@ -30,31 +38,56 @@ struct TermLabelFixture { } }; -TEST_F("require that label can be mapped to term", TermLabelFixture) { - EXPECT_EQUAL((ITermData*)&f1.queryEnv.getTerms()[0], util::getTermByLabel(f1.queryEnv, "foo")); - EXPECT_EQUAL((ITermData*)0, util::getTermByLabel(f1.queryEnv, "bar")); - EXPECT_EQUAL((ITermData*)&f1.queryEnv.getTerms()[2], util::getTermByLabel(f1.queryEnv, "baz")); - EXPECT_EQUAL((ITermData*)0, util::getTermByLabel(f1.queryEnv, "fox")); - EXPECT_EQUAL((ITermData*)0, util::getTermByLabel(f1.queryEnv, "unknown")); +TEST(UtilsTest, require_that_label_can_be_mapped_to_term) +{ + TermLabelFixture f1; + EXPECT_EQ((ITermData*)&f1.queryEnv.getTerms()[0], getTermByLabel(f1.queryEnv, "foo")); + EXPECT_EQ((ITermData*)0, getTermByLabel(f1.queryEnv, "bar")); + EXPECT_EQ((ITermData*)&f1.queryEnv.getTerms()[2], getTermByLabel(f1.queryEnv, "baz")); + EXPECT_EQ((ITermData*)0, getTermByLabel(f1.queryEnv, "fox")); + EXPECT_EQ((ITermData*)0, getTermByLabel(f1.queryEnv, "unknown")); } template <typename T> -void verifyStrToNum() { - EXPECT_EQUAL(-17, static_cast<long>(strToNum<T>("-17"))); - EXPECT_EQUAL(-1, static_cast<long>(strToNum<T>("-1"))); - EXPECT_EQUAL(0, static_cast<long>(strToNum<T>("0"))); - EXPECT_EQUAL(1, static_cast<long>(strToNum<T>("1"))); - EXPECT_EQUAL(17, static_cast<long>(strToNum<T>("17"))); - EXPECT_EQUAL(0, static_cast<long>(strToNum<T>("0x0"))); - EXPECT_EQUAL(1, static_cast<long>(strToNum<T>("0x1"))); - EXPECT_EQUAL(27, static_cast<long>(strToNum<T>("0x1b"))); +void verifyStrToNum(const std::string& label) { + SCOPED_TRACE(label); + EXPECT_EQ(-17, static_cast<long>(strToNum<T>("-17"))); + EXPECT_EQ(-1, static_cast<long>(strToNum<T>("-1"))); + EXPECT_EQ(0, static_cast<long>(strToNum<T>("0"))); + EXPECT_EQ(1, static_cast<long>(strToNum<T>("1"))); + EXPECT_EQ(17, static_cast<long>(strToNum<T>("17"))); + EXPECT_EQ(0, static_cast<long>(strToNum<T>("0x0"))); + EXPECT_EQ(1, static_cast<long>(strToNum<T>("0x1"))); + EXPECT_EQ(27, static_cast<long>(strToNum<T>("0x1b"))); +} + +TEST(UtilsTest, verify_str2Num) +{ + verifyStrToNum<int8_t>("int8_t"); + verifyStrToNum<int16_t>("int16_t"); + verifyStrToNum<int32_t>("int32_t"); + verifyStrToNum<int64_t>("int64_t"); } -TEST("verify str2Num") { - verifyStrToNum<int8_t>(); - verifyStrToNum<int16_t>(); - verifyStrToNum<int32_t>(); - verifyStrToNum<int64_t>(); +TEST(UtilsTest, lookup_document_frequency) +{ + using OptDF = std::optional<DocumentFrequency>; + IndexEnvironment index_env;; + QueryEnvironment query_env(&index_env); + query_env.getTerms() = std::vector<SimpleTermData>{make_term(0), make_term(5), make_term(6), make_term(10)}; + // Properties not used due to bad unique id + query_env.getProperties().add("vespa.term.0.docfreq", "11"); + query_env.getProperties().add("vespa.term.0.docfreq", "17"); + // Incomplete properties, thus not used + query_env.getProperties().add("vespa.term.6.docfreq", "5"); + // Complete properties + query_env.getProperties().add("vespa.term.10.docfreq", "10"); + query_env.getProperties().add("vespa.term.10.docfreq", "15"); + EXPECT_EQ(OptDF(), lookup_document_frequency(query_env, 0)); // bad unique id + EXPECT_EQ(OptDF(), lookup_document_frequency(query_env, 1)); // missing properties + EXPECT_EQ(OptDF(), lookup_document_frequency(query_env, 2)); // incomplete properties + EXPECT_EQ(OptDF({10, 15}), lookup_document_frequency(query_env, 3)); + EXPECT_EQ(OptDF(), lookup_document_frequency(query_env, 4)); // term not found } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp index 5b3e3c356ac..f7c6b9fd7ed 100644 --- a/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp +++ b/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/indexenvironment.h> diff --git a/searchlib/src/tests/fef/fef_test.cpp b/searchlib/src/tests/fef/fef_test.cpp index 0f2de1665e8..37089cfa5ea 100644 --- a/searchlib/src/tests/fef/fef_test.cpp +++ b/searchlib/src/tests/fef/fef_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/objectstore.h> diff --git a/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp b/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp index 7200566d735..cb1dcc6fb8b 100644 --- a/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp +++ b/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/util/fileheadertk.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/grouping/grouping_test.cpp b/searchlib/src/tests/grouping/grouping_test.cpp index d28ab1d3d66..952f5d2a5db 100644 --- a/searchlib/src/tests/grouping/grouping_test.cpp +++ b/searchlib/src/tests/grouping/grouping_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/extendableattributes.h> @@ -8,6 +8,8 @@ #include <vespa/searchlib/aggregation/hitsaggregationresult.h> #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/aggregation/predicates.h> +#include <vespa/searchlib/aggregation/modifiers.h> +#include <vespa/searchlib/expression/documentfieldnode.h> #include <vespa/searchlib/expression/fixedwidthbucketfunctionnode.h> #include <vespa/searchlib/test/make_attribute_map_lookup_node.h> #include <vespa/searchcommon/common/undefinedvalues.h> @@ -51,7 +53,7 @@ public: add(val); } } - AttrBuilder(const std::string &name) + explicit AttrBuilder(const std::string &name) : _attr(new A(name)), _attrSP(_attr) { @@ -127,19 +129,19 @@ private: ResultBuilder _result; IAttributeContext::UP _attrCtx; - AggregationContext(const AggregationContext &); - AggregationContext &operator=(const AggregationContext &); - public: AggregationContext(); + AggregationContext(const AggregationContext &) = delete; + AggregationContext &operator=(const AggregationContext &) = delete; ~AggregationContext(); ResultBuilder &result() { return _result; } - void add(AttributeVector::SP attr) { + void add(const AttributeVector::SP & attr) { _attrMan.add(attr); } void setup(Grouping &g) { - g.configureStaticStuff(ConfigureStaticParams(_attrCtx.get(), 0)); + g.configureStaticStuff(ConfigureStaticParams(_attrCtx.get(), nullptr)); } + const IAttributeContext & attrCtx() const { return *_attrCtx; } }; AggregationContext::AggregationContext() : _attrMan(), _result(), _attrCtx(_attrMan.createContext()) {} @@ -688,7 +690,7 @@ TEST("testAggregationGroupCapping") EXPECT_TRUE(testAggregation(ctx, request, expect)); } { - AddFunctionNode *add = new AddFunctionNode(); + auto add = std::make_unique<AddFunctionNode>(); add->addArg(MU<AggregationRefNode>(0)); add->appendArg(MU<ConstantNode>(MU<Int64ResultNode>(3))); @@ -697,7 +699,7 @@ TEST("testAggregationGroupCapping") .setLastLevel(1) .addLevel(std::move(GroupingLevel().setMaxGroups(3).setExpression(MU<AttributeNode>("attr")) .addAggregationResult(createAggr<SumAggregationResult>(MU<AttributeNode>("attr"))) - .addOrderBy(ExpressionNode::UP(add), false))); + .addOrderBy(std::move(add), false))); Group expect; expect.addChild(Group().setId(Int64ResultNode(7)).setRank(RawRank(7)) @@ -1826,4 +1828,31 @@ TEST("testAttributeMapLookup") testAggregationSimple(ctx, MaxAggregationResult(), Int64ResultNode(100), "smap{attribute(key2)}.weight"); } +TEST("test that non-attributes are converted to document field nodes") { + AggregationContext ctx; + ctx.add(IntAttrBuilder("attr").sp()); + + Grouping attrRequest; + attrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("attr")))); + aggregation::NonAttribute2DocumentAccessor optional2DocumentAccessor(ctx.attrCtx()); + attrRequest.select(optional2DocumentAccessor, optional2DocumentAccessor); + EXPECT_TRUE(attrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(AttributeNode::classId)); + + Grouping nonAttrRequest; + nonAttrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("non-attr")))); + nonAttrRequest.select(optional2DocumentAccessor, optional2DocumentAccessor); + EXPECT_TRUE(nonAttrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(DocumentFieldNode::classId)); +} + +TEST("test that attributes can be unconditionally converted to document field nodes") { + AggregationContext ctx; + ctx.add(IntAttrBuilder("attr").sp()); + + Grouping attrRequest; + attrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("attr")))); + aggregation::Attribute2DocumentAccessor attr2DocumentAccessor; + attrRequest.select(attr2DocumentAccessor, attr2DocumentAccessor); + EXPECT_TRUE(attrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(DocumentFieldNode::classId)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/grouping/hyperloglog_test.cpp b/searchlib/src/tests/grouping/hyperloglog_test.cpp index 61d81b76b08..281bad55735 100644 --- a/searchlib/src/tests/grouping/hyperloglog_test.cpp +++ b/searchlib/src/tests/grouping/hyperloglog_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("hyperloglog_test"); #include <vespa/searchlib/grouping/hyperloglog.h> #include <vespa/vespalib/objects/nboserializer.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using vespalib::NBOSerializer; using vespalib::nbostream; diff --git a/searchlib/src/tests/grouping/sketch_test.cpp b/searchlib/src/tests/grouping/sketch_test.cpp index e75e47266da..ae1052e9f59 100644 --- a/searchlib/src/tests/grouping/sketch_test.cpp +++ b/searchlib/src/tests/grouping/sketch_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchlib/grouping/sketch.h> #include <vespa/vespalib/objects/nboserializer.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp b/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp index 0c201da48d9..e65c5d812a4 100644 --- a/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp +++ b/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/extendableattributes.h> @@ -133,27 +133,18 @@ AggregationContext::AggregationContext() AggregationContext::~AggregationContext() = default; //----------------------------------------------------------------------------- -class Test : public TestApp +class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { public: + CheckAttributeReferences() : _numrefs(0) { } + int _numrefs; private: - bool testAggregation(AggregationContext &ctx, const Grouping &request, bool useEngine); - void benchmarkIntegerSum(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups); - void benchmarkIntegerCount(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups); - class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate - { - public: - CheckAttributeReferences() : _numrefs(0) { } - int _numrefs; - private: - void execute(vespalib::Identifiable &obj) override { - if (static_cast<AttributeNode &>(obj).getAttribute() != nullptr) { - _numrefs++; - } + void execute(vespalib::Identifiable &obj) override { + if (static_cast<AttributeNode &>(obj).getAttribute() != nullptr) { + _numrefs++; } - virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } - }; - int Main() override; + } + virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } }; //----------------------------------------------------------------------------- @@ -162,9 +153,7 @@ private: * Run the given grouping request and verify that the resulting group * tree matches the expected value. **/ -bool -Test::testAggregation(AggregationContext &ctx, const Grouping &request, bool useEngine) -{ +bool testAggregation(AggregationContext &ctx, const Grouping &request, bool useEngine) { Grouping tmp = request; // create local copy ctx.setup(tmp); if (useEngine) { @@ -183,9 +172,7 @@ Test::testAggregation(AggregationContext &ctx, const Grouping &request, bool use #define MU std::make_unique -void -Test::benchmarkIntegerSum(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups) -{ +void benchmarkIntegerSum(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups) { IntAttrBuilder attrB("attr0"); for (size_t i=0; i < numDocs; i++) { attrB.add(i); @@ -212,9 +199,7 @@ Test::benchmarkIntegerSum(bool useEngine, size_t numDocs, size_t numQueries, int } } -void -Test::benchmarkIntegerCount(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups) -{ +void benchmarkIntegerCount(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups) { IntAttrBuilder attrB("attr0"); for (size_t i=0; i < numDocs; i++) { attrB.add(i); @@ -241,34 +226,31 @@ Test::benchmarkIntegerCount(bool useEngine, size_t numDocs, size_t numQueries, i } } -int -Test::Main() -{ +TEST_MAIN() { size_t numDocs = 1000000; size_t numQueries = 1000; int64_t maxGroups = -1; bool useEngine = true; vespalib::string idType = "int"; vespalib::string aggrType = "sum"; - if (_argc > 1) { - useEngine = (strcmp(_argv[1], "tree") != 0); + if (argc > 1) { + useEngine = (strcmp(argv[1], "tree") != 0); } - if (_argc > 2) { - idType = _argv[2]; + if (argc > 2) { + idType = argv[2]; } - if (_argc > 3) { - aggrType = _argv[3]; + if (argc > 3) { + aggrType = argv[3]; } - if (_argc > 4) { - numDocs = strtol(_argv[4], nullptr, 0); + if (argc > 4) { + numDocs = strtol(argv[4], nullptr, 0); } - if (_argc > 5) { - numQueries = strtol(_argv[5], nullptr, 0); + if (argc > 5) { + numQueries = strtol(argv[5], nullptr, 0); } - if (_argc > 6) { - maxGroups = strtol(_argv[6], nullptr, 0); + if (argc > 6) { + maxGroups = strtol(argv[6], nullptr, 0); } - TEST_INIT("grouping_benchmark"); LOG(info, "sizeof(Group) = %ld", sizeof(Group)); LOG(info, "sizeof(ResultNode::CP) = %ld", sizeof(ResultNode::CP)); LOG(info, "sizeof(RawRank) = %ld", sizeof(RawRank)); @@ -290,7 +272,4 @@ Test::Main() } LOG(info, "rusage = {\n%s\n}", vespalib::RUsage::createSelf(start).toString().c_str()); ASSERT_EQUAL(0, kill(0, SIGPROF)); - TEST_DONE(); } - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/groupingengine/groupingengine_test.cpp b/searchlib/src/tests/groupingengine/groupingengine_test.cpp index e00d2dcdd46..140304e0da9 100644 --- a/searchlib/src/tests/groupingengine/groupingengine_test.cpp +++ b/searchlib/src/tests/groupingengine/groupingengine_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/extendableattributes.h> @@ -131,69 +131,37 @@ AggregationContext::~AggregationContext() {} //----------------------------------------------------------------------------- -class Test : public TestApp +class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { public: - bool testAggregation(AggregationContext &ctx, - const Grouping &request, - const Group &expect); - bool testMerge(const Grouping &a, const Grouping &b, - const Group &expect); - bool testMerge(const Grouping &a, const Grouping &b, const Grouping &c, - const Group &expect); - bool testPrune(const Grouping &a, const Grouping &b, - const Group &expect); - bool testPartialMerge(const Grouping &a, const Grouping &b, - const Group &expect); - void testAggregationSimple(); - void testAggregationLevels(); - void testAggregationMaxGroups(); - void testAggregationGroupOrder(); - void testAggregationGroupRank(); - void testAggregationGroupCapping(); - void testMergeSimpleSum(); - void testMergeLevels(); - void testMergeGroups(); - void testMergeTrees(); - void testPruneSimple(); - void testPruneComplex(); - void testPartialMerging(); - void testCount(); - void testTopN(); - void testFS4HitCollection(); - bool checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket); - bool checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt); - void testFixedWidthBuckets(); - void testThatNanIsConverted(); - void testNanSorting(); - void testGroupingEngineFromRequest(); - int Main() override; + CheckAttributeReferences() : _numrefs(0) { } + int _numrefs; private: - bool verifyEqual(const Group & a, const Group & b); - void testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr); - class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate - { - public: - CheckAttributeReferences() : _numrefs(0) { } - int _numrefs; - private: - virtual void execute(vespalib::Identifiable &obj) override { - if (static_cast<AttributeNode &>(obj).getAttribute() != NULL) { - _numrefs++; - } + virtual void execute(vespalib::Identifiable &obj) override { + if (static_cast<AttributeNode &>(obj).getAttribute() != NULL) { + _numrefs++; } - virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } - }; + } + virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } }; //----------------------------------------------------------------------------- +bool verifyEqual(const Group & a, const Group & b) +{ + bool ok = EXPECT_EQUAL(a.asString(), b.asString()); + if (!ok) { + std::cerr << a.asString() << std::endl << b.asString() << std::endl; + } + return ok; +} + /** * Run the given grouping request and verify that the resulting group * tree matches the expected value. **/ bool -Test::testAggregation(AggregationContext &ctx, +testAggregation(AggregationContext &ctx, const Grouping &request, const Group &expect) { @@ -210,21 +178,12 @@ Test::testAggregation(AggregationContext &ctx, return verifyEqual(*result, expect); } -bool Test::verifyEqual(const Group & a, const Group & b) -{ - bool ok = EXPECT_EQUAL(a.asString(), b.asString()); - if (!ok) { - std::cerr << a.asString() << std::endl << b.asString() << std::endl; - } - return ok; -} - /** * Merge the given grouping requests and verify that the resulting * group tree matches the expected value. **/ bool -Test::testMerge(const Grouping &a, const Grouping &b, +testMerge(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy @@ -249,7 +208,7 @@ Test::testMerge(const Grouping &a, const Grouping &b, * group tree matches the expected value. **/ bool -Test::testPrune(const Grouping &a, const Grouping &b, +testPrune(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy @@ -266,7 +225,7 @@ Test::testPrune(const Grouping &a, const Grouping &b, * partial request is correct. **/ bool -Test::testPartialMerge(const Grouping &a, const Grouping &b, +testPartialMerge(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy @@ -283,7 +242,7 @@ Test::testPartialMerge(const Grouping &a, const Grouping &b, * group tree matches the expected value. **/ bool -Test::testMerge(const Grouping &a, const Grouping &b, const Grouping &c, +testMerge(const Grouping &a, const Grouping &b, const Grouping &c, const Group &expect) { Grouping tmp = a; // create local copy @@ -298,28 +257,9 @@ Test::testMerge(const Grouping &a, const Grouping &b, const Grouping &c, //----------------------------------------------------------------------------- -/** - * Test collecting the sum of the values from a single attribute - * vector directly into the root node. Consider this a smoke test. - **/ -void -Test::testAggregationSimple() -{ - AggregationContext ctx; - ctx.result().add(0).add(1).add(2); - ctx.add(IntAttrBuilder("int").add(3).add(7).add(15).sp()); - ctx.add(FloatAttrBuilder("float").add(3).add(7).add(15).sp()); - ctx.add(StringAttrBuilder("string").add("3").add("7").add("15").sp()); - - char strsum[3] = {(char)-101, '5', 0}; - testAggregationSimpleSum(ctx, SumAggregationResult(), Int64ResultNode(25), FloatResultNode(25), StringResultNode(strsum)); - testAggregationSimpleSum(ctx, MinAggregationResult(), Int64ResultNode(3), FloatResultNode(3), StringResultNode("15")); - testAggregationSimpleSum(ctx, MaxAggregationResult(), Int64ResultNode(15), FloatResultNode(15), StringResultNode("7")); -} - #define MU std::make_unique -void Test::testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr) +void testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr) { ExpressionNode::CP clone(aggr); Grouping request; @@ -337,6 +277,25 @@ void Test::testAggregationSimpleSum(AggregationContext & ctx, const AggregationR EXPECT_TRUE(testAggregation(ctx, request, expect)); } +/** + * Test collecting the sum of the values from a single attribute + * vector directly into the root node. Consider this a smoke test. + **/ +void +testAggregationSimple() +{ + AggregationContext ctx; + ctx.result().add(0).add(1).add(2); + ctx.add(IntAttrBuilder("int").add(3).add(7).add(15).sp()); + ctx.add(FloatAttrBuilder("float").add(3).add(7).add(15).sp()); + ctx.add(StringAttrBuilder("string").add("3").add("7").add("15").sp()); + + char strsum[3] = {(char)-101, '5', 0}; + testAggregationSimpleSum(ctx, SumAggregationResult(), Int64ResultNode(25), FloatResultNode(25), StringResultNode(strsum)); + testAggregationSimpleSum(ctx, MinAggregationResult(), Int64ResultNode(3), FloatResultNode(3), StringResultNode("15")); + testAggregationSimpleSum(ctx, MaxAggregationResult(), Int64ResultNode(15), FloatResultNode(15), StringResultNode("7")); +} + GroupingLevel createGL(ExpressionNode::UP expr, ExpressionNode::UP result) { GroupingLevel l; @@ -374,7 +333,7 @@ createGL(size_t maxGroups, ExpressionNode::UP expr, ExpressionNode::UP result) { * lastLevel parameters. **/ void -Test::testAggregationLevels() +testAggregationLevels() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr0").add(10).add(10).sp()); @@ -496,7 +455,7 @@ Test::testAggregationLevels() * indicated by the maxgroups parameter. **/ void -Test::testAggregationMaxGroups() +testAggregationMaxGroups() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr").add(5).add(10).add(15).sp()); @@ -547,7 +506,7 @@ Test::testAggregationMaxGroups() * Verify that groups are sorted by group id **/ void -Test::testAggregationGroupOrder() +testAggregationGroupOrder() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr").add(10).add(25).add(35).add(5).add(20).add(15).add(30).sp()); @@ -574,7 +533,7 @@ Test::testAggregationGroupOrder() * Verify that groups are tagged with the appropriate rank value. **/ void -Test::testAggregationGroupRank() +testAggregationGroupRank() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr") @@ -624,7 +583,7 @@ createNumAggr(NumericResultNode::UP r, ExpressionNode::UP e) { } void -Test::testAggregationGroupCapping() +testAggregationGroupCapping() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr") @@ -748,7 +707,7 @@ Test::testAggregationGroupCapping() * smoke test. **/ void -Test::testMergeSimpleSum() +testMergeSimpleSum() { Grouping a = Grouping() .setRoot(Group() @@ -777,7 +736,7 @@ Test::testMergeSimpleSum() * Verify that frozen levels are not touched during merge. **/ void -Test::testMergeLevels() +testMergeLevels() { Grouping request = Grouping() .addLevel(createGL(MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -957,7 +916,7 @@ Test::testMergeLevels() * and that they are sorted by group id. **/ void -Test::testMergeGroups() +testMergeGroups() { Grouping request = Grouping().addLevel(createGL(MU<AttributeNode>("attr"))); @@ -1018,7 +977,7 @@ Test::testMergeGroups() * end result is as expected. **/ void -Test::testMergeTrees() +testMergeTrees() { Grouping request; request.addLevel(createGL(3, MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -1283,7 +1242,7 @@ Test::testMergeTrees() } void -Test::testPruneComplex() +testPruneComplex() { { // First level Group baseTree = Group() @@ -1423,7 +1382,7 @@ Test::testPruneComplex() * results. **/ void -Test::testPartialMerging() +testPartialMerging() { Grouping baseRequest; baseRequest.addLevel(createGL(MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -1588,7 +1547,7 @@ Test::testPartialMerging() * Test that pruning a simple grouping tree works. **/ void -Test::testPruneSimple() +testPruneSimple() { { Grouping request; @@ -1614,7 +1573,7 @@ Test::testPruneSimple() * that we init, calculate and ignore. **/ void -Test::testTopN() +testTopN() { AggregationContext ctx; ctx.result().add(0).add(1).add(2); @@ -1655,7 +1614,7 @@ Test::testTopN() * that we init, calculate and ignore. **/ void -Test::testCount() +testCount() { AggregationContext ctx; ctx.result().add(0).add(1).add(2); @@ -1676,7 +1635,7 @@ Test::testCount() //----------------------------------------------------------------------------- bool -Test::checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt) +checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt) { CountFS4Hits pop; Grouping tmp = g; @@ -1685,7 +1644,7 @@ Test::checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt) } void -Test::testFS4HitCollection() +testFS4HitCollection() { { // aggregation AggregationContext ctx; @@ -1801,7 +1760,7 @@ Test::testFS4HitCollection() } bool -Test::checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket) +checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket) { AggregationContext ctx; ctx.result().add(0); @@ -1821,7 +1780,7 @@ Test::checkBucket(const NumericResultNode &width, const NumericResultNode &value } void -Test::testFixedWidthBuckets() +testFixedWidthBuckets() { using Int = Int64ResultNode; using Float = FloatResultNode; @@ -1879,7 +1838,7 @@ Test::testFixedWidthBuckets() void -Test::testNanSorting() +testNanSorting() { // Attempt at reproducing issue with segfault when setting NaN value. Not // successful yet, so no point in running test. @@ -1913,7 +1872,7 @@ Test::testNanSorting() } void -Test::testThatNanIsConverted() +testThatNanIsConverted() { Group g; double myNan = std::sqrt(-1); @@ -1923,7 +1882,7 @@ Test::testThatNanIsConverted() } void -Test::testGroupingEngineFromRequest() +testGroupingEngineFromRequest() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr0").add(10).add(10).sp()); @@ -1943,19 +1902,8 @@ Test::testGroupingEngineFromRequest() //----------------------------------------------------------------------------- -struct RunDiff { ~RunDiff() { - [[maybe_unused]] int system_result = system("diff -u lhs.out rhs.out > diff.txt"); -}}; - -//----------------------------------------------------------------------------- - -int -Test::Main() -{ - RunDiff runDiff; - (void) runDiff; +TEST_MAIN() { TEST_DEBUG("lhs.out", "rhs.out"); - TEST_INIT("groupingengine_test"); testGroupingEngineFromRequest(); testAggregationSimple(); testAggregationLevels(); @@ -1978,7 +1926,6 @@ Test::Main() testTopN(); testThatNanIsConverted(); testNanSorting(); - TEST_DONE(); + TEST_DEBUG_STOP(); + [[maybe_unused]] int system_result = system("diff -u lhs.out rhs.out > diff.txt"); } - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/predicate/.gitignore b/searchlib/src/tests/predicate/.gitignore index eea4d347d05..56e1330505c 100644 --- a/searchlib/src/tests/predicate/.gitignore +++ b/searchlib/src/tests/predicate/.gitignore @@ -1,13 +1 @@ -searchlib_document_features_store_test_app -searchlib_predicate_bounds_posting_list_test_app -searchlib_predicate_index_test_app -searchlib_predicate_interval_posting_list_test_app -searchlib_predicate_interval_store_test_app -searchlib_predicate_range_term_expander_test_app -searchlib_predicate_ref_cache_test_app -searchlib_predicate_tree_analyzer_test_app -searchlib_predicate_tree_annotator_test_app -searchlib_predicate_zero_constraint_posting_list_test_app -searchlib_predicate_zstar_compressed_posting_list_test_app -searchlib_simple_index_test_app -searchlib_tree_crumbs_test_app +searchlib_predicate_vespa_test_app diff --git a/searchlib/src/tests/predicate/CMakeLists.txt b/searchlib/src/tests/predicate/CMakeLists.txt index d33e5617908..4ae06af1886 100644 --- a/searchlib/src/tests/predicate/CMakeLists.txt +++ b/searchlib/src/tests/predicate/CMakeLists.txt @@ -1,92 +1,21 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_predicate_index_test_app TEST +vespa_add_executable(searchlib_predicate_vespa_test_app TEST SOURCES + vespa_testrunner.cpp predicate_index_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_index_test_app COMMAND searchlib_predicate_index_test_app) -vespa_add_executable(searchlib_simple_index_test_app TEST - SOURCES simple_index_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_simple_index_test_app COMMAND searchlib_simple_index_test_app) -vespa_add_executable(searchlib_tree_crumbs_test_app TEST - SOURCES tree_crumbs_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_tree_crumbs_test_app COMMAND searchlib_tree_crumbs_test_app) -vespa_add_executable(searchlib_predicate_tree_analyzer_test_app TEST - SOURCES predicate_tree_analyzer_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_tree_analyzer_test_app COMMAND searchlib_predicate_tree_analyzer_test_app) -vespa_add_executable(searchlib_predicate_tree_annotator_test_app TEST - SOURCES predicate_tree_annotator_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_tree_annotator_test_app COMMAND searchlib_predicate_tree_annotator_test_app) -vespa_add_executable(searchlib_predicate_interval_store_test_app TEST - SOURCES predicate_interval_store_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_interval_store_test_app COMMAND searchlib_predicate_interval_store_test_app) -vespa_add_executable(searchlib_document_features_store_test_app TEST - SOURCES document_features_store_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_document_features_store_test_app COMMAND searchlib_document_features_store_test_app) -vespa_add_executable(searchlib_predicate_ref_cache_test_app TEST - SOURCES predicate_ref_cache_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_ref_cache_test_app COMMAND searchlib_predicate_ref_cache_test_app) -vespa_add_executable(searchlib_predicate_interval_posting_list_test_app TEST - SOURCES predicate_interval_posting_list_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_interval_posting_list_test_app COMMAND searchlib_predicate_interval_posting_list_test_app) -vespa_add_executable(searchlib_predicate_bounds_posting_list_test_app TEST - SOURCES predicate_bounds_posting_list_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_bounds_posting_list_test_app COMMAND searchlib_predicate_bounds_posting_list_test_app) -vespa_add_executable(searchlib_predicate_zero_constraint_posting_list_test_app TEST - SOURCES predicate_zero_constraint_posting_list_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_zero_constraint_posting_list_test_app COMMAND searchlib_predicate_zero_constraint_posting_list_test_app) -vespa_add_executable(searchlib_predicate_zstar_compressed_posting_list_test_app TEST - SOURCES predicate_zstar_compressed_posting_list_test.cpp - DEPENDS - vespa_searchlib -) -vespa_add_test(NAME searchlib_predicate_zstar_compressed_posting_list_test_app COMMAND searchlib_predicate_zstar_compressed_posting_list_test_app) -vespa_add_executable(searchlib_predicate_range_term_expander_test_app TEST - SOURCES predicate_range_term_expander_test.cpp DEPENDS vespa_searchlib ) -vespa_add_test(NAME searchlib_predicate_range_term_expander_test_app COMMAND searchlib_predicate_range_term_expander_test_app) +vespa_add_test(NAME searchlib_predicate_vespa_test_app COMMAND searchlib_predicate_vespa_test_app) diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp index 3d479a6d39a..01eaa75a71a 100644 --- a/searchlib/src/tests/predicate/document_features_store_test.cpp +++ b/searchlib/src/tests/predicate/document_features_store_test.cpp @@ -7,11 +7,7 @@ #include <vespa/searchlib/predicate/predicate_index.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/predicate/predicate_hash.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <string> - -#include <vespa/log/log.h> -LOG_SETUP("document_features_store_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -233,5 +229,3 @@ TEST("require that serialization cleans up wordstore") { } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp index 228b0eb242d..be7f4516bb2 100644 --- a/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp @@ -7,10 +7,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("predicate_bounds_posting_list_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -106,5 +103,3 @@ TEST("require that bounds posting list checks bounds.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_index_test.cpp b/searchlib/src/tests/predicate/predicate_index_test.cpp index 461fa46d4da..d0af12f93c7 100644 --- a/searchlib/src/tests/predicate/predicate_index_test.cpp +++ b/searchlib/src/tests/predicate/predicate_index_test.cpp @@ -5,16 +5,13 @@ #include <vespa/searchlib/predicate/simple_index.hpp> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/util/data_buffer_writer.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/log/log.h> -LOG_SETUP("predicate_index_test"); - using namespace search; using namespace search::predicate; using std::make_pair; @@ -454,5 +451,3 @@ TEST("require that predicate index saver protected by a generation guard observe } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp index ab49e28bb96..098c0238412 100644 --- a/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp @@ -7,10 +7,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("predicate_interval_posting_list_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -79,5 +76,3 @@ TEST("require that posting list can iterate.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp index 819563f64b8..d8c9691d421 100644 --- a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp +++ b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp @@ -1,13 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicate_interval_store. -#include <vespa/log/log.h> -LOG_SETUP("predicate_interval_store_test"); - #include <vespa/searchlib/predicate/predicate_interval_store.h> #include <vespa/searchlib/predicate/predicate_index.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vector> using namespace search; @@ -147,5 +144,3 @@ TEST("require that interval refs are reused for identical data.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp b/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp index 162829be5a3..2dce3c50b0a 100644 --- a/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp +++ b/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp @@ -1,12 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicate_range_term_expander. -#include <vespa/log/log.h> -LOG_SETUP("predicate_range_term_expander_test"); - #include <vespa/searchlib/predicate/predicate_range_term_expander.h> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using search::predicate::PredicateRangeTermExpander; using std::vector; @@ -328,5 +325,3 @@ TEST("require that search close to max uneven upper bound is sensible") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp b/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp index c8327033a8c..f62f3f807c5 100644 --- a/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp +++ b/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp @@ -1,11 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicate_ref_cache. -#include <vespa/log/log.h> -LOG_SETUP("predicate_ref_cache_test"); - #include <vespa/searchlib/predicate/predicate_ref_cache.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vector> using namespace search; @@ -101,5 +98,3 @@ TEST("require that cache handles large entries") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp b/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp index c766aa70bad..73a236aa443 100644 --- a/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp +++ b/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp @@ -1,13 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for PredicateTreeAnalyzer. -#include <vespa/log/log.h> -LOG_SETUP("PredicateTreeAnalyzer_test"); - #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_slime_builder.h> #include <vespa/searchlib/predicate/predicate_tree_analyzer.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::PredicateSlimeBuilder; using namespace search; @@ -152,5 +149,3 @@ TEST("require that multilevel AND stores sizes") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp b/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp index 82629527b4d..4d71f585910 100644 --- a/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp +++ b/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp @@ -7,12 +7,9 @@ #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/predicate/predicate_hash.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <sstream> -#include <vespa/log/log.h> -LOG_SETUP("PredicateTreeAnnotator_test"); - using document::Predicate; using std::ostringstream; using std::pair; @@ -403,5 +400,3 @@ TEST("require that open range works") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp index 9e9ac45f1ae..5907dce72ba 100644 --- a/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp @@ -4,10 +4,7 @@ #include <vespa/searchlib/predicate/predicate_zero_constraint_posting_list.h> #include <vespa/searchlib/predicate/predicate_index.h> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("predicate_zero_constraint_posting_list_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -54,5 +51,3 @@ TEST("require that posting list can iterate.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp index 0e99379568d..20cd0809473 100644 --- a/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp @@ -6,10 +6,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("predicate_zstar_compressed_posting_list_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -93,5 +90,3 @@ TEST("require that posting list can iterate.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/simple_index_test.cpp b/searchlib/src/tests/predicate/simple_index_test.cpp index 9b99ff8e809..964ff67bd3a 100644 --- a/searchlib/src/tests/predicate/simple_index_test.cpp +++ b/searchlib/src/tests/predicate/simple_index_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/predicate/simple_index_saver.hpp> #include <vespa/searchlib/predicate/nbo_write.h> #include <vespa/searchlib/util/data_buffer_writer.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreeroot.hpp> @@ -16,9 +16,6 @@ #include <vespa/vespalib/util/rcuvector.hpp> #include <map> -#include <vespa/log/log.h> -LOG_SETUP("simple_index_test"); - using namespace search; using namespace search::predicate; using vespalib::GenerationHolder; @@ -342,5 +339,3 @@ TEST_F("require that vector contains correct postings", Fixture) { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/tree_crumbs_test.cpp b/searchlib/src/tests/predicate/tree_crumbs_test.cpp index f5ff488fdc0..76bfd02ee50 100644 --- a/searchlib/src/tests/predicate/tree_crumbs_test.cpp +++ b/searchlib/src/tests/predicate/tree_crumbs_test.cpp @@ -1,11 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for TreeCrumbs. -#include <vespa/log/log.h> -LOG_SETUP("TreeCrumbs_test"); - #include <vespa/searchlib/predicate/tree_crumbs.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace search::predicate; @@ -60,5 +57,3 @@ TEST("require that crumbs can set custom initial char") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/vespa_testrunner.cpp b/searchlib/src/tests/predicate/vespa_testrunner.cpp new file mode 100644 index 00000000000..d812605710e --- /dev/null +++ b/searchlib/src/tests/predicate/vespa_testrunner.cpp @@ -0,0 +1,8 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Unit tests for predicate_index. +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/log/log.h> +LOG_SETUP("predicate_test"); + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/query/customtypevisitor_test.cpp b/searchlib/src/tests/query/customtypevisitor_test.cpp index fb89f2ef061..702bed6e50e 100644 --- a/searchlib/src/tests/query/customtypevisitor_test.cpp +++ b/searchlib/src/tests/query/customtypevisitor_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/string_term_vector.h> #include <vespa/searchlib/query/tree/termnodes.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("customtypevisitor_test"); diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index bfce382b684..e1265680f7b 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/tree/string_term_vector.h> #include <vespa/searchlib/query/tree/termnodes.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("query_visitor_test"); diff --git a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp index 29ef179385d..4ae94c17804 100644 --- a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp +++ b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/util/rawbuf.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("stackdumpquerycreator_test"); diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index f7745da174c..77d0099afdb 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mysearch.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/flow.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index 490f221d1d8..a8707bb6f7e 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mysearch.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/flow.h> diff --git a/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp b/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp index d9b7d5b3192..fda2e84402a 100644 --- a/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp +++ b/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/get_weight_from_node.h> diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp index 95e80cd08b8..ddb9cab18ab 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/multibitvectoriterator.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/common/bitvectoriterator.h> @@ -17,15 +17,11 @@ using namespace search; //----------------------------------------------------------------------------- -class Test : public vespalib::TestApp +struct Fixture { -public: - ~Test() {} void benchmark(); - int Main() override; template <typename T> void testSearch(bool strict); -private: void searchAndCompare(SearchIterator::UP s, uint32_t docIdLimit); void setup(); std::vector< BitVector::UP > _bvs; @@ -35,9 +31,22 @@ private: bool _optimize; vespalib::string _type; std::vector<int> _fillLimits; + + Fixture(int argc, char **argv) { + _type = argv[1]; + _strict = vespalib::string(argv[2]) == vespalib::string("strict"); + _optimize = vespalib::string(argv[3]) == vespalib::string("optimize"); + _numSearch = strtoul(argv[4], NULL, 0); + _numDocs = strtoul(argv[5], NULL, 0); + for (int i(6); i < argc; i++) { + _fillLimits.push_back((RAND_MAX/100) * strtoul(argv[i], NULL, 0)); + } + } + ~Fixture(); }; +Fixture::~Fixture() = default; -void Test::setup() +void Fixture::setup() { for(size_t i(0); i < _fillLimits.size(); i++) { _bvs.push_back(BitVector::create(_numDocs)); @@ -76,7 +85,7 @@ seek(SearchIterator & s, uint32_t docIdLimit) } void -Test::benchmark() +Fixture::benchmark() { if (_type == "and") { LOG(info, "Testing 'and'"); @@ -93,7 +102,7 @@ Test::benchmark() template <typename T> void -Test::testSearch(bool strict) +Fixture::testSearch(bool strict) { TermFieldMatchData tfmd; MultiSearch::Children andd; @@ -109,29 +118,15 @@ Test::testSearch(bool strict) LOG(info, "Found %ld hits", h.size()); } -int -Test::Main() -{ - TEST_INIT("multibitvectoriterator_benchmark"); - if (_argc < 6) { - LOG(info, "%s <'and/or'> <'strict/no-strict'> <'optimize/no-optimize> <numsearch> <numdocs> <fill 1> [<fill N>]", _argv[0]); - return -1; - } - _type = _argv[1]; - _strict = vespalib::string(_argv[2]) == vespalib::string("strict"); - _optimize = vespalib::string(_argv[3]) == vespalib::string("optimize"); - _numSearch = strtoul(_argv[4], NULL, 0); - _numDocs = strtoul(_argv[5], NULL, 0); - for (int i(6); i < _argc; i++) { - _fillLimits.push_back((RAND_MAX/100) * strtoul(_argv[i], NULL, 0)); +TEST_MAIN() { + if (argc < 6) { + LOG(info, "%s <'and/or'> <'strict/no-strict'> <'optimize/no-optimize> <numsearch> <numdocs> <fill 1> [<fill N>]", argv[0]); + exit(1); } - LOG(info, "Start setup of '%s' isearch with %ld vectors with %d documents", _type.c_str(), _fillLimits.size(), _numDocs); - setup(); + Fixture fixture(argc, argv); + LOG(info, "Start setup of '%s' isearch with %ld vectors with %d documents", fixture._type.c_str(), fixture._fillLimits.size(), fixture._numDocs); + fixture.setup(); LOG(info, "Start benchmark"); - benchmark(); + fixture.benchmark(); LOG(info, "Done benchmark"); - TEST_FLUSH(); - TEST_DONE(); } - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp index ffa2905ce0e..1ae6cff447e 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp @@ -11,7 +11,7 @@ #include <vespa/searchlib/queryeval/field_spec.h> #include <vespa/searchlib/queryeval/predicate_blueprint.h> #include <vespa/searchlib/predicate/predicate_hash.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("predicate_blueprint_test"); diff --git a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp index a69b4c7a45d..b02df263f82 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("predicate_search_test"); #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/queryeval/predicate_search.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/arraysize.h> using search::fef::TermFieldMatchData; diff --git a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp index 3e779bdca14..184d21fcaae 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp +++ b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp @@ -10,7 +10,7 @@ #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/weight.h> #include <vespa/vespalib/util/testclock.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("simple_phrase_test"); @@ -46,46 +46,6 @@ struct MyTerm : public search::queryeval::SimpleLeafBlueprint { } }; -class Test : public vespalib::TestApp { - void requireThatIteratorFindsSimplePhrase(bool useBlueprint); - void requireThatIteratorFindsLongPhrase(bool useBlueprint); - void requireThatStrictIteratorFindsNextMatch(bool useBlueprint); - void requireThatPhrasesAreUnpacked(bool useBlueprint, bool unpack_normal_features, bool unpack_interleaved_features); - void requireThatTermsCanBeEvaluatedInPriorityOrder(); - void requireThatBlueprintExposesFieldWithEstimate(); - void requireThatBlueprintForcesPositionDataOnChildren(); - -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("phrasesearch_test"); - - TEST_DO(requireThatIteratorFindsSimplePhrase(false)); - TEST_DO(requireThatIteratorFindsLongPhrase(false)); - TEST_DO(requireThatStrictIteratorFindsNextMatch(false)); - TEST_DO(requireThatPhrasesAreUnpacked(false, true, false)); - TEST_DO(requireThatPhrasesAreUnpacked(false, true, true)); - TEST_DO(requireThatPhrasesAreUnpacked(false, false, false)); - TEST_DO(requireThatPhrasesAreUnpacked(false, false, true)); - TEST_DO(requireThatTermsCanBeEvaluatedInPriorityOrder()); - - TEST_DO(requireThatIteratorFindsSimplePhrase(true)); - TEST_DO(requireThatIteratorFindsLongPhrase(true)); - TEST_DO(requireThatStrictIteratorFindsNextMatch(true)); - TEST_DO(requireThatPhrasesAreUnpacked(true, true, false)); - TEST_DO(requireThatPhrasesAreUnpacked(true, true, true)); - TEST_DO(requireThatPhrasesAreUnpacked(true, false, false)); - TEST_DO(requireThatPhrasesAreUnpacked(true, false, true)); - TEST_DO(requireThatBlueprintExposesFieldWithEstimate()); - TEST_DO(requireThatBlueprintForcesPositionDataOnChildren()); - - TEST_DONE(); -} - const string field = "field"; const uint32_t fieldId = 1; const uint32_t doc_match = 42; @@ -198,73 +158,85 @@ PhraseSearchTest::PhraseSearchTest(bool expiredDoom) {} PhraseSearchTest::~PhraseSearchTest() = default; -void Test::requireThatIteratorFindsSimplePhrase(bool useBlueprint) { - PhraseSearchTest test; - test.addTerm("foo", 0).addTerm("bar", 1); +TEST("requireThatIteratorFindsSimplePhrase") { + for (bool useBlueprint: {false, true}) { + PhraseSearchTest test; + test.addTerm("foo", 0).addTerm("bar", 1); - test.fetchPostings(useBlueprint); - unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_TRUE(search->seek(doc_match)); - EXPECT_TRUE(!search->seek(doc_no_match)); + test.fetchPostings(useBlueprint); + unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_TRUE(search->seek(doc_match)); + EXPECT_TRUE(!search->seek(doc_no_match)); + } } -void Test::requireThatIteratorFindsLongPhrase(bool useBlueprint) { - PhraseSearchTest test; - test.addTerm("foo", 0).addTerm("bar", 0).addTerm("baz", 0) - .addTerm("qux", 1); - - test.fetchPostings(useBlueprint); - unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_TRUE(search->seek(doc_match)); - EXPECT_TRUE(!search->seek(doc_no_match)); +TEST("requireThatIteratorFindsLongPhrase") { + for (bool useBlueprint: {false, true}) { + PhraseSearchTest test; + test.addTerm("foo", 0).addTerm("bar", 0).addTerm("baz", 0) + .addTerm("qux", 1); + + test.fetchPostings(useBlueprint); + unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_TRUE(search->seek(doc_match)); + EXPECT_TRUE(!search->seek(doc_no_match)); + } } -void Test::requireThatStrictIteratorFindsNextMatch(bool useBlueprint) { - PhraseSearchTest test; - test.setStrict(true); - test.addTerm("foo", 0).addTerm("bar", 1); - - test.fetchPostings(useBlueprint); - unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_EQUAL(doc_match, search->getDocId()); - EXPECT_TRUE(!search->seek(doc_no_match)); - EXPECT_TRUE(search->isAtEnd()); +TEST("requireThatStrictIteratorFindsNextMatch") { + for (bool useBlueprint: {false, true}) { + PhraseSearchTest test; + test.setStrict(true); + test.addTerm("foo", 0).addTerm("bar", 1); + + test.fetchPostings(useBlueprint); + unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_EQUAL(doc_match, search->getDocId()); + EXPECT_TRUE(!search->seek(doc_no_match)); + EXPECT_TRUE(search->isAtEnd()); + } } -void Test::requireThatPhrasesAreUnpacked(bool useBlueprint, bool unpack_normal_features, bool unpack_interleaved_features) { - PhraseSearchTest test; - test.addTerm("foo", FakeResult() - .doc(doc_match).pos(1).pos(11).pos(21).field_length(30).num_occs(3)); - test.addTerm("bar", FakeResult() - .doc(doc_match).pos(2).pos(16).pos(22).field_length(30).num_occs(3)); - test.writable_term_field_match_data().setNeedNormalFeatures(unpack_normal_features); - test.writable_term_field_match_data().setNeedInterleavedFeatures(unpack_interleaved_features); - test.fetchPostings(useBlueprint); - unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); - EXPECT_TRUE(search->seek(doc_match)); - search->unpack(doc_match); - - EXPECT_EQUAL(doc_match, test.tmd().getDocId()); - if (unpack_normal_features) { - EXPECT_EQUAL(2, std::distance(test.tmd().begin(), test.tmd().end())); - EXPECT_EQUAL(1u, test.tmd().begin()->getPosition()); - EXPECT_EQUAL(21u, (test.tmd().begin() + 1)->getPosition()); - } else { - EXPECT_EQUAL(0, std::distance(test.tmd().begin(), test.tmd().end())); - } - if (unpack_interleaved_features) { - EXPECT_EQUAL(2u, test.tmd().getNumOccs()); - EXPECT_EQUAL(30u, test.tmd().getFieldLength()); - } else { - EXPECT_EQUAL(0u, test.tmd().getNumOccs()); - EXPECT_EQUAL(0u, test.tmd().getFieldLength()); +TEST("requireThatPhrasesAreUnpacked") { + for (bool useBlueprint: {false, true}) { + for (bool unpack_normal_features: {false, true}) { + for (bool unpack_interleaved_features: {false, true}) { + PhraseSearchTest test; + test.addTerm("foo", FakeResult() + .doc(doc_match).pos(1).pos(11).pos(21).field_length(30).num_occs(3)); + test.addTerm("bar", FakeResult() + .doc(doc_match).pos(2).pos(16).pos(22).field_length(30).num_occs(3)); + test.writable_term_field_match_data().setNeedNormalFeatures(unpack_normal_features); + test.writable_term_field_match_data().setNeedInterleavedFeatures(unpack_interleaved_features); + test.fetchPostings(useBlueprint); + unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); + EXPECT_TRUE(search->seek(doc_match)); + search->unpack(doc_match); + + EXPECT_EQUAL(doc_match, test.tmd().getDocId()); + if (unpack_normal_features) { + EXPECT_EQUAL(2, std::distance(test.tmd().begin(), test.tmd().end())); + EXPECT_EQUAL(1u, test.tmd().begin()->getPosition()); + EXPECT_EQUAL(21u, (test.tmd().begin() + 1)->getPosition()); + } else { + EXPECT_EQUAL(0, std::distance(test.tmd().begin(), test.tmd().end())); + } + if (unpack_interleaved_features) { + EXPECT_EQUAL(2u, test.tmd().getNumOccs()); + EXPECT_EQUAL(30u, test.tmd().getFieldLength()); + } else { + EXPECT_EQUAL(0u, test.tmd().getNumOccs()); + EXPECT_EQUAL(0u, test.tmd().getFieldLength()); + } + } + } } } -void Test::requireThatTermsCanBeEvaluatedInPriorityOrder() { +TEST("requireThatTermsCanBeEvaluatedInPriorityOrder") { vector<uint32_t> order; order.push_back(2); order.push_back(0); @@ -280,9 +252,7 @@ void Test::requireThatTermsCanBeEvaluatedInPriorityOrder() { EXPECT_TRUE(!search->seek(doc_no_match)); } -void -Test::requireThatBlueprintExposesFieldWithEstimate() -{ +TEST("requireThatBlueprintExposesFieldWithEstimate") { FieldSpec f("foo", 1, 1); SimplePhraseBlueprint phrase(f, false); ASSERT_TRUE(phrase.getState().numFields() == 1); @@ -305,9 +275,7 @@ Test::requireThatBlueprintExposesFieldWithEstimate() EXPECT_EQUAL(5u, phrase.getState().estimate().estHits); } -void -Test::requireThatBlueprintForcesPositionDataOnChildren() -{ +TEST("requireThatBlueprintForcesPositionDataOnChildren") { FieldSpec f("foo", 1, 1, true); SimplePhraseBlueprint phrase(f, false); EXPECT_TRUE(f.isFilter()); @@ -316,4 +284,4 @@ Test::requireThatBlueprintForcesPositionDataOnChildren() } // namespace -TEST_APPHOOK(Test); +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/sort/sort_test.cpp b/searchlib/src/tests/sort/sort_test.cpp index f6d24d2889c..f0047e6fe0e 100644 --- a/searchlib/src/tests/sort/sort_test.cpp +++ b/searchlib/src/tests/sort/sort_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/common/sortspec.h> #include <vespa/searchlib/common/converters.h> diff --git a/searchlib/src/tests/sort/sortbenchmark.cpp b/searchlib/src/tests/sort/sortbenchmark.cpp index 3a93e359efc..2f18fcfff8c 100644 --- a/searchlib/src/tests/sort/sortbenchmark.cpp +++ b/searchlib/src/tests/sort/sortbenchmark.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/sort.h> #include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/buffer.h> @@ -8,21 +8,15 @@ using vespalib::Array; using vespalib::ConstBufferRef; -class Test : public vespalib::TestApp -{ -public: +struct Test { using V = std::vector<uint32_t>; std::vector< std::vector<uint32_t> > _data; - int Main() override; void generateVectors(size_t numVectors, size_t values); V merge(); void twoWayMerge(); V cat() const; - Test(); - ~Test() override; + ~Test(); }; - -Test::Test() = default; Test::~Test() = default; void @@ -82,31 +76,27 @@ Test::cat() const return c; } -TEST_APPHOOK(Test); - -int Test::Main() -{ - TEST_INIT("sortbenchmark"); +TEST_MAIN() { size_t numVectors(11); size_t values(10000000); vespalib::string type("radix"); - if (_argc > 1) { - values = strtol(_argv[1], NULL, 0); - if (_argc > 2) { - numVectors = strtol(_argv[2], NULL, 0); - if (_argc > 2) { - type = _argv[3]; + if (argc > 1) { + values = strtol(argv[1], NULL, 0); + if (argc > 2) { + numVectors = strtol(argv[2], NULL, 0); + if (argc > 2) { + type = argv[3]; } } } - + Test test; printf("Start with %ld vectors with %ld values and type '%s'(radix, qsort, merge)\n", numVectors, values, type.c_str()); - generateVectors(numVectors, values); + test.generateVectors(numVectors, values); printf("Start cat\n"); - V v = cat(); + auto v = test.cat(); printf("Cat %ld values\n", v.size()); if (type == "merge") { - V m = merge(); + auto m = test.merge(); printf("Merged %ld values\n", m.size()); } else if (type == "qsort") { std::sort(v.begin(), v.end()); @@ -116,6 +106,4 @@ int Test::Main() S(&v[0], v.size()); printf("sorted %ld value with radix::sort\n", v.size()); } - - TEST_DONE(); } diff --git a/searchlib/src/tests/sortspec/multilevelsort_test.cpp b/searchlib/src/tests/sortspec/multilevelsort_test.cpp index f3bf363645e..86e8a932510 100644 --- a/searchlib/src/tests/sortspec/multilevelsort_test.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchlib/attribute/attributemanager.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <type_traits> #include <cinttypes> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/transactionlog/chunks_test.cpp b/searchlib/src/tests/transactionlog/chunks_test.cpp index 76045786895..e057e853d0d 100644 --- a/searchlib/src/tests/transactionlog/chunks_test.cpp +++ b/searchlib/src/tests/transactionlog/chunks_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/transactionlog/chunks.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <atomic> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index 9ba9780f8ed..07f43e98cd3 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -2,7 +2,7 @@ #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchlib/test/directory_handler.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/document/util/bytebuffer.h> diff --git a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp index dcf4d15181b..8faec949d86 100644 --- a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp +++ b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp @@ -1,10 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/util/drainingbufferwriter.h> #include <vespa/vespalib/util/rand48.h> +#include <cassert> namespace search { @@ -20,7 +21,7 @@ public: static constexpr size_t BUFFER_SIZE = 262144; StoreBufferWriter(); - ~StoreBufferWriter(); + ~StoreBufferWriter() override; void flush() override; size_t getBytesWritten() const { return _bytesWritten; } diff --git a/searchlib/src/tests/util/rawbuf_test.cpp b/searchlib/src/tests/util/rawbuf_test.cpp index cee340481f8..5f8001b6af3 100644 --- a/searchlib/src/tests/util/rawbuf_test.cpp +++ b/searchlib/src/tests/util/rawbuf_test.cpp @@ -2,7 +2,7 @@ #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("rawbuf_test"); diff --git a/searchlib/src/vespa/searchlib/aggregation/grouping.h b/searchlib/src/vespa/searchlib/aggregation/grouping.h index f64aa2f5a3d..77d5655e5d2 100644 --- a/searchlib/src/vespa/searchlib/aggregation/grouping.h +++ b/searchlib/src/vespa/searchlib/aggregation/grouping.h @@ -73,10 +73,8 @@ public: void postMerge(); void preAggregate(bool isOrdered); void prune(const Grouping & b); - void aggregate(DocId from, DocId to); - void aggregate(DocId docId, HitRank rank = 0); - void aggregate(const document::Document & doc, HitRank rank = 0); - void aggregate(const RankedHit * rankedHit, unsigned int len); + void aggregate(DocId docId, HitRank rank); + void aggregate(const document::Document & doc, HitRank rank); void convertToGlobalId(const IDocumentMetaStore &metaStore); void postAggregate(); void postProcess(); @@ -84,6 +82,9 @@ public: void cleanTemporary(); void configureStaticStuff(const expression::ConfigureStaticParams & params); void cleanupAttributeReferences(); + // Only used by tests + void aggregate(DocId from, DocId to); + void aggregate(const RankedHit * rankedHit, unsigned int len); }; } diff --git a/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp b/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp index 81664ce66eb..89e6e6685e9 100644 --- a/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp @@ -4,10 +4,10 @@ #include "grouping.h" #include <vespa/searchlib/expression/multiargfunctionnode.h> #include <vespa/searchlib/expression/attributenode.h> -#include <vespa/searchlib/expression/attribute_map_lookup_node.h> #include <vespa/searchlib/expression/documentfieldnode.h> #include <vespa/searchlib/expression/interpolated_document_field_lookup_node.h> #include <vespa/searchlib/expression/interpolatedlookupfunctionnode.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> using namespace search::expression; @@ -20,44 +20,32 @@ AttributeNodeReplacer::check(const vespalib::Identifiable &obj) const } void +AttributeNodeReplacer::replaceRecurse(ExpressionNode * exp, std::function<void(ExpressionNodeUP)> && modifier) { + if (exp == nullptr) return; + if (exp->inherits(AttributeNode::classId)) { + auto replacementNode = getReplacementNode(static_cast<const AttributeNode &>(*exp)); + if (replacementNode) { + modifier(std::move(replacementNode)); + } + } else { + exp->select(*this, *this); + } +} + +void AttributeNodeReplacer::execute(vespalib::Identifiable &obj) { if (obj.getClass().inherits(GroupingLevel::classId)) { - GroupingLevel & g(static_cast<GroupingLevel &>(obj)); - if (g.getExpression().getRoot()->inherits(AttributeNode::classId)) { - auto replacementNode = getReplacementNode(static_cast<const AttributeNode &>(*g.getExpression().getRoot())); - if (replacementNode) { - g.setExpression(std::move(replacementNode)); - } - } else { - g.getExpression().getRoot()->select(*this, *this); - } + auto & g(static_cast<GroupingLevel &>(obj)); + replaceRecurse(g.getExpression().getRoot(), [&g](ExpressionNodeUP replacement) { g.setExpression(std::move(replacement)); }); g.groupPrototype().select(*this, *this); } else if(obj.getClass().inherits(AggregationResult::classId)) { - AggregationResult & a(static_cast<AggregationResult &>(obj)); - ExpressionNode * e(a.getExpression()); - if (e) { - if (e->inherits(AttributeNode::classId)) { - auto replacementNode = getReplacementNode(static_cast<const AttributeNode &>(*e)); - if (replacementNode) { - a.setExpression(std::move(replacementNode)); - } - } else { - e->select(*this, *this); - } - } + auto & a(static_cast<AggregationResult &>(obj)); + replaceRecurse(a.getExpression(), [&a](ExpressionNodeUP replacement) { a.setExpression(std::move(replacement)); }); } else if(obj.getClass().inherits(MultiArgFunctionNode::classId)) { MultiArgFunctionNode::ExpressionNodeVector & v(static_cast<MultiArgFunctionNode &>(obj).expressionNodeVector()); - for(size_t i(0), m(v.size()); i < m; i++) { - ExpressionNode::CP & e(v[i]); - if (e->inherits(AttributeNode::classId)) { - auto replacementNode = getReplacementNode(static_cast<const AttributeNode &>(*e)); - if (replacementNode) { - e = std::move(replacementNode); - } - } else { - e->select(*this, *this); - } + for (auto & e : v) { + replaceRecurse(e.get(), [&e](ExpressionNodeUP replacement) noexcept { e = std::move(replacement); }); } } } @@ -72,6 +60,14 @@ Attribute2DocumentAccessor::getReplacementNode(const AttributeNode &attributeNod return std::make_unique<DocumentFieldNode>(attributeNode.getAttributeName()); } +std::unique_ptr<ExpressionNode> +NonAttribute2DocumentAccessor::getReplacementNode(const expression::AttributeNode &attributeNode) { + if (_attrCtx.getAttribute(attributeNode.getAttributeName()) == nullptr) { + return Attribute2DocumentAccessor::getReplacementNode(attributeNode); + } + return {}; +} + } // this function was added by ../../forcelink.sh diff --git a/searchlib/src/vespa/searchlib/aggregation/modifiers.h b/searchlib/src/vespa/searchlib/aggregation/modifiers.h index 934e7111ced..9b0f5d8186d 100644 --- a/searchlib/src/vespa/searchlib/aggregation/modifiers.h +++ b/searchlib/src/vespa/searchlib/aggregation/modifiers.h @@ -4,28 +4,43 @@ #include <vespa/vespalib/objects/objectoperation.h> #include <vespa/vespalib/objects/objectpredicate.h> #include <memory> +#include <functional> namespace search::expression { + class ExpressionNode; + class AttributeNode; +} -class ExpressionNode; -class AttributeNode; - +namespace search::attribute { + class IAttributeContext; } namespace search::aggregation { class AttributeNodeReplacer : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { +protected: + using ExpressionNodeUP = std::unique_ptr<expression::ExpressionNode>; private: + void replaceRecurse(expression::ExpressionNode * exp, std::function<void(ExpressionNodeUP)> && modifier); void execute(vespalib::Identifiable &obj) override; bool check(const vespalib::Identifiable &obj) const override; - virtual std::unique_ptr<search::expression::ExpressionNode> getReplacementNode(const search::expression::AttributeNode &attributeNode) = 0; + virtual ExpressionNodeUP getReplacementNode(const expression::AttributeNode &attributeNode) = 0; }; class Attribute2DocumentAccessor : public AttributeNodeReplacer { +protected: + ExpressionNodeUP getReplacementNode(const expression::AttributeNode &attributeNode) override; +}; + +class NonAttribute2DocumentAccessor : public Attribute2DocumentAccessor +{ +public: + explicit NonAttribute2DocumentAccessor(const attribute::IAttributeContext &attrCtx) noexcept : _attrCtx(attrCtx) {} private: - std::unique_ptr<search::expression::ExpressionNode> getReplacementNode(const search::expression::AttributeNode &attributeNode) override; + ExpressionNodeUP getReplacementNode(const expression::AttributeNode &attributeNode) override; + const attribute::IAttributeContext &_attrCtx; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributecontext.h b/searchlib/src/vespa/searchlib/attribute/attributecontext.h index a02e05abe4f..bd98031ee66 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributecontext.h +++ b/searchlib/src/vespa/searchlib/attribute/attributecontext.h @@ -29,7 +29,7 @@ private: const IAttributeVector *getAttribute(AttributeMap & map, const string & name, bool stableEnum) const; const IAttributeVector *getAttributeMtSafe(AttributeMap & map, const string & name, bool stableEnum) const; public: - AttributeContext(const IAttributeManager & manager); + explicit AttributeContext(const IAttributeManager & manager); ~AttributeContext() override; // Implements IAttributeContext diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 27c2b6d5e41..b468d653032 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -16,7 +16,6 @@ vespa_add_library(searchlib_features OBJECT distance_calculator_bundle.cpp distancefeature.cpp distancetopathfeature.cpp - documenttestutils.cpp dotproductfeature.cpp element_completeness_feature.cpp element_similarity_feature.cpp diff --git a/searchlib/src/vespa/searchlib/features/document_frequency.h b/searchlib/src/vespa/searchlib/features/document_frequency.h new file mode 100644 index 00000000000..f84e12d9e5c --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/document_frequency.h @@ -0,0 +1,26 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> + +namespace search::features::util { + +/* + * Struct containing the raw data used to calculate significance. + */ +struct DocumentFrequency { + uint64_t frequency; // number of documents containing the word + uint64_t count; // total number of documents + + DocumentFrequency(uint64_t document_frequency_in, uint64_t document_count_in) + : frequency(document_frequency_in), + count(document_count_in) + { + } + bool operator==(const DocumentFrequency& rhs) const noexcept { + return frequency == rhs.frequency && count == rhs.count; + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/documenttestutils.cpp b/searchlib/src/vespa/searchlib/features/documenttestutils.cpp deleted file mode 100644 index 5962cb32573..00000000000 --- a/searchlib/src/vespa/searchlib/features/documenttestutils.cpp +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - - -#include "utils.h" -#include <vespa/searchlib/fef/itablemanager.h> -#include <vespa/searchlib/fef/properties.h> -#include <vespa/searchlib/fef/itermdata.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/stllike/asciistream.h> - -#include <cmath> -#include <ostream> - -#include <vespa/vespalib/util/issue.h> -using vespalib::Issue; - -#include <vespa/log/log.h> -LOG_SETUP(".features.utils"); -using namespace search::fef; - -namespace search::features::util { - -feature_t -lookupConnectedness(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) -{ - if (termId == 0) { - return fallback; // no previous term - } - - const ITermData * data = env.getTerm(termId); - const ITermData * prev = env.getTerm(termId - 1); - if (data == nullptr || prev == nullptr) { - return fallback; // default value - } - return lookupConnectedness(env, data->getUniqueId(), prev->getUniqueId(), fallback); -} - -feature_t -lookupConnectedness(const search::fef::IQueryEnvironment& env, - uint32_t currUniqueId, uint32_t prevUniqueId, feature_t fallback) -{ - // Connectedness of 0.5 between term with unique id 2 and term with unique id 1 is represented as: - // [vespa.term.2.connexity: "1", vespa.term.2.connexity: "0.5"] - vespalib::asciistream os; - os << "vespa.term." << currUniqueId << ".connexity"; - Property p = env.getProperties().lookup(os.str()); - if (p.size() == 2) { - // we have a defined connectedness with the previous term - if (strToNum<uint32_t>(p.getAt(0)) == prevUniqueId) { - return strToNum<feature_t>(p.getAt(1)); - } - } - return fallback; -} - -feature_t -lookupSignificance(const search::fef::IQueryEnvironment& env, const ITermData& term, feature_t fallback) -{ - // Significance of 0.5 for term with unique id 1 is represented as: - // [vespa.term.1.significance: "0.5"] - vespalib::asciistream os; - os << "vespa.term." << term.getUniqueId() << ".significance"; - Property p = env.getProperties().lookup(os.str()); - if (p.found()) { - return strToNum<feature_t>(p.get()); - } - return fallback; -} - -feature_t -lookupSignificance(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) -{ - const ITermData* term = env.getTerm(termId); - if (term == nullptr) { - return fallback; - } - return lookupSignificance(env, *term, fallback); -} - -double -getRobertsonSparckJonesWeight(double docCount, double docsInCorpus) -{ - return std::log((docsInCorpus - docCount + 0.5)/(docCount + 0.5)); -} - -static const double N = 1000000.0; - -feature_t -getSignificance(double docFreq) -{ - if (docFreq < (1.0/N)) { - docFreq = 1.0/N; - } - if (docFreq > 1.0) { - docFreq = 1.0; - } - double d = std::log(docFreq)/std::log(1.0/N); - return 0.5 + 0.5 * d; -#if 0 - double n = docFreq * N; - n = (n == 0) ? 1 : (n > N ? N : n); - double a = getRobertsonSparckJonesWeight(1, N + 1); - double b = getRobertsonSparckJonesWeight(N + 1, N + 1); - double w = getRobertsonSparckJonesWeight(n, N + 1); - return ((w - b)/(a - b)); -#endif -} - -feature_t -getSignificance(const search::fef::ITermData& termData) -{ - using FRA = search::fef::ITermFieldRangeAdapter; - double df = 0; - for (FRA iter(termData); iter.valid(); iter.next()) { - df = std::max(df, iter.get().getDocFreq()); - } - - feature_t signif = getSignificance(df); - LOG(debug, "getSignificance %e %f [ %e %f ] = %e", df, df, df * N, df * N, signif); - return signif; -} - -const search::fef::Table * -lookupTable(const search::fef::IIndexEnvironment & env, const vespalib::string & featureName, - const vespalib::string & table, const vespalib::string & fieldName, const vespalib::string & fallback) -{ - vespalib::string tn1 = env.getProperties().lookup(featureName, table).get(fallback); - vespalib::string tn2 = env.getProperties().lookup(featureName, table, fieldName).get(tn1); - const search::fef::Table * retval = env.getTableManager().getTable(tn2); - if (retval == nullptr) { - LOG(warning, "Could not find the %s '%s' to be used for field '%s' in feature '%s'", - table.c_str(), tn2.c_str(), fieldName.c_str(), featureName.c_str()); - } - return retval; -} - -const search::fef::ITermData * -getTermByLabel(const search::fef::IQueryEnvironment &env, const vespalib::string &label) -{ - // Labeling the query item with unique id '5' with the label 'foo' - // is represented as: [vespa.label.foo.id: "5"] - vespalib::asciistream os; - os << "vespa.label." << label << ".id"; - Property p = env.getProperties().lookup(os.str()); - if (!p.found()) { - return 0; - } - uint32_t uid = strToNum<uint32_t>(p.get()); - if (uid == 0) { - Issue::report("Query label '%s' was attached to invalid unique id: '%s'", - label.c_str(), p.get().c_str()); - return 0; - } - for (uint32_t i(0), m(env.getNumTerms()); i < m; ++i) { - const ITermData *term = env.getTerm(i); - if (term->getUniqueId() == uid) { - return term; - } - } - Issue::report("Query label '%s' was attached to non-existing unique id: '%s'", - label.c_str(), p.get().c_str()); - return 0; -} - -} diff --git a/searchlib/src/vespa/searchlib/features/queryterm.cpp b/searchlib/src/vespa/searchlib/features/queryterm.cpp index d3a8a4f6cf4..0960a421abb 100644 --- a/searchlib/src/vespa/searchlib/features/queryterm.cpp +++ b/searchlib/src/vespa/searchlib/features/queryterm.cpp @@ -12,7 +12,7 @@ QueryTerm QueryTermFactory::create(const IQueryEnvironment & env, uint32_t termIdx, bool lookupConnectedness) { const ITermData *termData = env.getTerm(termIdx); - feature_t fallback = util::getSignificance(*termData); + feature_t fallback = util::calculate_legacy_significance(*termData); feature_t significance = features::util::lookupSignificance(env, termIdx, fallback); feature_t connectedness = 0; if (lookupConnectedness) { diff --git a/searchlib/src/vespa/searchlib/features/termfeature.cpp b/searchlib/src/vespa/searchlib/features/termfeature.cpp index e3ce4f26833..e22315745cd 100644 --- a/searchlib/src/vespa/searchlib/features/termfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termfeature.cpp @@ -21,7 +21,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env, _significance(0) { if (_termData != nullptr) { - feature_t fallback = util::getSignificance(*_termData); + feature_t fallback = util::calculate_legacy_significance(*_termData); _significance = util::lookupSignificance(env, termId, fallback); } } diff --git a/searchlib/src/vespa/searchlib/features/utils.cpp b/searchlib/src/vespa/searchlib/features/utils.cpp index 92758c58262..6555db03824 100644 --- a/searchlib/src/vespa/searchlib/features/utils.cpp +++ b/searchlib/src/vespa/searchlib/features/utils.cpp @@ -1,7 +1,20 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "utils.hpp" +#include <vespa/searchlib/fef/itablemanager.h> +#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/itermdata.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/issue.h> #include <charconv> +#include <cmath> +#include <ostream> + +#include <vespa/log/log.h> +LOG_SETUP(".features.utils"); + +using vespalib::Issue; +using namespace search::fef; namespace search::features::util { @@ -40,4 +53,160 @@ template <> int16_t strToNum<int16_t>(vespalib::stringref str) { return strToIn template <> int32_t strToNum<int32_t>(vespalib::stringref str) { return strToInt<int32_t>(str); } template <> int64_t strToNum<int64_t>(vespalib::stringref str) { return strToInt<int64_t>(str); } +feature_t +lookupConnectedness(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) +{ + if (termId == 0) { + return fallback; // no previous term + } + + const ITermData * data = env.getTerm(termId); + const ITermData * prev = env.getTerm(termId - 1); + if (data == nullptr || prev == nullptr) { + return fallback; // default value + } + return lookupConnectedness(env, data->getUniqueId(), prev->getUniqueId(), fallback); +} + +feature_t +lookupConnectedness(const search::fef::IQueryEnvironment& env, + uint32_t currUniqueId, uint32_t prevUniqueId, feature_t fallback) +{ + // Connectedness of 0.5 between term with unique id 2 and term with unique id 1 is represented as: + // [vespa.term.2.connexity: "1", vespa.term.2.connexity: "0.5"] + vespalib::asciistream os; + os << "vespa.term." << currUniqueId << ".connexity"; + Property p = env.getProperties().lookup(os.str()); + if (p.size() == 2) { + // we have a defined connectedness with the previous term + if (strToNum<uint32_t>(p.getAt(0)) == prevUniqueId) { + return strToNum<feature_t>(p.getAt(1)); + } + } + return fallback; +} + +feature_t +lookupSignificance(const search::fef::IQueryEnvironment& env, const ITermData& term, feature_t fallback) +{ + // Significance of 0.5 for term with unique id 1 is represented as: + // [vespa.term.1.significance: "0.5"] + vespalib::asciistream os; + os << "vespa.term." << term.getUniqueId() << ".significance"; + Property p = env.getProperties().lookup(os.str()); + if (p.found()) { + return strToNum<feature_t>(p.get()); + } + return fallback; +} + +feature_t +lookupSignificance(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) +{ + const ITermData* term = env.getTerm(termId); + if (term == nullptr) { + return fallback; + } + return lookupSignificance(env, *term, fallback); +} + +static const double N = 1000000.0; + +feature_t +calculate_legacy_significance(double docFreq) +{ + if (docFreq < (1.0/N)) { + docFreq = 1.0/N; + } + if (docFreq > 1.0) { + docFreq = 1.0; + } + double d = std::log(docFreq)/std::log(1.0/N); + return 0.5 + 0.5 * d; +} + +feature_t +calculate_legacy_significance(const search::fef::ITermData& termData) +{ + using FRA = search::fef::ITermFieldRangeAdapter; + double df = 0; + for (FRA iter(termData); iter.valid(); iter.next()) { + df = std::max(df, iter.get().getDocFreq()); + } + + feature_t signif = calculate_legacy_significance(df); + LOG(debug, "calculate_legacy_significance %e %f [ %e %f ] = %e", df, df, df * N, df * N, signif); + return signif; +} + +const search::fef::Table * +lookupTable(const search::fef::IIndexEnvironment & env, const vespalib::string & featureName, + const vespalib::string & table, const vespalib::string & fieldName, const vespalib::string & fallback) +{ + vespalib::string tn1 = env.getProperties().lookup(featureName, table).get(fallback); + vespalib::string tn2 = env.getProperties().lookup(featureName, table, fieldName).get(tn1); + const search::fef::Table * retval = env.getTableManager().getTable(tn2); + if (retval == nullptr) { + LOG(warning, "Could not find the %s '%s' to be used for field '%s' in feature '%s'", + table.c_str(), tn2.c_str(), fieldName.c_str(), featureName.c_str()); + } + return retval; +} + +const search::fef::ITermData * +getTermByLabel(const search::fef::IQueryEnvironment &env, const vespalib::string &label) +{ + // Labeling the query item with unique id '5' with the label 'foo' + // is represented as: [vespa.label.foo.id: "5"] + vespalib::asciistream os; + os << "vespa.label." << label << ".id"; + Property p = env.getProperties().lookup(os.str()); + if (!p.found()) { + return 0; + } + uint32_t uid = strToNum<uint32_t>(p.get()); + if (uid == 0) { + Issue::report("Query label '%s' was attached to invalid unique id: '%s'", + label.c_str(), p.get().c_str()); + return 0; + } + for (uint32_t i(0), m(env.getNumTerms()); i < m; ++i) { + const ITermData *term = env.getTerm(i); + if (term->getUniqueId() == uid) { + return term; + } + } + Issue::report("Query label '%s' was attached to non-existing unique id: '%s'", + label.c_str(), p.get().c_str()); + return 0; +} + +std::optional<DocumentFrequency> +lookup_document_frequency(const search::fef::IQueryEnvironment& env, const ITermData& term) +{ + vespalib::asciistream os; + auto unique_id = term.getUniqueId(); + if (unique_id != 0) { + os << "vespa.term." << unique_id << ".docfreq"; + Property p = env.getProperties().lookup(os.str()); + if (p.size() == 2) { + // we have a defined document frequency + auto document_frequency = strToNum<uint64_t>(p.getAt(0)); + auto document_count = strToNum<uint64_t>(p.getAt(1)); + return DocumentFrequency(document_frequency, document_count); + } + } + return {}; +} + +std::optional<DocumentFrequency> +lookup_document_frequency(const search::fef::IQueryEnvironment& env, uint32_t termId) +{ + const ITermData* term = env.getTerm(termId); + if (term == nullptr) { + return {}; + } + return lookup_document_frequency(env, *term); +} + } diff --git a/searchlib/src/vespa/searchlib/features/utils.h b/searchlib/src/vespa/searchlib/features/utils.h index b7b84013630..a0ca4b8be2a 100644 --- a/searchlib/src/vespa/searchlib/features/utils.h +++ b/searchlib/src/vespa/searchlib/features/utils.h @@ -2,6 +2,7 @@ #pragma once +#include "document_frequency.h" #include <vespa/searchlib/fef/iqueryenvironment.h> #include <vespa/searchlib/fef/table.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> @@ -10,6 +11,7 @@ #include <vespa/searchlib/common/feature.h> #include <vespa/vespalib/util/string_hash.h> #include <limits> +#include <optional> namespace search::features::util { @@ -71,22 +73,6 @@ inline feature_t getAsFeature<vespalib::stringref>(vespalib::stringref value) { return vespalib::hash2d(value); } - -/** - * This method inputs a value to cap to the range [capFloor, capCeil] and then normalize this - * value to the unit range [0, 1]. - * - * @param val The value to unit normalize. - * @param capFloor The minimum value of the cap range. - * @param capCeil The maximum value of the cap range. - * @return The unit normalized value. - */ -template <typename T> -T unitNormalize(const T &val, const T &capFloor, const T &capCeil) -{ - return (std::max(capFloor, std::min(capCeil, val)) - capFloor) / (capCeil - capFloor); -} - /** * Returns the normalized strength with which the given term is connected to the previous term in the query. * Uses the property map of the query environment to lookup this data. @@ -134,19 +120,12 @@ feature_t lookupSignificance(const search::fef::IQueryEnvironment& env, const se feature_t lookupSignificance(const search::fef::IQueryEnvironment & env, uint32_t termId, feature_t fallback = 0.0f); /** - * Returns the Robertson-Sparck-Jones weight based on the given document count - * (number of documents containing the term) and the number of documents in the corpus. - * This weight is a variant of inverse document frequency. - */ -double getRobertsonSparckJonesWeight(double docCount, double docsInCorpus); - -/** * Returns the significance based on the given scaled number of documents containing the term. * * @param docFreq The scaled number of documents containing the term. * @return The significance. */ -feature_t getSignificance(double docFreq); +feature_t calculate_legacy_significance(double docFreq); /** * Returns the significance based on max known frequency of the term @@ -154,7 +133,7 @@ feature_t getSignificance(double docFreq); * @param termData Data for the term * @return The significance. */ -feature_t getSignificance(const search::fef::ITermData &termData); +feature_t calculate_legacy_significance(const search::fef::ITermData& termData); /** * Lookups a table by using the properties and the table manager in the given index environment. @@ -214,4 +193,10 @@ getTermFieldHandle(const search::fef::IQueryEnvironment &env, uint32_t termId, u const search::fef::ITermData * getTermByLabel(const search::fef::IQueryEnvironment &env, const vespalib::string &label); +std::optional<DocumentFrequency> +lookup_document_frequency(const search::fef::IQueryEnvironment& env, const search::fef::ITermData& term); + +std::optional<DocumentFrequency> +lookup_document_frequency(const search::fef::IQueryEnvironment& env, uint32_t termId); + } diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index 116e677d439..3da68db4c34 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -8,6 +8,7 @@ #include "queryvisitor.h" #include "string_term_vector.h" #include "termnodes.h" +#include <cassert> namespace search::query { @@ -29,8 +30,8 @@ public: private: void visitNodes(const std::vector<Node *> &nodes) { - for (size_t i = 0; i < nodes.size(); ++i) { - nodes[i]->accept(*this); + for (auto node : nodes) { + node->accept(*this); } } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index f9ab5d705c9..c815d5e7ed3 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -42,7 +42,7 @@ constexpr float alloc_grow_factor = 0.3; // TODO: Adjust these numbers to what we accept as max in config. constexpr size_t max_level_array_size = 16; constexpr size_t max_link_array_size = 193; -constexpr vespalib::duration MAX_COUNT_DURATION(100ms); +constexpr vespalib::duration MAX_COUNT_DURATION(1000ms); const vespalib::string hnsw_max_squared_norm = "hnsw.max_squared_norm"; diff --git a/searchsummary/src/tests/juniper/SrcTestSuite.cpp b/searchsummary/src/tests/juniper/SrcTestSuite.cpp index 955ed01fd85..0a45d6ffc95 100644 --- a/searchsummary/src/tests/juniper/SrcTestSuite.cpp +++ b/searchsummary/src/tests/juniper/SrcTestSuite.cpp @@ -6,7 +6,7 @@ #include "queryparserTest.h" #include "matchobjectTest.h" #include "auxTest.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> /** * The SrcTestSuite class runs all the unit tests for the src module. * diff --git a/searchsummary/src/tests/juniper/appender_test.cpp b/searchsummary/src/tests/juniper/appender_test.cpp index 97d07998e3f..17cd4a77472 100644 --- a/searchsummary/src/tests/juniper/appender_test.cpp +++ b/searchsummary/src/tests/juniper/appender_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #define _NEED_SUMMARY_CONFIG_IMPL #include <vespa/juniper/SummaryConfig.h> diff --git a/searchsummary/src/tests/juniper/auxTestApp.cpp b/searchsummary/src/tests/juniper/auxTestApp.cpp index 62b2d3f934c..1344188e249 100644 --- a/searchsummary/src/tests/juniper/auxTestApp.cpp +++ b/searchsummary/src/tests/juniper/auxTestApp.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "auxTest.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> void Usage(char* s) { diff --git a/searchsummary/src/tests/juniper/matchobjectTestApp.cpp b/searchsummary/src/tests/juniper/matchobjectTestApp.cpp index bd199f9d1d6..abcc496e717 100644 --- a/searchsummary/src/tests/juniper/matchobjectTestApp.cpp +++ b/searchsummary/src/tests/juniper/matchobjectTestApp.cpp @@ -2,7 +2,7 @@ #include "matchobjectTest.h" #include "testenv.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/juniper/wildcard_match.h> #include <iostream> diff --git a/searchsummary/src/tests/juniper/mcandTestApp.cpp b/searchsummary/src/tests/juniper/mcandTestApp.cpp index 38a3cdee367..e92bf8b89d3 100644 --- a/searchsummary/src/tests/juniper/mcandTestApp.cpp +++ b/searchsummary/src/tests/juniper/mcandTestApp.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mcandTest.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> int main(int argc, char **argv) { juniper::TestEnv te(argc, argv, TEST_PATH("./testclient.rc").c_str()); diff --git a/searchsummary/src/tests/juniper/queryparserTestApp.cpp b/searchsummary/src/tests/juniper/queryparserTestApp.cpp index 95fd198cffe..c818afc5533 100644 --- a/searchsummary/src/tests/juniper/queryparserTestApp.cpp +++ b/searchsummary/src/tests/juniper/queryparserTestApp.cpp @@ -2,7 +2,7 @@ #include "queryparserTest.h" #include "testenv.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> int main(int argc, char **argv) { diff --git a/searchsummary/src/tests/juniper/queryvisitor_test.cpp b/searchsummary/src/tests/juniper/queryvisitor_test.cpp index a5862c0aec3..f4501ef1909 100644 --- a/searchsummary/src/tests/juniper/queryvisitor_test.cpp +++ b/searchsummary/src/tests/juniper/queryvisitor_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <memory> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/juniper/queryhandle.h> #include <vespa/juniper/queryvisitor.h> diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 5f97900b557..341cbd310bd 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -152,7 +152,6 @@ public class SlobrokMonitorManagerImpl extends AbstractComponent implements Slob case "logserver": case "metricsproxy": case "slobrok": - case "transactionlogserver": // TODO: Remove when oldest Vespa version in use is 8.354 return Optional.empty(); case "qrserver": diff --git a/slobrok/src/tests/configure/configure.cpp b/slobrok/src/tests/configure/configure.cpp index d16c241b905..e6f17d23f01 100644 --- a/slobrok/src/tests/configure/configure.cpp +++ b/slobrok/src/tests/configure/configure.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/slobrok/sbregister.h> #include <vespa/slobrok/server/slobrokserver.h> diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index a18591616f4..87a031b9e33 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -10,7 +10,7 @@ vespa_define_module( vespa_documentapi vespa_document vespalib - vdslib + vespa_vdslib vespa_persistence EXTERNAL_DEPENDS diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index 6aff5c52598..a20902422b0 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -1,9 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> -#include <tests/common/storage_config_set.h> +#include <vespa/config-stor-distribution.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/datatype/documenttype.h> @@ -24,7 +25,7 @@ #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/config-stor-distribution.h> +#include <vespa/vespalib/testkit/test_path.h> #include <future> #include <vespa/log/log.h> @@ -115,22 +116,23 @@ public: BucketManagerTest::~BucketManagerTest() = default; -#define ASSERT_DUMMYLINK_REPLY_COUNT(link, count) \ - if (link->getNumReplies() != count) { \ - std::ostringstream ost; \ - ost << "Expected there to be " << count << " replies in link, but " \ - << "found " << link->getNumReplies() << ":\n"; \ - for (uint32_t i=0; i<link->getNumReplies(); ++i) { \ - ost << link->getReply(i)->getType() << "\n"; \ - } \ - FAIL() << ost.str(); \ +void check_dummy_link_reply_count(const DummyStorageLink& link, size_t expected_count) { + if (link.getNumReplies() != expected_count) { + std::ostringstream ost; + ost << "Expected there to be " << expected_count << " replies in link, but " + << "found " << link.getNumReplies() << ":\n"; + for (uint32_t i = 0; i < link.getNumReplies(); ++i) { + ost << link.getReply(i)->getType() << "\n"; + } + FAIL() << ost.str(); } +} void BucketManagerTest::setupTestEnvironment() { _config = StorageConfigSet::make_storage_node_config(); auto repo = std::make_shared<const DocumentTypeRepo>( - *ConfigGetter<DocumenttypesConfig>::getConfig("config-doctypes", FileSpec("../config-doctypes.cfg"))); + *ConfigGetter<DocumenttypesConfig>::getConfig("config-doctypes", FileSpec(TEST_PATH("../config-doctypes.cfg")))); _top = std::make_unique<DummyStorageLink>(); _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->config_uri()); _node->setTypeRepo(repo); @@ -312,7 +314,7 @@ TEST_F(BucketManagerTest, DISABLED_request_bucket_info_with_state) { { LOG(info, "Waiting for response from 3 request bucket info messages"); _top->waitForMessages(3, 5); - ASSERT_DUMMYLINK_REPLY_COUNT(_top, 3); + ASSERT_NO_FATAL_FAILURE(check_dummy_link_reply_count(*_top, 3)); std::map<uint64_t, api::RequestBucketInfoReply::SP> replies; for (uint32_t i=0; i<3; ++i) { replies[_top->getReply(i)->getMsgId()] @@ -356,7 +358,7 @@ TEST_F(BucketManagerTest, request_bucket_info_with_list) { _top->sendDown(cmd); _top->waitForMessages(1, 5); - ASSERT_DUMMYLINK_REPLY_COUNT(_top, 1); + ASSERT_NO_FATAL_FAILURE(check_dummy_link_reply_count(*_top, 1)); auto reply = std::dynamic_pointer_cast<api::RequestBucketInfoReply>(_top->getReply(0)); _top->reset(); ASSERT_TRUE(reply.get()); @@ -526,7 +528,7 @@ class ConcurrentOperationFixture { public: explicit ConcurrentOperationFixture(BucketManagerTest& self) : _self(self), - _state(std::make_shared<lib::ClusterState>("distributor:1 storage:1")) + _state(std::make_shared<lib::ClusterState>("version:2 distributor:1 storage:1")) { _self.setupTestEnvironment(); _self._top->open(); diff --git a/storage/src/tests/common/testnodestateupdater.cpp b/storage/src/tests/common/testnodestateupdater.cpp index f9671617352..5d1d7a085b9 100644 --- a/storage/src/tests/common/testnodestateupdater.cpp +++ b/storage/src/tests/common/testnodestateupdater.cpp @@ -25,17 +25,28 @@ TestNodeStateUpdater::getClusterStateBundle() const } void +TestNodeStateUpdater::patch_distribution(std::shared_ptr<const lib::Distribution> distribution) +{ + _clusterStateBundle = _clusterStateBundle->clone_with_new_distribution( + lib::DistributionConfigBundle::of(std::move(distribution))); +} + +void TestNodeStateUpdater::setClusterState(std::shared_ptr<const lib::ClusterState> c) { - setClusterStateBundle(std::make_shared<const lib::ClusterStateBundle>(*c)); + setClusterStateBundle(std::make_shared<const lib::ClusterStateBundle>(std::move(c))); } void TestNodeStateUpdater::setClusterStateBundle(std::shared_ptr<const lib::ClusterStateBundle> clusterStateBundle) { + auto existing_distr = _clusterStateBundle->distribution_config_bundle(); _clusterStateBundle = std::move(clusterStateBundle); - for (uint32_t i = 0; i < _listeners.size(); ++i) { - _listeners[i]->handleNewState(); + if (!_clusterStateBundle->has_distribution_config() && existing_distr) { + _clusterStateBundle = _clusterStateBundle->clone_with_new_distribution(existing_distr); + } + for (auto* listener : _listeners) { + listener->handleNewState(); } } diff --git a/storage/src/tests/common/testnodestateupdater.h b/storage/src/tests/common/testnodestateupdater.h index e5418c238d5..6a00e9a2264 100644 --- a/storage/src/tests/common/testnodestateupdater.h +++ b/storage/src/tests/common/testnodestateupdater.h @@ -1,11 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::TestNodeStateUpdater - * \ingroup common - * - * \brief Test implementation of the node state updater. - */ - #pragma once #include <vespa/storage/common/nodestateupdater.h> @@ -14,6 +7,7 @@ namespace storage::lib { class ClusterState; class ClusterStateBundle; + class Distribution; } namespace storage { @@ -53,6 +47,7 @@ public: _current = std::make_shared<lib::NodeState>(state); } + void patch_distribution(std::shared_ptr<const lib::Distribution> distribution); void setClusterState(std::shared_ptr<const lib::ClusterState> c); void setClusterStateBundle(std::shared_ptr<const lib::ClusterStateBundle> clusterStateBundle); diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index d811f100aec..b2b82a46850 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -6,10 +6,10 @@ #include <vespa/storage/config/config-stor-distributormanager.h> #include <vespa/storage/config/config-stor-visitordispatcher.h> #include <vespa/config-stor-distribution.h> -#include <vespa/config-fleetcontroller.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/time.h> #include <vespa/config/subscription/configuri.h> @@ -59,6 +59,7 @@ TestStorageApp::TestStorageApp(StorageComponentRegisterImpl::UP compReg, auto distr = std::make_shared<lib::Distribution>( lib::Distribution::getDefaultDistributionConfig(redundancy, nodeCount)); _compReg.setDistribution(distr); + _nodeStateUpdater.patch_distribution(distr); } TestStorageApp::~TestStorageApp() = default; @@ -69,6 +70,7 @@ TestStorageApp::setDistribution(Redundancy redundancy, NodeCount nodeCount) auto distr = std::make_shared<lib::Distribution>( lib::Distribution::getDefaultDistributionConfig(redundancy, nodeCount)); _compReg.setDistribution(distr); + _nodeStateUpdater.patch_distribution(distr); } void @@ -83,6 +85,12 @@ TestStorageApp::setClusterState(const lib::ClusterState& c) _nodeStateUpdater.setClusterState(std::make_shared<lib::ClusterState>(c)); } +void +TestStorageApp::set_cluster_state_bundle(std::shared_ptr<const lib::ClusterStateBundle> state_bundle) +{ + _nodeStateUpdater.setClusterStateBundle(std::move(state_bundle)); +} + namespace { NodeIndex node_index_from_config(const config::ConfigUri& uri) { diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 04fa6996e15..c423761a9a2 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -73,6 +73,7 @@ public: void setDistribution(Redundancy, NodeCount); void setTypeRepo(std::shared_ptr<const document::DocumentTypeRepo> repo); void setClusterState(const lib::ClusterState&); + void set_cluster_state_bundle(std::shared_ptr<const lib::ClusterStateBundle>); // Utility functions for getting a hold of currently used bits. Practical // to avoid adding extra components in the tests. @@ -81,7 +82,7 @@ public: std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() { return _compReg.getTypeRepo(); } const document::BucketIdFactory& getBucketIdFactory() { return _compReg.getBucketIdFactory(); } TestNodeStateUpdater& getStateUpdater() { return _nodeStateUpdater; } - std::shared_ptr<lib::Distribution> & getDistribution() { return _compReg.getDistribution(); } + std::shared_ptr<const lib::Distribution> getDistribution() { return _compReg.getDistribution(); } TestNodeStateUpdater& getNodeStateUpdater() { return _nodeStateUpdater; } uint16_t getIndex() const { return _compReg.getIndex(); } const NodeIdentity& node_identity() const noexcept { return _node_identity; } diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 8982b02f2b7..6993882f479 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -53,6 +53,7 @@ struct ChangedBucketOwnershipHandlerTest : Test { void applyDistribution(Redundancy, NodeCount); void applyClusterState(const lib::ClusterState&); + void apply_cluster_state_bundle(std::shared_ptr<const lib::ClusterStateBundle>); document::BucketId nextOwnedBucket( uint16_t wantedOwner, @@ -84,6 +85,21 @@ struct ChangedBucketOwnershipHandlerTest : Test { return lib::ClusterState("distributor:4 storage:1 .0.s:d"); } + static std::shared_ptr<const lib::DistributionConfigBundle> make_distr_bundle(uint16_t node_count) { + return lib::DistributionConfigBundle::of(lib::Distribution::getDefaultDistributionConfig(1, node_count)); + } + + static std::shared_ptr<const lib::ClusterStateBundle> make_state_bundle_with_config( + vespalib::stringref state_str, uint16_t node_count) + { + return std::make_shared<const lib::ClusterStateBundle>( + std::make_shared<const lib::ClusterState>(state_str), + lib::ClusterStateBundle::BucketSpaceStateMapping{}, + std::nullopt, + make_distr_bundle(node_count), + false); + } + void SetUp() override; }; @@ -194,6 +210,7 @@ hasOnlySetSystemStateCmdQueued(DummyStorageLink& link) { void ChangedBucketOwnershipHandlerTest::applyDistribution(Redundancy redundancy, NodeCount nodeCount) { + // TODO set distribution via state bundle instead _app->setDistribution(redundancy, nodeCount); _handler->storageDistributionChanged(); } @@ -205,6 +222,13 @@ ChangedBucketOwnershipHandlerTest::applyClusterState(const lib::ClusterState& st _handler->reloadClusterState(); } +void +ChangedBucketOwnershipHandlerTest::apply_cluster_state_bundle(std::shared_ptr<const lib::ClusterStateBundle> state_bundle) +{ + _app->set_cluster_state_bundle(std::move(state_bundle)); + _handler->reloadClusterState(); +} + TEST_F(ChangedBucketOwnershipHandlerTest, enumerate_buckets_belonging_on_changed_nodes) { lib::ClusterState stateBefore("distributor:4 storage:1"); applyDistribution(Redundancy(1), NodeCount(4)); @@ -225,7 +249,7 @@ TEST_F(ChangedBucketOwnershipHandlerTest, enumerate_buckets_belonging_on_changed EXPECT_TRUE(hasAbortedNoneOf(cmd, node2Buckets)); // Handler must swallow abort replies - _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply().release())); + _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply())); EXPECT_EQ(size_t(0), _top->getNumReplies()); } @@ -312,7 +336,7 @@ TEST_F(ChangedBucketOwnershipHandlerTest, ownership_changed_on_distributor_up_ed EXPECT_TRUE(hasAbortedNoneOf(cmd, node2Buckets)); // Handler must swallow abort replies - _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply().release())); + _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply())); EXPECT_EQ(0, _top->getNumReplies()); } @@ -345,6 +369,32 @@ TEST_F(ChangedBucketOwnershipHandlerTest, distribution_config_change_updates_own sendAndExpectAbortedCreateBucket(2); } +TEST_F(ChangedBucketOwnershipHandlerTest, distribution_config_via_state_bundle_change_updates_ownership) { + apply_cluster_state_bundle(make_state_bundle_with_config("version:2 distributor:3 storage:1", 3)); + // Apply new distribution config containing only 1 distributor, meaning + // any messages sent from >1 must be aborted. + // This test case is a bit dodgy since the CC should never send a state with more nodes in it than + // the distribution config allows for when _it_ is responsible for also sending the config. + apply_cluster_state_bundle(make_state_bundle_with_config("version:3 distributor:3 storage:1", 1)); + sendAndExpectAbortedCreateBucket(2); +} + +TEST_F(ChangedBucketOwnershipHandlerTest, ignore_internal_config_once_state_bundle_with_config_received) { + apply_cluster_state_bundle(make_state_bundle_with_config("version:2 distributor:3 storage:3", 1)); + applyDistribution(Redundancy(1), NodeCount(3)); + // Bundle config says 1 node, internal config says 3. Trust the bundle(tm). + sendAndExpectAbortedCreateBucket(2); +} + +TEST_F(ChangedBucketOwnershipHandlerTest, revert_to_internal_config_if_distribution_no_longer_received_in_state_bundle) { + apply_cluster_state_bundle(make_state_bundle_with_config("version:2 distributor:1 storage:3", 3)); + + applyDistribution(Redundancy(1), NodeCount(1)); // not yet used + applyClusterState(lib::ClusterState("version:3 distributor:3 storage:3")); // no bundle config; revert to internal + + sendAndExpectAbortedCreateBucket(2); +} + /** * Generate and dispatch a message of the given type with the provided * arguments as if that message was sent from distributor 1. Messages will diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 1eb6bf5dd9a..365b9efeff0 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -19,6 +19,7 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/documentapi/messagebus/messages/testandsetcondition.h> using document::Bucket; @@ -71,7 +72,7 @@ struct DocumentApiConverterTest : Test { DocumentApiConverterTest() : _bucketResolver(std::make_shared<MockBucketResolver>()), - _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig("../config-doctypes.cfg"))), + _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig(TEST_PATH("../config-doctypes.cfg")))), _html_type(*_repo->getDocumentType("text/html")) { } diff --git a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp index c3641b9bc56..e59f6d22080 100644 --- a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp @@ -121,7 +121,7 @@ struct SetStateFixture : FixtureBase { } static lib::ClusterStateBundle dummy_baseline_bundle_with_deferred_activation(bool deferred) { - return lib::ClusterStateBundle(lib::ClusterState("version:123 distributor:3 storage:3"), {}, deferred); + return {lib::ClusterState("version:123 distributor:3 storage:3"), {}, deferred}; } }; @@ -166,6 +166,16 @@ TEST_F(ClusterControllerApiRpcServiceTest, set_distribution_states_rpc_with_feed f.assert_request_received_and_propagated(bundle); } +TEST_F(ClusterControllerApiRpcServiceTest, can_receive_cluster_state_bundle_with_embedded_distribution_config) { + auto distr_cfg = lib::DistributionConfigBundle::of(lib::Distribution::getDefaultDistributionConfig(3, 14)); + SetStateFixture f; + lib::ClusterStateBundle bundle( + std::make_shared<const lib::ClusterState>("version:123 distributor:3 storage:3"), + {}, std::nullopt, std::move(distr_cfg), false); + + f.assert_request_received_and_propagated(bundle); +} + TEST_F(ClusterControllerApiRpcServiceTest, compressed_bundle_is_transparently_uncompressed) { SetStateFixture f; auto state_str = make_compressable_state_string(); diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index 79246cb3ce1..45ad44b1152 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -10,6 +10,7 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/storageserver/statemanager.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/config-stor-distribution.h> #include <vespa/vespalib/gtest/gtest.h> using storage::lib::NodeState; @@ -38,6 +39,23 @@ struct StateManagerTest : Test, NodeStateReporter { return cmd; } + static std::shared_ptr<const lib::ClusterStateBundle> make_state_bundle_with_config( + vespalib::stringref state_str, uint16_t num_nodes) + { + auto state = std::make_shared<const ClusterState>(state_str); + auto distr = lib::DistributionConfigBundle::of(lib::Distribution::getDefaultDistributionConfig(1, num_nodes)); + return std::make_shared<lib::ClusterStateBundle>(std::move(state), + lib::ClusterStateBundle::BucketSpaceStateMapping{}, + std::nullopt, std::move(distr), false); + } + + + static std::shared_ptr<api::SetSystemStateCommand> make_set_state_cmd_with_config( + vespalib::stringref state_str, uint16_t num_nodes) + { + return std::make_shared<api::SetSystemStateCommand>(make_state_bundle_with_config(state_str, num_nodes)); + } + void get_single_reply(std::shared_ptr<api::StorageReply>& reply_out); void get_only_ok_reply(std::shared_ptr<api::StorageReply>& reply_out); void force_current_cluster_state_version(uint32_t version, uint16_t cc_index); @@ -56,6 +74,10 @@ struct StateManagerTest : Test, NodeStateReporter { void report(vespalib::JsonStream &) const override {} void extract_cluster_state_version_from_host_info(uint32_t& version_out); + + static vespalib::string to_string(const lib::Distribution::DistributionConfig& cfg) { + return lib::Distribution(cfg).serialized(); + } }; StateManagerTest::StateManagerTest() @@ -148,26 +170,107 @@ StateManagerTest::extract_cluster_state_version_from_host_info(uint32_t& version version_out = clusterStateVersionCursor.asLong(); } -TEST_F(StateManagerTest, cluster_state) { - std::shared_ptr<api::StorageReply> reply; - // Verify initial state on startup - auto currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); +TEST_F(StateManagerTest, cluster_state_and_config_has_expected_values_at_bootstrap) { + auto initial_bundle = _manager->getClusterStateBundle(); + auto currentState = initial_bundle->getBaselineClusterState(); EXPECT_EQ("cluster:d", currentState->toString(false)); EXPECT_EQ(currentState->getVersion(), 0); + // Distribution config should be equal to the config the node is running with. + ASSERT_TRUE(initial_bundle->has_distribution_config()); + EXPECT_EQ(to_string(initial_bundle->distribution_config_bundle()->config()), + _node->getComponentRegister().getDistribution()->serialized()); + auto currentNodeState = _manager->getCurrentNodeState(); EXPECT_EQ("s:d", currentNodeState->toString(false)); +} + +TEST_F(StateManagerTest, can_receive_state_bundle_without_distribution_config) { + ClusterState send_state("version:2 distributor:1 storage:4 .2.s:m"); + auto cmd = std::make_shared<api::SetSystemStateCommand>(send_state); + _upper->sendDown(cmd); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(send_state, *current_bundle->getBaselineClusterState()); + // Distribution config should be unchanged from bootstrap. + ASSERT_TRUE(current_bundle->has_distribution_config()); + EXPECT_EQ(to_string(current_bundle->distribution_config_bundle()->config()), + _node->getComponentRegister().getDistribution()->serialized()); + + auto current_node_state = _manager->getCurrentNodeState(); + EXPECT_EQ("s:m", current_node_state->toString(false)); +} + +TEST_F(StateManagerTest, can_receive_state_bundle_with_distribution_config) { + auto cmd = make_set_state_cmd_with_config("version:2 distributor:1 storage:4 .2.s:m", 5); + EXPECT_NE(to_string(cmd->getClusterStateBundle().distribution_config_bundle()->config()), + _node->getComponentRegister().getDistribution()->serialized()); + _upper->sendDown(cmd); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(*current_bundle, cmd->getClusterStateBundle()); // also compares distribution configs +} - ClusterState sendState("storage:4 .2.s:m"); - auto cmd = std::make_shared<api::SetSystemStateCommand>(sendState); +TEST_F(StateManagerTest, receiving_cc_bundle_with_distribution_config_disables_node_distribution_config_propagation) { + auto cmd = make_set_state_cmd_with_config("version:2 distributor:1 storage:4 .2.s:m", 5); _upper->sendDown(cmd); + std::shared_ptr<api::StorageReply> reply; ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + // Explicitly setting distribution config should not propagate to the active state bundle + // since we've flipped to expecting config from the cluster controllers instead. + auto distr = std::make_shared<lib::Distribution>(lib::Distribution::getDefaultDistributionConfig(2, 7)); + _node->getComponentRegister().setDistribution(distr); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(*current_bundle, cmd->getClusterStateBundle()); // unchanged +} + +TEST_F(StateManagerTest, internal_distribution_config_is_propagated_if_none_yet_received_from_cc) { + _upper->sendDown(make_set_state_cmd("version:10 distributor:1 storage:4", 0)); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + auto expected_bundle = make_state_bundle_with_config("version:10 distributor:1 storage:4", 7); + // Explicitly set internal config + _node->getComponentRegister().setDistribution(expected_bundle->distribution_config_bundle()->default_distribution_sp()); + _manager->storageDistributionChanged(); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(*current_bundle, *expected_bundle); +} + +TEST_F(StateManagerTest, revert_to_internal_config_if_cc_no_longer_sends_distribution_config) { + // Initial state bundle _with_ distribution config + auto cmd = make_set_state_cmd_with_config("version:2 distributor:1 storage:4 .2.s:m", 5); + _upper->sendDown(cmd); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + auto current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(to_string(current_bundle->distribution_config_bundle()->config()), + to_string(cmd->getClusterStateBundle().distribution_config_bundle()->config())); + + // CC then sends a new bundle _without_ config + _upper->sendDown(make_set_state_cmd("version:3 distributor:1 storage:4", 0)); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + + // Config implicitly reverted to the active internal config + current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(to_string(current_bundle->distribution_config_bundle()->config()), + _node->getComponentRegister().getDistribution()->serialized()); - currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); - EXPECT_EQ(sendState, *currentState); + // Explicitly set internal config + auto expected_bundle = make_state_bundle_with_config("version:3 distributor:1 storage:4", 7); + _node->getComponentRegister().setDistribution(expected_bundle->distribution_config_bundle()->default_distribution_sp()); + _manager->storageDistributionChanged(); - currentNodeState = _manager->getCurrentNodeState(); - EXPECT_EQ("s:m", currentNodeState->toString(false)); + // Internal config shall have taken effect, overriding that of the initial bundle + current_bundle = _manager->getClusterStateBundle(); + EXPECT_EQ(*current_bundle, *expected_bundle); } TEST_F(StateManagerTest, accept_lower_state_versions_if_strict_requirement_disabled) { diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 1f20a19ec51..280afe8fb91 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -526,12 +526,12 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac using RBISP = std::shared_ptr<api::RequestBucketInfoCommand>; std::map<uint16_t, RBISP> requests; - // TODO fetch distribution from bundle as well - auto distribution(_component.getBucketSpaceRepo().get(bucketSpace).getDistribution()); - auto clusterStateBundle(_component.getStateUpdater().getClusterStateBundle()); - assert(clusterStateBundle); - lib::ClusterState::CSP clusterState(clusterStateBundle->getDerivedClusterState(bucketSpace)); - assert(clusterState.get()); + auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); + assert(clusterStateBundle && clusterStateBundle->has_distribution_config()); + auto clusterState = clusterStateBundle->getDerivedClusterState(bucketSpace); + assert(clusterState); + const auto distribution = clusterStateBundle->bucket_space_distribution_or_nullptr(bucketSpace); + assert(distribution); const auto our_hash = distribution->getNodeGraph().getDistributionConfigHash(); diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp index 92b5257b991..d66219794c7 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.cpp +++ b/storage/src/vespa/storage/common/content_bucket_space.cpp @@ -10,6 +10,7 @@ ClusterStateAndDistribution::ClusterStateAndDistribution( : _cluster_state(std::move(cluster_state)), _distribution(std::move(distribution)) { + assert(_cluster_state && _distribution); } ClusterStateAndDistribution::~ClusterStateAndDistribution() = default; @@ -48,34 +49,6 @@ ContentBucketSpace::state_and_distribution() const noexcept { return _state_and_distribution; } -void -ContentBucketSpace::setClusterState(std::shared_ptr<const lib::ClusterState> clusterState) -{ - std::lock_guard guard(_lock); - _state_and_distribution = _state_and_distribution->with_new_state(std::move(clusterState)); -} - -std::shared_ptr<const lib::ClusterState> -ContentBucketSpace::getClusterState() const -{ - std::lock_guard guard(_lock); - return _state_and_distribution->_cluster_state; -} - -void -ContentBucketSpace::setDistribution(std::shared_ptr<const lib::Distribution> distribution) -{ - std::lock_guard guard(_lock); - _state_and_distribution = _state_and_distribution->with_new_distribution(std::move(distribution)); -} - -std::shared_ptr<const lib::Distribution> -ContentBucketSpace::getDistribution() const -{ - std::lock_guard guard(_lock); - return _state_and_distribution->_distribution; -} - bool ContentBucketSpace::getNodeUpInLastNodeStateSeenByProvider() const { diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h index eb48640c97b..67ee4209d35 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.h +++ b/storage/src/vespa/storage/common/content_bucket_space.h @@ -54,14 +54,6 @@ public: void set_state_and_distribution(std::shared_ptr<const ClusterStateAndDistribution> state_and_distr) noexcept; [[nodiscard]] std::shared_ptr<const ClusterStateAndDistribution> state_and_distribution() const noexcept; - // TODO deprecate; only use atomic state+distribution setter - void setClusterState(std::shared_ptr<const lib::ClusterState> clusterState); - // TODO deprecate; only use atomic state+distribution getter - std::shared_ptr<const lib::ClusterState> getClusterState() const; - // TODO deprecate; only use atomic state+distribution setter - void setDistribution(std::shared_ptr<const lib::Distribution> distribution); - // TODO deprecate; only use atomic state+distribution getter - std::shared_ptr<const lib::Distribution> getDistribution() const; bool getNodeUpInLastNodeStateSeenByProvider() const; void setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider); diff --git a/storage/src/vespa/storage/common/storagecomponent.h b/storage/src/vespa/storage/common/storagecomponent.h index 677d5652fe3..ba12fc666a9 100644 --- a/storage/src/vespa/storage/common/storagecomponent.h +++ b/storage/src/vespa/storage/common/storagecomponent.h @@ -61,7 +61,7 @@ public: const std::shared_ptr<const document::FieldSetRepo> fieldSetRepo; }; using UP = std::unique_ptr<StorageComponent>; - using DistributionSP = std::shared_ptr<lib::Distribution>; + using DistributionSP = std::shared_ptr<const lib::Distribution>; /** * Node type is supposed to be set immediately, and never be updated. diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp index ab1cbf0b4d7..4d1daa329fd 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp @@ -23,11 +23,9 @@ ServiceLayerComponentRegisterImpl::registerServiceLayerComponent(ServiceLayerMan } void -ServiceLayerComponentRegisterImpl::setDistribution(std::shared_ptr<lib::Distribution> distribution) +ServiceLayerComponentRegisterImpl::setDistribution(std::shared_ptr<const lib::Distribution> distribution) { - _bucketSpaceRepo.get(document::FixedBucketSpaces::default_space()).setDistribution(distribution); - auto global_distr = lib::GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); - _bucketSpaceRepo.get(document::FixedBucketSpaces::global_space()).setDistribution(global_distr); + // TODO remove this override entirely? StorageComponentRegisterImpl::setDistribution(distribution); } diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index 1589192b92e..3478c0b3b9c 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -34,7 +34,7 @@ public: } void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; - void setDistribution(std::shared_ptr<lib::Distribution> distribution) override; + void setDistribution(std::shared_ptr<const lib::Distribution> distribution) override; }; } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp index bd0853a3524..ab1d80dc0b9 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp @@ -91,7 +91,7 @@ StorageComponentRegisterImpl::setBucketIdFactory(const document::BucketIdFactory } void -StorageComponentRegisterImpl::setDistribution(std::shared_ptr<lib::Distribution> distribution) +StorageComponentRegisterImpl::setDistribution(std::shared_ptr<const lib::Distribution> distribution) { std::lock_guard lock(_componentLock); _distribution = distribution; diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h index abb60051fe1..271b38399b6 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h @@ -21,16 +21,16 @@ class StorageComponentRegisterImpl { using BucketspacesConfig = vespa::config::content::core::internal::InternalBucketspacesType; - std::mutex _componentLock; - std::vector<StorageComponent*> _components; - vespalib::string _clusterName; - const lib::NodeType* _nodeType; - uint16_t _index; + std::mutex _componentLock; + std::vector<StorageComponent*> _components; + vespalib::string _clusterName; + const lib::NodeType* _nodeType; + uint16_t _index; std::shared_ptr<const document::DocumentTypeRepo> _docTypeRepo; - document::BucketIdFactory _bucketIdFactory; - std::shared_ptr<lib::Distribution> _distribution; - NodeStateUpdater* _nodeStateUpdater; - BucketspacesConfig _bucketSpacesConfig; + document::BucketIdFactory _bucketIdFactory; + std::shared_ptr<const lib::Distribution> _distribution; + NodeStateUpdater* _nodeStateUpdater; + BucketspacesConfig _bucketSpacesConfig; public: using UP = std::unique_ptr<StorageComponentRegisterImpl>; @@ -38,12 +38,12 @@ public: StorageComponentRegisterImpl(); ~StorageComponentRegisterImpl() override; - const lib::NodeType& getNodeType() const { return *_nodeType; } - uint16_t getIndex() const { return _index; } - std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() { return _docTypeRepo; } - const document::BucketIdFactory& getBucketIdFactory() { return _bucketIdFactory; } - std::shared_ptr<lib::Distribution> & getDistribution() { return _distribution; } - NodeStateUpdater& getNodeStateUpdater() { return *_nodeStateUpdater; } + [[nodiscard]] const lib::NodeType& getNodeType() const noexcept { return *_nodeType; } + [[nodiscard]] uint16_t getIndex() const noexcept { return _index; } + [[nodiscard]] std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() const noexcept { return _docTypeRepo; } + [[nodiscard]] const document::BucketIdFactory& getBucketIdFactory() const noexcept { return _bucketIdFactory; } + [[nodiscard]] const std::shared_ptr<const lib::Distribution>& getDistribution() const noexcept { return _distribution; } + [[nodiscard]] NodeStateUpdater& getNodeStateUpdater() noexcept { return *_nodeStateUpdater; } void registerStorageComponent(StorageComponent&) override; @@ -51,7 +51,7 @@ public: virtual void setNodeStateUpdater(NodeStateUpdater& updater); virtual void setDocumentTypeRepo(std::shared_ptr<const document::DocumentTypeRepo>); virtual void setBucketIdFactory(const document::BucketIdFactory&); - virtual void setDistribution(std::shared_ptr<lib::Distribution>); + virtual void setDistribution(std::shared_ptr<const lib::Distribution>); virtual void setBucketSpacesConfig(const BucketspacesConfig&); }; diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp index d5de11c7d6f..f571f6272b8 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp @@ -16,13 +16,14 @@ using document::BucketSpace; namespace storage { uint16_t -BucketOwnershipNotifier::getOwnerDistributorForBucket(const document::Bucket &bucket) const +BucketOwnershipNotifier::getOwnerDistributorForBucket(const document::Bucket& bucket) const { try { - // TODO use state updater bundle for everything? - auto distribution(_component.getBucketSpaceRepo().get(bucket.getBucketSpace()).getDistribution()); - const auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); - const auto &clusterState = *clusterStateBundle->getDerivedClusterState(bucket.getBucketSpace()); + const auto state_bundle = _component.getStateUpdater().getClusterStateBundle(); + assert(state_bundle && state_bundle->has_distribution_config()); + const auto* distribution = state_bundle->distribution_config_bundle()->bucket_space_distribution_or_nullptr_raw(bucket.getBucketSpace()); + assert(distribution); + const auto& clusterState = *state_bundle->getDerivedClusterState(bucket.getBucketSpace()); return (distribution->getIdealDistributorNode(clusterState, bucket.getBucketId())); // If we get exceptions there aren't any distributors, so they'll have // to explicitly fetch all bucket info eventually anyway. @@ -39,7 +40,7 @@ BucketOwnershipNotifier::getOwnerDistributorForBucket(const document::Bucket &bu } bool -BucketOwnershipNotifier::distributorOwns(uint16_t distributor, const document::Bucket &bucket) const +BucketOwnershipNotifier::distributorOwns(uint16_t distributor, const document::Bucket& bucket) const { return (distributor == getOwnerDistributorForBucket(bucket)); } @@ -47,7 +48,7 @@ BucketOwnershipNotifier::distributorOwns(uint16_t distributor, const document::B void BucketOwnershipNotifier::sendNotifyBucketToDistributor( uint16_t distributorIndex, - const document::Bucket &bucket, + const document::Bucket& bucket, const api::BucketInfo& infoToSend) { if (!infoToSend.valid()) { @@ -71,7 +72,7 @@ BucketOwnershipNotifier::sendNotifyBucketToDistributor( } void -BucketOwnershipNotifier::logNotification(const document::Bucket &bucket, +BucketOwnershipNotifier::logNotification(const document::Bucket& bucket, uint16_t sourceIndex, uint16_t currentOwnerIndex, const api::BucketInfo& newInfo) @@ -88,7 +89,7 @@ BucketOwnershipNotifier::logNotification(const document::Bucket &bucket, void BucketOwnershipNotifier::notifyIfOwnershipChanged( - const document::Bucket &bucket, + const document::Bucket& bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend) { @@ -111,7 +112,7 @@ BucketOwnershipNotifier::notifyIfOwnershipChanged( void BucketOwnershipNotifier::sendNotifyBucketToCurrentOwner( - const document::Bucket &bucket, + const document::Bucket& bucket, const api::BucketInfo& infoToSend) { uint16_t distributor(getOwnerDistributorForBucket(bucket)); @@ -134,7 +135,7 @@ NotificationGuard::~NotificationGuard() } void -NotificationGuard::notifyIfOwnershipChanged(const document::Bucket &bucket, +NotificationGuard::notifyIfOwnershipChanged(const document::Bucket& bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend) { @@ -142,7 +143,7 @@ NotificationGuard::notifyIfOwnershipChanged(const document::Bucket &bucket, } void -NotificationGuard::notifyAlways(const document::Bucket &bucket, +NotificationGuard::notifyAlways(const document::Bucket& bucket, const api::BucketInfo& infoToSend) { BucketToCheck bc(bucket, 0xffff, infoToSend); diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.h b/storage/src/vespa/storage/persistence/bucketownershipnotifier.h index fc3d9209f5f..93a2c1a2104 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.h +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.h @@ -11,29 +11,29 @@ namespace storage { class BucketOwnershipNotifier { - const ServiceLayerComponent & _component; - MessageSender & _sender; + const ServiceLayerComponent& _component; + MessageSender& _sender; public: BucketOwnershipNotifier(const ServiceLayerComponent& component, MessageSender& sender) : _component(component), _sender(sender) {} - bool distributorOwns(uint16_t distributor, const document::Bucket &bucket) const; - void notifyIfOwnershipChanged(const document::Bucket &bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend); - void sendNotifyBucketToCurrentOwner(const document::Bucket &bucket, const api::BucketInfo& infoToSend); + [[nodiscard]] bool distributorOwns(uint16_t distributor, const document::Bucket& bucket) const; + void notifyIfOwnershipChanged(const document::Bucket& bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend); + void sendNotifyBucketToCurrentOwner(const document::Bucket& bucket, const api::BucketInfo& infoToSend); private: enum IndexMeta { FAILED_TO_RESOLVE = 0xffff }; - void sendNotifyBucketToDistributor(uint16_t distributorIndex, const document::Bucket &bucket, + void sendNotifyBucketToDistributor(uint16_t distributorIndex, const document::Bucket& bucket, const api::BucketInfo& infoToSend); // Returns either index or FAILED_TO_RESOLVE uint16_t getOwnerDistributorForBucket(const document::Bucket &bucket) const; - void logNotification(const document::Bucket &bucket, uint16_t sourceIndex, + void logNotification(const document::Bucket& bucket, uint16_t sourceIndex, uint16_t currentOwnerIndex, const api::BucketInfo& newInfo); }; @@ -60,7 +60,7 @@ class NotificationGuard BucketOwnershipNotifier& _notifier; std::vector<BucketToCheck> _bucketsToCheck; public: - NotificationGuard(BucketOwnershipNotifier& notifier) + explicit NotificationGuard(BucketOwnershipNotifier& notifier) : _notifier(notifier), _bucketsToCheck() {} @@ -69,8 +69,8 @@ public: ~NotificationGuard(); - void notifyIfOwnershipChanged(const document::Bucket &bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend); - void notifyAlways(const document::Bucket &bucket, const api::BucketInfo& infoToSend); + void notifyIfOwnershipChanged(const document::Bucket& bucket, uint16_t sourceIndex, const api::BucketInfo& infoToSend); + void notifyAlways(const document::Bucket& bucket, const api::BucketInfo& infoToSend); }; } // storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 495497d507d..e44490388e2 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -887,9 +887,9 @@ bool FileStorManager::maintenance_in_all_spaces(const lib::Node& node) const noexcept { for (const auto& elem : _component.getBucketSpaceRepo()) { - const ContentBucketSpace& bucket_space = *elem.second; - auto derived_cluster_state = bucket_space.getClusterState(); - if (!derived_cluster_state->getNodeState(node).getState().oneOf("m")) { + const auto space_state_and_distr = elem.second->state_and_distribution(); + const auto& derived_cluster_state = space_state_and_distr->cluster_state(); + if (!derived_cluster_state.getNodeState(node).getState().oneOf("m")) { return false; } } @@ -915,8 +915,7 @@ FileStorManager::maybe_log_received_cluster_state() { if (LOG_WOULD_LOG(debug)) { auto cluster_state_bundle = _component.getStateUpdater().getClusterStateBundle(); - auto baseline_state = cluster_state_bundle->getBaselineClusterState(); - LOG(debug, "FileStorManager received baseline cluster state '%s'", baseline_state->toString().c_str()); + LOG(debug, "FileStorManager received baseline cluster state '%s'", cluster_state_bundle->toString().c_str()); } } @@ -951,16 +950,17 @@ FileStorManager::updateState() void FileStorManager::storageDistributionChanged() { - updateState(); } void FileStorManager::propagateClusterStates() { auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); - for (const auto &elem : _component.getBucketSpaceRepo()) { - // TODO also distribution! bundle and repo must be 1-1 - elem.second->setClusterState(clusterStateBundle->getDerivedClusterState(elem.first)); + assert(clusterStateBundle->has_distribution_config()); + for (const auto& elem : _component.getBucketSpaceRepo()) { + elem.second->set_state_and_distribution(std::make_shared<ClusterStateAndDistribution>( + clusterStateBundle->getDerivedClusterState(elem.first), + clusterStateBundle->bucket_space_distribution_or_nullptr(elem.first))); } } diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp index 25829f3d391..8702cb1272b 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp @@ -31,10 +31,11 @@ ChangedBucketOwnershipHandler::ChangedBucketOwnershipHandler( _stateLock(), _currentState(), // Not set yet, so ownership will not be valid _currentOwnership(std::make_shared<OwnershipState>( - _component.getBucketSpaceRepo(), _currentState)), + _currentState, lib::DistributionConfigBundle::of(_component.getDistribution()))), _abortQueuedAndPendingOnStateChange(false), _abortMutatingIdealStateOps(false), - _abortMutatingExternalLoadOps(false) + _abortMutatingExternalLoadOps(false), + _receiving_distribution_config_from_cc(false) { on_configure(bootstrap_config); _component.registerMetric(_metrics); @@ -61,16 +62,29 @@ ChangedBucketOwnershipHandler::reloadClusterState() { std::lock_guard guard(_stateLock); const auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); - setCurrentOwnershipWithStateNoLock(*clusterStateBundle); + setCurrentOwnershipWithStateNoLock(std::move(clusterStateBundle)); } void -ChangedBucketOwnershipHandler::setCurrentOwnershipWithStateNoLock( - const lib::ClusterStateBundle& newState) +ChangedBucketOwnershipHandler::setCurrentOwnershipWithStateNoLock(std::shared_ptr<const lib::ClusterStateBundle> new_state) { - _currentState = std::make_shared<const lib::ClusterStateBundle>(newState); - _currentOwnership = std::make_shared<const OwnershipState>( - _component.getBucketSpaceRepo(), _currentState); + LOG(debug, "Setting new ownership state bundle: %s", new_state->toString().c_str()); + std::shared_ptr<const lib::DistributionConfigBundle> distributions; + _currentState = std::move(new_state); + // This partially duplicates distribution config fallback logic from StateManager, but that's because + // this component has the same approach to intercepting both state commands and distribution config + // changes. + // `new_state` can come straight from a SetSystemStateCommand, which may or may not have a state bundle + // with distribution config strapped to it. + if (_currentState->has_distribution_config()) { + distributions = _currentState->distribution_config_bundle(); + } else { + distributions = lib::DistributionConfigBundle::of(_component.getDistribution()); + LOG(debug, "No distribution config in bundle; using current host config of '%s'", + distributions->default_distribution().getNodeGraph().getDistributionConfigHash().c_str()); + } + _receiving_distribution_config_from_cc = _currentState->has_distribution_config(); + _currentOwnership = std::make_shared<const OwnershipState>(_currentState, std::move(distributions)); } namespace { @@ -98,17 +112,11 @@ ChangedBucketOwnershipHandler::Metrics::Metrics(metrics::MetricSet* owner) {} ChangedBucketOwnershipHandler::Metrics::~Metrics() = default; -ChangedBucketOwnershipHandler::OwnershipState::OwnershipState(const ContentBucketSpaceRepo &contentBucketSpaceRepo, - std::shared_ptr<const lib::ClusterStateBundle> state) - : _distributions(), - _state(std::move(state)) +ChangedBucketOwnershipHandler::OwnershipState::OwnershipState(std::shared_ptr<const lib::ClusterStateBundle> state, + std::shared_ptr<const lib::DistributionConfigBundle> distributions) + : _state(std::move(state)), + _distributions(std::move(distributions)) { - for (const auto &elem : contentBucketSpaceRepo) { - auto distribution = elem.second->getDistribution(); - if (distribution) { - _distributions.emplace(elem.first, std::move(distribution)); - } - } } @@ -123,19 +131,15 @@ ChangedBucketOwnershipHandler::OwnershipState::getBaselineState() const } uint16_t -ChangedBucketOwnershipHandler::OwnershipState::ownerOf( - const document::Bucket& bucket) const +ChangedBucketOwnershipHandler::OwnershipState::ownerOf(const document::Bucket& bucket) const { - auto distributionItr = _distributions.find(bucket.getBucketSpace()); - assert(distributionItr != _distributions.end()); - const auto &distribution = *distributionItr->second; - const auto &derivedState = *_state->getDerivedClusterState(bucket.getBucketSpace()); + const auto* distribution = _distributions->bucket_space_distribution_or_nullptr_raw(bucket.getBucketSpace()); + assert(distribution); + const auto& derivedState = *_state->getDerivedClusterState(bucket.getBucketSpace()); try { - return distribution.getIdealDistributorNode(derivedState, bucket.getBucketId()); + return distribution->getIdealDistributorNode(derivedState, bucket.getBucketId()); } catch (lib::TooFewBucketBitsInUseException& e) { - LOGBP(debug, - "Too few bucket bits used for %s to be assigned to " - "a distributor.", + LOGBP(debug, "Too few bucket bits used for %s to be assigned to a distributor.", bucket.toString().c_str()); } catch (lib::NoDistributorsAvailableException& e) { LOGBP(warning, @@ -144,11 +148,9 @@ ChangedBucketOwnershipHandler::OwnershipState::ownerOf( "for available distributors before reaching this code path! " "Cluster state is '%s', distribution is '%s'", derivedState.toString().c_str(), - distribution.toString().c_str()); + distribution->toString().c_str()); } catch (const std::exception& e) { - LOG(error, - "Got unknown exception while resolving distributor: %s", - e.what()); + LOG(error, "Got unknown exception while resolving distributor: %s", e.what()); } return FAILED_TO_RESOLVE; } @@ -162,9 +164,7 @@ ChangedBucketOwnershipHandler::OwnershipState::storageNodeUp(document::BucketSpa } void -ChangedBucketOwnershipHandler::logTransition( - const lib::ClusterState& currentState, - const lib::ClusterState& newState) const +ChangedBucketOwnershipHandler::logTransition(const lib::ClusterState& currentState, const lib::ClusterState& newState) { LOG(debug, "State transition '%s' -> '%s' changes distributor bucket ownership, " @@ -269,7 +269,7 @@ public: { std::lock_guard guard(_owner._stateLock); old_ownership = _owner._currentOwnership; - _owner.setCurrentOwnershipWithStateNoLock(_command->getClusterStateBundle()); + _owner.setCurrentOwnershipWithStateNoLock(_command->cluster_state_bundle_ptr()); new_ownership = _owner._currentOwnership; } assert(new_ownership->valid()); @@ -287,7 +287,7 @@ public: new_ownership->getBaselineState().toString().c_str()); return _owner.sendDown(_command);; } - _owner.logTransition(old_ownership->getBaselineState(), new_ownership->getBaselineState()); + logTransition(old_ownership->getBaselineState(), new_ownership->getBaselineState()); metrics::MetricTimer duration_timer; auto predicate = _owner.makeLazyAbortPredicate(old_ownership, new_ownership); @@ -327,17 +327,19 @@ ChangedBucketOwnershipHandler::onSetSystemState( * Invoked whenever a distribution config change happens and is called in the * context of the config updater thread (which is why we have to lock). */ + // TODO remove this when there are no more state bundles without distribution config void ChangedBucketOwnershipHandler::storageDistributionChanged() { std::lock_guard guard(_stateLock); - _currentOwnership = std::make_shared<OwnershipState>( - _component.getBucketSpaceRepo(), _currentState); + if (!_receiving_distribution_config_from_cc) { + _currentOwnership = std::make_shared<OwnershipState>( + _currentState, lib::DistributionConfigBundle::of(_component.getDistribution())); + } } bool -ChangedBucketOwnershipHandler::isMutatingIdealStateOperation( - const api::StorageMessage& msg) const +ChangedBucketOwnershipHandler::isMutatingIdealStateOperation(const api::StorageMessage& msg) { switch (msg.getType().getId()) { case api::MessageType::CREATEBUCKET_ID: @@ -357,8 +359,7 @@ ChangedBucketOwnershipHandler::isMutatingIdealStateOperation( bool -ChangedBucketOwnershipHandler::isMutatingExternalOperation( - const api::StorageMessage& msg) const +ChangedBucketOwnershipHandler::isMutatingExternalOperation(const api::StorageMessage& msg) { switch (msg.getType().getId()) { case api::MessageType::PUT_ID: @@ -418,8 +419,7 @@ ChangedBucketOwnershipHandler::abortOperation(api::StorageCommand& cmd) } bool -ChangedBucketOwnershipHandler::isMutatingCommandAndNeedsChecking( - const api::StorageMessage& msg) const +ChangedBucketOwnershipHandler::isMutatingCommandAndNeedsChecking(const api::StorageMessage& msg) const { if (enabledIdealStateAborting() && isMutatingIdealStateOperation(msg)) { return true; diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h index 801534385f7..2c32704d64b 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h @@ -25,6 +25,7 @@ namespace lib { class ClusterState; class ClusterStateBundle; class Distribution; + class DistributionConfigBundle; } /** @@ -77,20 +78,20 @@ public: */ class OwnershipState { using BucketSpace = document::BucketSpace; - std::unordered_map<BucketSpace, std::shared_ptr<const lib::Distribution>, BucketSpace::hash> _distributions; std::shared_ptr<const lib::ClusterStateBundle> _state; + std::shared_ptr<const lib::DistributionConfigBundle> _distributions; public: using SP = std::shared_ptr<OwnershipState>; using CSP = std::shared_ptr<const OwnershipState>; - OwnershipState(const ContentBucketSpaceRepo &contentBucketSpaceRepo, - std::shared_ptr<const lib::ClusterStateBundle> state); + OwnershipState(std::shared_ptr<const lib::ClusterStateBundle> state, + std::shared_ptr<const lib::DistributionConfigBundle> distributions); ~OwnershipState(); static const uint16_t FAILED_TO_RESOLVE = 0xffff; [[nodiscard]] bool valid() const noexcept { - return (!_distributions.empty() && _state); + return (_distributions && _state); } /** @@ -124,20 +125,21 @@ private: std::atomic<bool> _abortQueuedAndPendingOnStateChange; std::atomic<bool> _abortMutatingIdealStateOps; std::atomic<bool> _abortMutatingExternalLoadOps; + bool _receiving_distribution_config_from_cc; std::unique_ptr<AbortBucketOperationsCommand::AbortPredicate> makeLazyAbortPredicate( const OwnershipState::CSP& oldOwnership, const OwnershipState::CSP& newOwnership) const; - void logTransition(const lib::ClusterState& currentState, - const lib::ClusterState& newState) const; + static void logTransition(const lib::ClusterState& currentState, + const lib::ClusterState& newState); /** * Creates a new immutable OwnershipState based on the current distribution * and the provided cluster state and assigns it to _currentOwnership. */ - void setCurrentOwnershipWithStateNoLock(const lib::ClusterStateBundle&); + void setCurrentOwnershipWithStateNoLock(std::shared_ptr<const lib::ClusterStateBundle>); /** * Grabs _stateLock and returns a shared_ptr to the current ownership @@ -147,9 +149,9 @@ private: bool isMutatingCommandAndNeedsChecking(const api::StorageMessage&) const; - bool isMutatingIdealStateOperation(const api::StorageMessage&) const; + static bool isMutatingIdealStateOperation(const api::StorageMessage&); - bool isMutatingExternalOperation(const api::StorageMessage&) const; + static bool isMutatingExternalOperation(const api::StorageMessage&); /** * Returns whether the operation in cmd has a bucket whose ownership in * the current cluster state does not match the distributor marked as diff --git a/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h b/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h index 81bd0897dbb..53b613ce2ab 100644 --- a/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h @@ -21,8 +21,8 @@ class ClusterStateBundleCodec { public: virtual ~ClusterStateBundleCodec() = default; - virtual EncodedClusterStateBundle encode(const lib::ClusterStateBundle&) const = 0; - virtual std::shared_ptr<const lib::ClusterStateBundle> decode(const EncodedClusterStateBundle&) const = 0; + [[nodiscard]] virtual EncodedClusterStateBundle encode(const lib::ClusterStateBundle&) const = 0; + [[nodiscard]] virtual std::shared_ptr<const lib::ClusterStateBundle> decode(const EncodedClusterStateBundle&) const = 0; }; } diff --git a/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp index 38d3f929549..f143066ac34 100644 --- a/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp @@ -1,9 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "slime_cluster_state_bundle_codec.h" +#include <vespa/config-stor-distribution.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/configgen/configpayload.h> +#include <vespa/config/print/configdatabuffer.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/vdslib/state/clusterstate.h> #include <vespa/vdslib/state/cluster_state_bundle.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/data/slime/array_traverser.h> +#include <vespa/vespalib/data/slime/object_traverser.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> @@ -17,6 +23,7 @@ using vespalib::compression::CompressionConfig; using vespalib::compression::decompress; using vespalib::compression::compress; using vespalib::Memory; +using DistributionConfigBuilder = storage::lib::Distribution::DistributionConfigBuilder; using namespace vespalib::slime; namespace storage::rpc { @@ -48,21 +55,94 @@ vespalib::string serialize_state(const lib::ClusterState& state) { return as.str(); } -const Memory StatesField("states"); const Memory BaselineField("baseline"); -const Memory SpacesField("spaces"); -const Memory DeferredActivationField("deferred-activation"); -const Memory FeedBlockField("feed-block"); const Memory BlockFeedInClusterField("block-feed-in-cluster"); +const Memory DeferredActivationField("deferred-activation"); const Memory DescriptionField("description"); +const Memory DistributionConfigField("distribution-config"); +const Memory FeedBlockField("feed-block"); +const Memory SpacesField("spaces"); +const Memory StatesField("states"); + +// Important: these conversion routines are NOT complete and NOT general! They are only to be used +// by code transitively used by unit tests that expect a particular type subset and "shape" of config. + +void convert_struct(const Inspector& in, Cursor& out); + +struct ConfigArrayConverter : ArrayTraverser { + Cursor& _out; + explicit ConfigArrayConverter(Cursor& out) noexcept: _out(out) {} + + void entry([[maybe_unused]] size_t idx, const Inspector& in) override { + assert(in.type().getId() == OBJECT::ID); + auto type = in["type"].asString(); + auto& value = in["value"]; + assert(value.valid()); + if (type == "int") { + _out.addLong(value.asLong()); + } else if (type == "bool") { + _out.addBool(value.asBool()); + } else if (type == "string") { + _out.addString(value.asString()); + } else if (type == "double") { + _out.addDouble(value.asDouble()); + } else if (type == "array") { + assert(value.type().getId() == ARRAY::ID); + ConfigArrayConverter arr_conv(_out.addArray()); + value.traverse(arr_conv); + } else if (type == "struct") { + convert_struct(value, _out.addObject()); + } else { + fprintf(stderr, "unknown array entry type '%s'\n", type.make_string().c_str()); + abort(); + } + } +}; + +struct ConfigObjectConverter : ObjectTraverser { + Cursor& _out; + explicit ConfigObjectConverter(Cursor& out) noexcept: _out(out) {} + + void field(const Memory& name, const Inspector& in) override { + assert(in.type().getId() == OBJECT::ID); + auto type = in["type"].asString(); + auto& value = in["value"]; + assert(value.valid()); + if (type == "int") { + _out.setLong(name, value.asLong()); + } else if (type == "bool") { + _out.setBool(name, value.asBool()); + } else if (type == "string") { + _out.setString(name, value.asString()); + } else if (type == "double") { + _out.setDouble(name, value.asDouble()); + } else if (type == "array") { + assert(value.type().getId() == ARRAY::ID); + ConfigArrayConverter arr_conv(_out.setArray(name)); + value.traverse(arr_conv); + } else if (type == "struct") { + convert_struct(value, _out.setObject(name)); + } else { + fprintf(stderr, "unknown struct entry type '%s'\n", type.make_string().c_str()); + abort(); + } + } +}; + +void convert_struct(const Inspector& in, Cursor& out) { + ConfigObjectConverter conv(out); + in.traverse(conv); +} +void convert_to_config_payload(const Inspector& in, Cursor& out) { + convert_struct(in["configPayload"], out); } +} // anon ns + // Only used from unit tests; the cluster controller encodes all bundles // we decode in practice. -EncodedClusterStateBundle SlimeClusterStateBundleCodec::encode( - const lib::ClusterStateBundle& bundle) const -{ +EncodedClusterStateBundle SlimeClusterStateBundleCodec::encode(const lib::ClusterStateBundle& bundle) const { vespalib::Slime slime; Cursor& root = slime.setObject(); if (bundle.deferredActivation()) { @@ -81,6 +161,15 @@ EncodedClusterStateBundle SlimeClusterStateBundleCodec::encode( feed_block.setString(DescriptionField, bundle.feed_block()->description()); } + if (bundle.has_distribution_config()) { + Cursor& distr_root = root.setObject(DistributionConfigField); + ::config::ConfigDataBuffer buf; + bundle.distribution_config_bundle()->config().serialize(buf); + // There is no way in C++ to directly serialize to the actual payload format we expect to + // deserialize, so we have to manually convert the type-annotated config snapshot :I + convert_to_config_payload(buf.slimeObject().get(), distr_root); + } + OutputBuf out_buf(4_Ki); BinaryFormat::encode(slime, out_buf); ConstBufferRef to_compress(out_buf.getBuf().getData(), out_buf.getBuf().getDataLen()); @@ -130,7 +219,7 @@ std::shared_ptr<const lib::ClusterStateBundle> SlimeClusterStateBundleCodec::dec BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), slime); Inspector& root = slime.get(); Inspector& states = root[StatesField]; - lib::ClusterState baseline(states[BaselineField].asString().make_string()); + auto baseline = std::make_shared<lib::ClusterState>(states[BaselineField].asString().make_string()); Inspector& spaces = states[SpacesField]; lib::ClusterStateBundle::BucketSpaceStateMapping space_states; @@ -138,16 +227,21 @@ std::shared_ptr<const lib::ClusterStateBundle> SlimeClusterStateBundleCodec::dec spaces.traverse(inserter); const bool deferred_activation = root[DeferredActivationField].asBool(); // Defaults to false if not set. + std::shared_ptr<const lib::DistributionConfigBundle> distribution_config; + std::optional<lib::ClusterStateBundle::FeedBlock> feed_block; Inspector& fb = root[FeedBlockField]; if (fb.valid()) { - lib::ClusterStateBundle::FeedBlock feed_block(fb[BlockFeedInClusterField].asBool(), - fb[DescriptionField].asString().make_string()); - return std::make_shared<lib::ClusterStateBundle>(baseline, std::move(space_states), feed_block, deferred_activation); + feed_block = lib::ClusterStateBundle::FeedBlock(fb[BlockFeedInClusterField].asBool(), + fb[DescriptionField].asString().make_string()); } - - // TODO add shared_ptr constructor for baseline? - return std::make_shared<lib::ClusterStateBundle>(baseline, std::move(space_states), deferred_activation); + Inspector& dc = root[DistributionConfigField]; + if (dc.valid()) { + auto raw_cfg = std::make_unique<DistributionConfigBuilder>(::config::ConfigPayload(dc)); + distribution_config = lib::DistributionConfigBundle::of(std::move(raw_cfg)); + } + return std::make_shared<lib::ClusterStateBundle>(std::move(baseline), std::move(space_states), std::move(feed_block), + std::move(distribution_config), deferred_activation); } } diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index a2106dce8d2..560982fb6c0 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -29,7 +29,18 @@ using vespalib::make_string_short::fmt; namespace storage { namespace { - constexpr vespalib::duration MAX_TIMEOUT = 600s; +constexpr vespalib::duration MAX_TIMEOUT = 600s; + +[[nodiscard]] std::shared_ptr<const lib::ClusterStateBundle> +make_bootstrap_state_bundle(std::shared_ptr<const lib::Distribution> config) { + return std::make_shared<const lib::ClusterStateBundle>( + std::make_shared<lib::ClusterState>(), + lib::ClusterStateBundle::BucketSpaceStateMapping{}, + std::nullopt, + lib::DistributionConfigBundle::of(std::move(config)), + false); +} + } struct StateManager::StateManagerMetrics : metrics::MetricSet { @@ -62,7 +73,8 @@ StateManager::StateManager(StorageComponentRegister& compReg, _listenerLock(), _nodeState(std::make_shared<lib::NodeState>(_component.getNodeType(), lib::State::DOWN)), _nextNodeState(), - _systemState(std::make_shared<const ClusterStateBundle>(lib::ClusterState())), + _configured_distribution(_component.getDistribution()), + _systemState(make_bootstrap_state_bundle(_configured_distribution)), _nextSystemState(), _reported_host_info_cluster_state_version(0), _stateListeners(), @@ -81,6 +93,7 @@ StateManager::StateManager(StorageComponentRegister& compReg, _noThreadTestMode(testMode), _grabbedExternalLock(false), _require_strictly_increasing_cluster_state_versions(false), + _receiving_distribution_config_from_cc(false), _notifyingListeners(false), _requested_almost_immediate_node_state_replies(false) { @@ -170,8 +183,7 @@ lib::NodeState::CSP StateManager::getCurrentNodeState() const { std::lock_guard lock(_stateLock); - return std::make_shared<const lib::NodeState> - (_systemState->getBaselineClusterState()->getNodeState(thisNode())); + return std::make_shared<const lib::NodeState>(_systemState->getBaselineClusterState()->getNodeState(thisNode())); } std::shared_ptr<const lib::ClusterStateBundle> @@ -181,6 +193,28 @@ StateManager::getClusterStateBundle() const return _systemState; } +// TODO remove when distribution config is only received from cluster controller +void +StateManager::storageDistributionChanged() +{ + { + std::lock_guard lock(_stateLock); + _configured_distribution = _component.getDistribution(); + if (_receiving_distribution_config_from_cc) { + return; // nothing more to do + } + // Avoid losing any pending state if this callback happens in the middle of a + // state update. This edge case is practically impossible to unit test today... + const auto patch_state = _nextSystemState ? _nextSystemState : _systemState; + _nextSystemState = patch_state->clone_with_new_distribution( + lib::DistributionConfigBundle::of(_configured_distribution)); + } + // We've assembled a new state bundle based on the (non-distribution carrying) state + // bundle from the cluster controller and our own internal config. Propagate it as one + // unit to the internal components. + notifyStateListeners(); +} + void StateManager::addStateListener(StateListener& listener) { @@ -316,12 +350,12 @@ using BucketSpaceToTransitionString = std::unordered_map<document::BucketSpace, document::BucketSpace::hash>; void -considerInsertDerivedTransition(const lib::State ¤tBaseline, - const lib::State &newBaseline, - const lib::State ¤tDerived, - const lib::State &newDerived, - const document::BucketSpace &bucketSpace, - BucketSpaceToTransitionString &transitions) +considerInsertDerivedTransition(const lib::State& currentBaseline, + const lib::State& newBaseline, + const lib::State& currentDerived, + const lib::State& newDerived, + const document::BucketSpace& bucketSpace, + BucketSpaceToTransitionString& transitions) { bool considerDerivedTransition = ((currentDerived != newDerived) && ((currentDerived != currentBaseline) || (newDerived != newBaseline))); @@ -333,28 +367,28 @@ considerInsertDerivedTransition(const lib::State ¤tBaseline, } BucketSpaceToTransitionString -calculateDerivedClusterStateTransitions(const ClusterStateBundle ¤tState, - const ClusterStateBundle &newState, +calculateDerivedClusterStateTransitions(const ClusterStateBundle& currentState, + const ClusterStateBundle& newState, const lib::Node node) { BucketSpaceToTransitionString result; - const lib::State ¤tBaseline = currentState.getBaselineClusterState()->getNodeState(node).getState(); - const lib::State &newBaseline = newState.getBaselineClusterState()->getNodeState(node).getState(); - for (const auto &entry : currentState.getDerivedClusterStates()) { - const lib::State ¤tDerived = entry.second->getNodeState(node).getState(); - const lib::State &newDerived = newState.getDerivedClusterState(entry.first)->getNodeState(node).getState(); + const lib::State& currentBaseline = currentState.getBaselineClusterState()->getNodeState(node).getState(); + const lib::State& newBaseline = newState.getBaselineClusterState()->getNodeState(node).getState(); + for (const auto& entry : currentState.getDerivedClusterStates()) { + const lib::State& currentDerived = entry.second->getNodeState(node).getState(); + const lib::State& newDerived = newState.getDerivedClusterState(entry.first)->getNodeState(node).getState(); considerInsertDerivedTransition(currentBaseline, newBaseline, currentDerived, newDerived, entry.first, result); } - for (const auto &entry : newState.getDerivedClusterStates()) { - const lib::State &newDerived = entry.second->getNodeState(node).getState(); - const lib::State ¤tDerived = currentState.getDerivedClusterState(entry.first)->getNodeState(node).getState(); + for (const auto& entry : newState.getDerivedClusterStates()) { + const lib::State& newDerived = entry.second->getNodeState(node).getState(); + const lib::State& currentDerived = currentState.getDerivedClusterState(entry.first)->getNodeState(node).getState(); considerInsertDerivedTransition(currentBaseline, newBaseline, currentDerived, newDerived, entry.first, result); } return result; } vespalib::string -transitionsToString(const BucketSpaceToTransitionString &transitions) +transitionsToString(const BucketSpaceToTransitionString& transitions) { if (transitions.empty()) { return ""; @@ -362,7 +396,7 @@ transitionsToString(const BucketSpaceToTransitionString &transitions) vespalib::asciistream stream; stream << "["; bool first = true; - for (const auto &entry : transitions) { + for (const auto& entry : transitions) { if (!first) { stream << ", "; } @@ -435,7 +469,7 @@ StateManager::onGetNodeState(const api::GetNodeStateCommand::SP& cmd) } void -StateManager::mark_controller_as_having_observed_explicit_node_state(const std::unique_lock<std::mutex> &, uint16_t controller_index) { +StateManager::mark_controller_as_having_observed_explicit_node_state(const std::unique_lock<std::mutex>&, uint16_t controller_index) { _controllers_observed_explicit_node_state.emplace(controller_index); } @@ -483,7 +517,16 @@ StateManager::try_set_cluster_state_bundle(std::shared_ptr<const ClusterStateBun } } _last_accepted_cluster_state_time = now; - _nextSystemState = std::move(c); + _receiving_distribution_config_from_cc = c->has_distribution_config(); + if (!c->has_distribution_config()) { + LOG(debug, "Next state bundle '%s' does not have distribution config; patching in existing config '%s'", + c->toString().c_str(), _configured_distribution->getNodeGraph().getDistributionConfigHash().c_str()); + _nextSystemState = c->clone_with_new_distribution(lib::DistributionConfigBundle::of(_configured_distribution)); + } else { + LOG(debug, "Next state bundle is '%s'", c->toString().c_str()); + // TODO print what's changed in distribution config? + _nextSystemState = std::move(c); + } } notifyStateListeners(); return std::nullopt; diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index d116f968731..d7a139f7ced 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -55,6 +55,7 @@ class StateManager : public NodeStateUpdater, std::mutex _listenerLock; std::shared_ptr<lib::NodeState> _nodeState; std::shared_ptr<lib::NodeState> _nextNodeState; + std::shared_ptr<const lib::Distribution> _configured_distribution; // From config system, not from CC std::shared_ptr<const ClusterStateBundle> _systemState; std::shared_ptr<const ClusterStateBundle> _nextSystemState; uint32_t _reported_host_info_cluster_state_version; @@ -78,6 +79,7 @@ class StateManager : public NodeStateUpdater, bool _noThreadTestMode; bool _grabbedExternalLock; bool _require_strictly_increasing_cluster_state_versions; + bool _receiving_distribution_config_from_cc; std::atomic<bool> _notifyingListeners; std::atomic<bool> _requested_almost_immediate_node_state_replies; @@ -102,6 +104,8 @@ public: lib::NodeState::CSP getCurrentNodeState() const override; std::shared_ptr<const ClusterStateBundle> getClusterStateBundle() const override; + void storageDistributionChanged() override; + void addStateListener(StateListener&) override; void removeStateListener(StateListener&) override; diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index 20d2624609e..7ecb91d387d 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -6,7 +6,7 @@ vespa_define_module( vespa_config vespalib vespa_document - vdslib + vespa_vdslib vespa_searchlib vespa_searchsummary diff --git a/streamingvisitors/src/tests/searcher/searcher_test.cpp b/streamingvisitors/src/tests/searcher/searcher_test.cpp index 84c3a542661..b3e0ad304fe 100644 --- a/streamingvisitors/src/tests/searcher/searcher_test.cpp +++ b/streamingvisitors/src/tests/searcher/searcher_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/searchlib/query/streaming/fuzzy_term.h> diff --git a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp index 8fecff0531d..dd687ec7c31 100644 --- a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp +++ b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/process/process.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/app_vbench/app_vbench_test.cpp b/vbench/src/tests/app_vbench/app_vbench_test.cpp index 1a166f164e2..97ab828fc66 100644 --- a/vbench/src/tests/app_vbench/app_vbench_test.cpp +++ b/vbench/src/tests/app_vbench/app_vbench_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/process/process.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp b/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp index c9af4138142..aa4e9b0b832 100644 --- a/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp +++ b/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/dispatcher/dispatcher_test.cpp b/vbench/src/tests/dispatcher/dispatcher_test.cpp index a3978ec5111..a6a0438b232 100644 --- a/vbench/src/tests/dispatcher/dispatcher_test.cpp +++ b/vbench/src/tests/dispatcher/dispatcher_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp b/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp index 88a852785be..09c2d475224 100644 --- a/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp +++ b/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/handler_thread/handler_thread_test.cpp b/vbench/src/tests/handler_thread/handler_thread_test.cpp index bd36556efa2..637166e6d1a 100644 --- a/vbench/src/tests/handler_thread/handler_thread_test.cpp +++ b/vbench/src/tests/handler_thread/handler_thread_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/util/time.h> diff --git a/vbench/src/tests/hex_number/hex_number_test.cpp b/vbench/src/tests/hex_number/hex_number_test.cpp index 96b55a5791d..7d535116a19 100644 --- a/vbench/src/tests/hex_number/hex_number_test.cpp +++ b/vbench/src/tests/hex_number/hex_number_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/http_client/http_client_test.cpp b/vbench/src/tests/http_client/http_client_test.cpp index fcbea2edd58..468207f4742 100644 --- a/vbench/src/tests/http_client/http_client_test.cpp +++ b/vbench/src/tests/http_client/http_client_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/http_connection/http_connection_test.cpp b/vbench/src/tests/http_connection/http_connection_test.cpp index 0e993eef0b1..f3cc45a58d9 100644 --- a/vbench/src/tests/http_connection/http_connection_test.cpp +++ b/vbench/src/tests/http_connection/http_connection_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp b/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp index 7ed011866e1..66c8899d84e 100644 --- a/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp +++ b/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp b/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp index 7c25f255e14..47d5df354ca 100644 --- a/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp +++ b/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/line_reader/line_reader_test.cpp b/vbench/src/tests/line_reader/line_reader_test.cpp index 7cf81bd10f0..48082f5be78 100644 --- a/vbench/src/tests/line_reader/line_reader_test.cpp +++ b/vbench/src/tests/line_reader/line_reader_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp b/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp index 54cf94a57b1..b14508778f3 100644 --- a/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp +++ b/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/qps_tagger/qps_tagger_test.cpp b/vbench/src/tests/qps_tagger/qps_tagger_test.cpp index 1bec90042e7..80eace04f12 100644 --- a/vbench/src/tests/qps_tagger/qps_tagger_test.cpp +++ b/vbench/src/tests/qps_tagger/qps_tagger_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/request_dumper/request_dumper_test.cpp b/vbench/src/tests/request_dumper/request_dumper_test.cpp index 71925767582..a4cf7dc8f1d 100644 --- a/vbench/src/tests/request_dumper/request_dumper_test.cpp +++ b/vbench/src/tests/request_dumper/request_dumper_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/request_sink/request_sink_test.cpp b/vbench/src/tests/request_sink/request_sink_test.cpp index 334dfdd5142..606aa0577c2 100644 --- a/vbench/src/tests/request_sink/request_sink_test.cpp +++ b/vbench/src/tests/request_sink/request_sink_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/server_spec/server_spec_test.cpp b/vbench/src/tests/server_spec/server_spec_test.cpp index 930d14f824a..49a430184ce 100644 --- a/vbench/src/tests/server_spec/server_spec_test.cpp +++ b/vbench/src/tests/server_spec/server_spec_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/server_tagger/server_tagger_test.cpp b/vbench/src/tests/server_tagger/server_tagger_test.cpp index e824b9fa0f1..32c59aa3177 100644 --- a/vbench/src/tests/server_tagger/server_tagger_test.cpp +++ b/vbench/src/tests/server_tagger/server_tagger_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/socket/socket_test.cpp b/vbench/src/tests/socket/socket_test.cpp index 67588896b28..8ab0bdd9d65 100644 --- a/vbench/src/tests/socket/socket_test.cpp +++ b/vbench/src/tests/socket/socket_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> diff --git a/vbench/src/tests/taint/taint_test.cpp b/vbench/src/tests/taint/taint_test.cpp index 4596f824ffd..cb95dc7da7e 100644 --- a/vbench/src/tests/taint/taint_test.cpp +++ b/vbench/src/tests/taint/taint_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/time_queue/time_queue_test.cpp b/vbench/src/tests/time_queue/time_queue_test.cpp index 4e505822105..46b2d67f684 100644 --- a/vbench/src/tests/time_queue/time_queue_test.cpp +++ b/vbench/src/tests/time_queue/time_queue_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/timer/timer_test.cpp b/vbench/src/tests/timer/timer_test.cpp index 18defccee26..3cc15173135 100644 --- a/vbench/src/tests/timer/timer_test.cpp +++ b/vbench/src/tests/timer/timer_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/util/time.h> diff --git a/vdslib/src/tests/container/CMakeLists.txt b/vdslib/src/tests/container/CMakeLists.txt index 4b5b935092d..d257c327439 100644 --- a/vdslib/src/tests/container/CMakeLists.txt +++ b/vdslib/src/tests/container/CMakeLists.txt @@ -5,6 +5,6 @@ vespa_add_library(vdslib_containertest searchresulttest.cpp documentsummarytest.cpp DEPENDS - vdslib + vespa_vdslib GTest::GTest ) diff --git a/vdslib/src/tests/distribution/CMakeLists.txt b/vdslib/src/tests/distribution/CMakeLists.txt index ad9f9722a0c..2fee0838feb 100644 --- a/vdslib/src/tests/distribution/CMakeLists.txt +++ b/vdslib/src/tests/distribution/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_library(vdslib_testdistribution distributiontest.cpp grouptest.cpp DEPENDS - vdslib + vespa_vdslib GTest::GTest ) diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index 2038c5706f5..9d590c72559 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -11,11 +11,14 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/lexical_cast.h> +#include <vespa/vespalib/test/test_data.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/size_literals.h> #include <gmock/gmock.h> #include <chrono> +#include <filesystem> #include <fstream> #include <iterator> #include <thread> @@ -30,7 +33,35 @@ T readConfig(const config::ConfigUri & uri) return *config::ConfigGetter<T>::getConfig(uri.getConfigId(), uri.getContext()); } -TEST(DistributionTest, test_verify_java_distributions) +class DistributionTest : public ::testing::Test, public vespalib::test::TestData<DistributionTest> { +protected: + DistributionTest(); + ~DistributionTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +DistributionTest::DistributionTest() + : ::testing::Test(), + vespalib::test::TestData<DistributionTest>() +{ +} + +DistributionTest::~DistributionTest() = default; + +void +DistributionTest::SetUpTestSuite() +{ + setup_test_data(TEST_PATH("distribution/testdata"), "distribution-testdata"); +} + +void +DistributionTest::TearDownTestSuite() +{ + tear_down_test_data(); +} + +TEST_F(DistributionTest, test_verify_java_distributions) { std::vector<std::string> tests; tests.push_back("capacity"); @@ -39,16 +70,19 @@ TEST(DistributionTest, test_verify_java_distributions) tests.push_back("retired"); for (uint32_t i=0; i<tests.size(); ++i) { std::string test = tests[i]; + std::string java_cfg(source_testdata() + "/java_" + test + ".cfg"); + std::string java_testfile(source_testdata() + "/java_" + test + ".distribution"); + std::string cpp_testfile(build_testdata() + "/cpp_" + test + ".distribution"); std::string mystate; { - std::ifstream in("distribution/testdata/java_" + test + ".state"); + std::ifstream in(source_testdata() + "/java_" + test + ".state"); in >> mystate; in.close(); } ClusterState state(mystate); Distribution distr(readConfig<vespa::config::content::StorDistributionConfig>( - config::ConfigUri("file:distribution/testdata/java_" + test + ".cfg"))); - std::ofstream of("distribution/testdata/cpp_" + test + ".distribution"); + config::ConfigUri("file:" + java_cfg))); + std::ofstream of(cpp_testfile); long maxBucket = 1; long mask = 0; @@ -74,10 +108,13 @@ TEST(DistributionTest, test_verify_java_distributions) } of.close(); std::ostringstream cmd; - cmd << "diff -u " << "distribution/testdata/cpp_" << test << ".distribution " - << "distribution/testdata/java_" << test << ".distribution"; + + cmd << "diff -u " << cpp_testfile << " " << java_testfile; int result = system(cmd.str().c_str()); - EXPECT_EQ(0, result) << "Failed distribution sync test: " + test; + EXPECT_EQ(0, result) << "Failed distribution sync test: " << test; + if (result == 0) { + std::filesystem::remove(cpp_testfile); + } } } @@ -186,9 +223,9 @@ auto readFile(const std::string & filename) { return buf; } -TEST(DistributionTest, test_verify_java_distributions_2) +TEST_F(DistributionTest, test_verify_java_distributions_2) { - vespalib::DirectoryList files(vespalib::listDirectory("distribution/testdata")); + vespalib::DirectoryList files(vespalib::listDirectory(source_testdata())); for (uint32_t i=0, n=files.size(); i<n; ++i) { size_t pos = files[i].find(".java.results"); if (pos == vespalib::string::npos || pos + 13 != files[i].size()) { @@ -200,14 +237,14 @@ TEST(DistributionTest, test_verify_java_distributions_2) using namespace vespalib::slime; vespalib::Slime slime; - auto buf = readFile("distribution/testdata/" + files[i]); + auto buf = readFile(source_testdata() + "/" + files[i]); auto size = JsonFormat::decode({&buf[0], buf.size()}, slime); if (size == 0) { std::cerr << "\n\nSize of " << files[i] << " is 0. Maybe is not generated yet? Taking a 5 second nap!"; std::this_thread::sleep_for(std::chrono::seconds(5)); - buf = readFile("distribution/testdata/" + files[i]); + buf = readFile(source_testdata() + "/" + files[i]); size = JsonFormat::decode({&buf[0], buf.size()}, slime); if (size == 0) { @@ -245,12 +282,12 @@ TEST(DistributionTest, test_verify_java_distributions_2) } } -TEST(DistributionTest, test_unchanged_distribution) +TEST_F(DistributionTest, test_unchanged_distribution) { ClusterState state("distributor:10 storage:10"); Distribution distr(Distribution::getDefaultDistributionConfig(3, 10)); - std::ifstream in("distribution/testdata/41-distributordistribution"); + std::ifstream in(source_testdata() + "/41-distributordistribution"); for (unsigned i = 0; i < 64_Ki; i++) { uint16_t node = distr.getIdealDistributorNode(state, document::BucketId(16, i), "u"); @@ -353,7 +390,7 @@ std::vector<uint16_t> createNodeCountList(const std::string& source, std::vector EXPECT_EQ(exp123, cnt123); \ } -TEST(DistributionTest, test_down) +TEST_F(DistributionTest, test_down) { ASSERT_BUCKET_NODE_COUNTS( MyTest().state("storage:10 .4.s:m .5.s:m .6.s:d .7.s:d .9.s:r") @@ -366,7 +403,7 @@ TEST(DistributionTest, test_down) "0:+ 1:+ 2:+ 3:+ 8:+ 9:+"); } -TEST(DistributionTest, test_serialize_deserialize) +TEST_F(DistributionTest, test_serialize_deserialize) { MyTest t1; MyTest t2; @@ -374,7 +411,7 @@ TEST(DistributionTest, test_serialize_deserialize) EXPECT_EQ(t1.getNodeCounts(), t2.getNodeCounts()); } -TEST(DistributionTest, test_initializing) +TEST_F(DistributionTest, test_initializing) { ASSERT_BUCKET_NODE_COUNTS( MyTest().state("distributor:3 .0.s:i .1.s:i .2.s:i") @@ -383,7 +420,7 @@ TEST(DistributionTest, test_initializing) "0:+ 1:+ 2:+"); } -TEST(DistributionTest, testHighSplitBit) +TEST_F(DistributionTest, testHighSplitBit) { // Only 3 nodes of 10 are up => all copies should end on the 3 nodes and // none on the down nodes @@ -422,7 +459,7 @@ TEST(DistributionTest, testHighSplitBit) EXPECT_EQ(ost1.str(), ost2.str()); } -TEST(DistributionTest, test_distribution) +TEST_F(DistributionTest, test_distribution) { const int min_buckets = 64_Ki; const int max_buckets = 64_Ki; @@ -478,7 +515,7 @@ TEST(DistributionTest, test_distribution) } } -TEST(DistributionTest, test_move) +TEST_F(DistributionTest, test_move) { // This test is quite fragile, it will break if the ideal state algorithm is // changed in such a way that Bucket 0x8b4f67ae remains on node 0 and 1 if @@ -511,7 +548,7 @@ TEST(DistributionTest, test_move) EXPECT_EQ(1, int(it-diff.begin())); } -TEST(DistributionTest, test_move_constraints) +TEST_F(DistributionTest, test_move_constraints) { ClusterState clusterState("storage:10"); @@ -598,7 +635,7 @@ TEST(DistributionTest, test_move_constraints) } } -TEST(DistributionTest, test_distribution_bits) +TEST_F(DistributionTest, test_distribution_bits) { ClusterState state1("bits:16 distributor:10"); ClusterState state2("bits:19 distributor:10"); @@ -619,7 +656,7 @@ TEST(DistributionTest, test_distribution_bits) EXPECT_NE(ost1.str(), ost2.str()); } -TEST(DistributionTest, test_redundancy_hierarchical_distribution) +TEST_F(DistributionTest, test_redundancy_hierarchical_distribution) { ClusterState state("storage:10 distributor:10"); @@ -633,7 +670,7 @@ TEST(DistributionTest, test_redundancy_hierarchical_distribution) } } -TEST(DistributionTest, test_hierarchical_distribution) +TEST_F(DistributionTest, test_hierarchical_distribution) { std::string distConfig( "redundancy 4\n" @@ -683,7 +720,7 @@ TEST(DistributionTest, test_hierarchical_distribution) EXPECT_EQ(expectedMains, mainNode); } -TEST(DistributionTest, test_group_capacity) +TEST_F(DistributionTest, test_group_capacity) { std::string distConfig( "redundancy 1\n" @@ -727,7 +764,7 @@ TEST(DistributionTest, test_group_capacity) EXPECT_EQ(1000 - group0count, group1count); } -TEST(DistributionTest, test_hierarchical_no_redistribution) +TEST_F(DistributionTest, test_hierarchical_no_redistribution) { std::string distConfig( "redundancy 2\n" @@ -871,7 +908,7 @@ std::string groupConfig("group[3]\n" "group[2].nodes[2].index 5\n"); } -TEST(DistributionTest, test_active_per_group) +TEST_F(DistributionTest, test_active_per_group) { using IndexList = Distribution::IndexList; // Disabled feature @@ -946,7 +983,7 @@ TEST(DistributionTest, test_active_per_group) } } -TEST(DistributionTest, test_hierarchical_distribute_less_than_redundancy) +TEST_F(DistributionTest, test_hierarchical_distribute_less_than_redundancy) { Distribution distr("redundancy 4\nactive_per_leaf_group true\n" + groupConfig); ClusterState state("storage:6"); @@ -974,7 +1011,7 @@ TEST(DistributionTest, test_hierarchical_distribute_less_than_redundancy) } } -TEST(DistributionTest, wildcard_top_level_distribution_gives_expected_node_results) { +TEST_F(DistributionTest, wildcard_top_level_distribution_gives_expected_node_results) { std::string raw_config = R"(redundancy 2 initial_redundancy 2 ensure_primary_persisted true @@ -1062,7 +1099,7 @@ std::string generate_state_with_n_nodes_up(int n_nodes) { } -TEST(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) { +TEST_F(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) { const int n_groups = 150; Distribution distr(generate_config_with_n_1node_groups(n_groups)); ClusterState state(generate_state_with_n_nodes_up(n_groups)); @@ -1075,7 +1112,7 @@ TEST(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) { fprintf(stderr, "%.10f seconds\n", min_time); } -TEST(DistributionTest, control_size_of_IndexList) { +TEST_F(DistributionTest, control_size_of_IndexList) { EXPECT_EQ(24u, sizeof(Distribution::IndexList)); } diff --git a/vdslib/src/tests/state/CMakeLists.txt b/vdslib/src/tests/state/CMakeLists.txt index 978b037973a..d8065a87879 100644 --- a/vdslib/src/tests/state/CMakeLists.txt +++ b/vdslib/src/tests/state/CMakeLists.txt @@ -6,6 +6,6 @@ vespa_add_library(vdslib_teststate grouptest.cpp nodestatetest.cpp DEPENDS - vdslib + vespa_vdslib GTest::GTest ) diff --git a/vdslib/src/vespa/vdslib/CMakeLists.txt b/vdslib/src/vespa/vdslib/CMakeLists.txt index b81a897390b..421283847b5 100644 --- a/vdslib/src/vespa/vdslib/CMakeLists.txt +++ b/vdslib/src/vespa/vdslib/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(vdslib +vespa_add_library(vespa_vdslib SOURCES $<TARGET_OBJECTS:vdslib_container> $<TARGET_OBJECTS:vdslib_state> diff --git a/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h b/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h index 69d94869f5b..a62c1f33e88 100644 --- a/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h +++ b/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h @@ -21,6 +21,11 @@ struct BucketSpaceDistributionConfigs { return (iter != space_configs.end()) ? iter->second : std::shared_ptr<const Distribution>(); } + [[nodiscard]] const Distribution* get_or_nullptr_raw(document::BucketSpace space) const noexcept { + auto iter = space_configs.find(space); + return (iter != space_configs.end()) ? iter->second.get() : nullptr; + } + static BucketSpaceDistributionConfigs from_default_distribution(std::shared_ptr<const Distribution>); }; diff --git a/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h b/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h index 7b68d4fb6aa..61287427274 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h +++ b/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h @@ -33,6 +33,9 @@ public: [[nodiscard]] std::shared_ptr<const Distribution> bucket_space_distribution_or_nullptr(document::BucketSpace space) const noexcept { return _bucket_space_distributions.get_or_nullptr(space); } + [[nodiscard]] const Distribution* bucket_space_distribution_or_nullptr_raw(document::BucketSpace space) const noexcept { + return _bucket_space_distributions.get_or_nullptr_raw(space); + } [[nodiscard]] const BucketSpaceDistributionConfigs& bucket_space_distributions() const noexcept { return _bucket_space_distributions; } diff --git a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp index 6ef971b4c46..536de9c895e 100644 --- a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp +++ b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp @@ -22,9 +22,8 @@ ClusterStateBundle::FeedBlock::operator==(const FeedBlock& rhs) const noexcept (_description == rhs._description); } -// TODO implement with ctor fwd -ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState) - : _baselineClusterState(std::make_shared<const ClusterState>(baselineClusterState)), +ClusterStateBundle::ClusterStateBundle(std::shared_ptr<const ClusterState> baseline_cluster_state) + : _baselineClusterState(std::move(baseline_cluster_state)), _derivedBucketSpaceStates(), _feed_block(), _distribution_bundle(), @@ -32,6 +31,11 @@ ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState) { } +ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState) + : ClusterStateBundle(std::make_shared<const ClusterState>(baselineClusterState)) +{ +} + ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState, BucketSpaceStateMapping derivedBucketSpaceStates) : _baselineClusterState(std::make_shared<const ClusterState>(baselineClusterState)), @@ -108,6 +112,14 @@ ClusterStateBundle::getDerivedClusterState(document::BucketSpace bucketSpace) co return _baselineClusterState; } +std::shared_ptr<const Distribution> +ClusterStateBundle::bucket_space_distribution_or_nullptr(document::BucketSpace space) const noexcept { + if (!_distribution_bundle) { + return {}; + } + return _distribution_bundle->bucket_space_distribution_or_nullptr(space); +} + uint32_t ClusterStateBundle::getVersion() const { diff --git a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h index 18f82176dd5..e6763a689d8 100644 --- a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h +++ b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h @@ -52,6 +52,7 @@ public: std::shared_ptr<const DistributionConfigBundle> _distribution_bundle; bool _deferredActivation; public: + explicit ClusterStateBundle(std::shared_ptr<const ClusterState> baseline_cluster_state); explicit ClusterStateBundle(const ClusterState& baselineClusterState); ClusterStateBundle(const ClusterState& baselineClusterState, BucketSpaceStateMapping derivedBucketSpaceStates); @@ -98,6 +99,7 @@ public: [[nodiscard]] const std::shared_ptr<const DistributionConfigBundle>& distribution_config_bundle() const noexcept { return _distribution_bundle; } + [[nodiscard]] std::shared_ptr<const Distribution> bucket_space_distribution_or_nullptr(document::BucketSpace space) const noexcept; [[nodiscard]] const std::optional<FeedBlock>& feed_block() const { return _feed_block; } [[nodiscard]] uint32_t getVersion() const; [[nodiscard]] bool deferredActivation() const noexcept { return _deferredActivation; } diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index cac79c3850e..57182446a7e 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -145,6 +145,10 @@ <artifactId>slf4j-api</artifactId> </exclusion> <exclusion> + <groupId>software.amazon.awssdk</groupId> + <artifactId>ssm</artifactId> + </exclusion> + <exclusion> <groupId>org.bouncycastle</groupId> <artifactId>*</artifactId> </exclusion> diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java deleted file mode 100644 index 9f37e3f4613..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api; - -public record DefaultSignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, - String data, IdentityDocument identityDocument) implements SignedIdentityDocument { - - public DefaultSignedIdentityDocument { - identityDocument = EntityBindingsMapper.fromIdentityDocumentData(data); - } - - public DefaultSignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, String data) { - this(signature,signingKeyVersion,documentVersion, data, null); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index ac620d2f6d4..123995721e9 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -5,9 +5,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.DefaultSignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.V4SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.V5SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.yolean.Exceptions; @@ -52,22 +53,32 @@ public class EntityBindingsMapper { } public static SignedIdentityDocument toSignedIdentityDocument(SignedIdentityDocumentEntity entity) { - if (entity instanceof DefaultSignedIdentityDocumentEntity docEntity) { - return new DefaultSignedIdentityDocument(docEntity.signature(), - docEntity.signingKeyVersion(), - docEntity.documentVersion(), - docEntity.data()); + if (entity instanceof V4SignedIdentityDocumentEntity docEntity) { + return new V4SignedIdentityDocument(docEntity.signature(), + docEntity.signingKeyVersion(), + docEntity.documentVersion(), + docEntity.data()); + } else if (entity instanceof V5SignedIdentityDocumentEntity docEntity) { + return new V5SignedIdentityDocument(docEntity.signature(), + docEntity.signingKeyVersion(), + docEntity.documentVersion(), + docEntity.data()); } else { throw new IllegalArgumentException("Unknown signed identity document type: " + entity.getClass().getName()); } } public static SignedIdentityDocumentEntity toSignedIdentityDocumentEntity(SignedIdentityDocument model) { - if (model instanceof DefaultSignedIdentityDocument defaultModel){ - return new DefaultSignedIdentityDocumentEntity(defaultModel.signature(), - defaultModel.signingKeyVersion(), - defaultModel.documentVersion(), - defaultModel.data()); + if (model instanceof V4SignedIdentityDocument defaultModel) { + return new V4SignedIdentityDocumentEntity(defaultModel.signature(), + defaultModel.v4SigningKeyVersion(), + defaultModel.documentVersion(), + defaultModel.data()); + } else if (model instanceof V5SignedIdentityDocument defaultModel){ + return new V5SignedIdentityDocumentEntity(defaultModel.signature(), + defaultModel.signingKeyVersion(), + defaultModel.documentVersion(), + defaultModel.data()); } else { throw new IllegalArgumentException("Unsupported model type: " + model.getClass().getName()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index 39629d878db..56b67694af7 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -8,12 +8,14 @@ package com.yahoo.vespa.athenz.identityprovider.api; */ public interface SignedIdentityDocument { - int DEFAULT_DOCUMENT_VERSION = 4; + int LEGACY_DOCUMENT_VERSION = 4; + int DEFAULT_DOCUMENT_VERSION = 5; default boolean outdated() { return documentVersion() < DEFAULT_DOCUMENT_VERSION; } IdentityDocument identityDocument(); String signature(); - int signingKeyVersion(); + String signingKeyVersion(); int documentVersion(); + String data(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V4SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V4SignedIdentityDocument.java new file mode 100644 index 00000000000..36836786da3 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V4SignedIdentityDocument.java @@ -0,0 +1,19 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api; + +public record V4SignedIdentityDocument(String signature, int v4SigningKeyVersion, int documentVersion, + String data, IdentityDocument identityDocument) implements SignedIdentityDocument { + + public V4SignedIdentityDocument { + identityDocument = EntityBindingsMapper.fromIdentityDocumentData(data); + } + + public V4SignedIdentityDocument(String signature, int v4SigningKeyVersion, int documentVersion, String data) { + this(signature, v4SigningKeyVersion, documentVersion, data, null); + } + + @Override + public String signingKeyVersion() { + return Integer.toString(v4SigningKeyVersion); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V5SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V5SignedIdentityDocument.java new file mode 100644 index 00000000000..644ca2eafb4 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V5SignedIdentityDocument.java @@ -0,0 +1,16 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.athenz.identityprovider.api; + +public record V5SignedIdentityDocument(String signature, String signingKeyVersion, int documentVersion, + String data, IdentityDocument identityDocument) implements SignedIdentityDocument { + + + public V5SignedIdentityDocument { + identityDocument = EntityBindingsMapper.fromIdentityDocumentData(data); + } + + public V5SignedIdentityDocument(String signature, String signingKeyVersion, int documentVersion, String data) { + this(signature,signingKeyVersion,documentVersion, data, null); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index d909849e9ce..dc2daa530e8 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; import com.fasterxml.jackson.databind.jsontype.TypeIdResolver; import com.fasterxml.jackson.databind.type.TypeFactory; +import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import java.io.IOException; import java.util.Objects; @@ -54,7 +55,12 @@ class SignedIdentityDocumentEntityTypeResolver implements TypeIdResolver { @Override public JavaType typeFromId(DatabindContext databindContext, String s) throws IOException { try { - Class<? extends SignedIdentityDocumentEntity> cls = DefaultSignedIdentityDocumentEntity.class; + int version = Integer.parseInt(s); + Class<? extends SignedIdentityDocumentEntity> cls = switch (version) { + case SignedIdentityDocument.LEGACY_DOCUMENT_VERSION -> V4SignedIdentityDocumentEntity.class; + case SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION -> V5SignedIdentityDocumentEntity.class; + default -> throw new IllegalArgumentException("Unknown document version: " + version); + }; return TypeFactory.defaultInstance().constructSpecializedType(javaType,cls); } catch (NumberFormatException e) { throw new IllegalArgumentException("Unable to deserialize document with version: \"%s\"".formatted(s)); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V4SignedIdentityDocumentEntity.java index 74fd43feb35..9c6af38377a 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V4SignedIdentityDocumentEntity.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.athenz.identityprovider.api.bindings; import com.fasterxml.jackson.annotation.JsonProperty; -public record DefaultSignedIdentityDocumentEntity( +public record V4SignedIdentityDocumentEntity( @JsonProperty("signature") String signature, @JsonProperty("signing-key-version") int signingKeyVersion, @JsonProperty("document-version") int documentVersion, diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V5SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V5SignedIdentityDocumentEntity.java new file mode 100644 index 00000000000..eece4b5f066 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V5SignedIdentityDocumentEntity.java @@ -0,0 +1,12 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api.bindings; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record V5SignedIdentityDocumentEntity( + @JsonProperty("signature") String signature, + @JsonProperty("signing-key-version") String signingKeyVersion, + @JsonProperty("document-version") int documentVersion, + @JsonProperty("data") String data) + implements SignedIdentityDocumentEntity { +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java index 43f32a3bae7..392faaaa339 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java @@ -2,23 +2,14 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.security.SignatureUtils; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.identityprovider.api.DefaultSignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; +import com.yahoo.vespa.athenz.identityprovider.api.V4SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.security.PrivateKey; import java.security.PublicKey; import java.security.Signature; -import java.security.SignatureException; -import java.time.Instant; import java.util.Base64; -import java.util.Set; -import java.util.TreeSet; import static java.nio.charset.StandardCharsets.UTF_8; @@ -42,17 +33,13 @@ public class IdentityDocumentSigner { } public boolean hasValidSignature(SignedIdentityDocument doc, PublicKey publicKey) { - if (doc instanceof DefaultSignedIdentityDocument signedDoc) { - try { - Signature signer = SignatureUtils.createVerifier(publicKey); - signer.initVerify(publicKey); - signer.update(signedDoc.data().getBytes(UTF_8)); - return signer.verify(Base64.getDecoder().decode(doc.signature())); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - } else { - throw new IllegalArgumentException("Unknown identity document type: " + doc.getClass().getName()); + try { + Signature signer = SignatureUtils.createVerifier(publicKey); + signer.initVerify(publicKey); + signer.update(doc.data().getBytes(UTF_8)); + return signer.verify(Base64.getDecoder().decode(doc.signature())); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); } } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index 3845d9db5b2..3479350cf26 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -6,7 +6,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.DefaultSignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.V4SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; @@ -53,7 +53,7 @@ public class IdentityDocumentSignerTest { String signature = signer.generateSignature(data, keyPair.getPrivate()); - SignedIdentityDocument signedIdentityDocument = new DefaultSignedIdentityDocument( + SignedIdentityDocument signedIdentityDocument = new V4SignedIdentityDocument( signature, KEY_VERSION, DEFAULT_DOCUMENT_VERSION, data); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); diff --git a/vespaclient-java/CMakeLists.txt b/vespaclient-java/CMakeLists.txt index 6db6c1c78c4..0c383a56233 100644 --- a/vespaclient-java/CMakeLists.txt +++ b/vespaclient-java/CMakeLists.txt @@ -13,4 +13,5 @@ vespa_install_script(src/main/sh/vespa-visit.sh vespa-visit bin) vespa_install_script(src/main/sh/vespa-visit-target.sh vespa-visit-target bin) vespa_install_script(src/main/sh/vespa-feed-perf vespa-feed-perf bin) vespa_install_script(src/main/sh/vespa-status-filedistribution.sh vespa-status-filedistribution bin) +vespa_install_script(src/main/sh/vespa-significance.sh vespa-significance bin) vespa_install_script(src/main/sh/vespa-curl-wrapper vespa-curl-wrapper libexec/vespa) diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/ClientParameters.java index 326b932cabc..ad14708015b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/ClientParameters.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/ClientParameters.java @@ -26,19 +26,24 @@ public class ClientParameters { // Document type identifier public final String docType; + // Zstandard compression + public final boolean zstCompression; + public ClientParameters( boolean help, String inputFile, String outputFile, String field, String language, - String docType) { + String docType, + boolean zstCompression) { this.help = help; this.inputFile = inputFile; this.outputFile = outputFile; this.field = field; this.language = language; this.docType = docType; + this.zstCompression = zstCompression; } public static class Builder { @@ -47,8 +52,8 @@ public class ClientParameters { private String outputFile; private String field; private String language; - private String docType; + private boolean zstCompression; public Builder setHelp(boolean help) { this.help = help; @@ -79,8 +84,13 @@ public class ClientParameters { return this; } + public Builder setZstCompression(String useZstCompression) { + this.zstCompression = Boolean.parseBoolean(useZstCompression); + return this; + } + public ClientParameters build() { - return new ClientParameters(help, inputFile, outputFile, field, language, docType); + return new ClientParameters(help, inputFile, outputFile, field, language, docType, zstCompression); } } } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/CommandLineOptions.java index e5d16854647..2deaee983fe 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/CommandLineOptions.java @@ -26,6 +26,7 @@ public class CommandLineOptions { public static final String FIELD_OPTION = "field"; public static final String LANGUAGE_OPTION = "language"; public static final String DOC_TYPE_OPTION = "doc-type"; + public static final String ZST_COMPRESSION = "zst-compression"; private final Options options = createOptions(); @@ -40,35 +41,46 @@ public class CommandLineOptions { .build()); options.addOption(Option.builder("i") + .required() .hasArg(true) .desc("Input file") .longOpt(INPUT_OPTION) .build()); options.addOption(Option.builder("i") + .required() .hasArg(true) .desc("Output file") .longOpt(OUTPUT_OPTION) .build()); options.addOption(Option.builder("f") + .required() .hasArg(true) .desc("Field to analyze") .longOpt(FIELD_OPTION) .build()); options.addOption(Option.builder("l") + .required() .hasArg(true) .desc("Language tag for output file") .longOpt(LANGUAGE_OPTION) .build()); options.addOption(Option.builder("d") + .required() .hasArg(true) .desc("Document type identifier") .longOpt(DOC_TYPE_OPTION) .build()); + options.addOption(Option.builder("zst") + .hasArg(true) + .desc("Use Zstandard compression") + .longOpt(ZST_COMPRESSION) + .build()); + return options; } @@ -93,6 +105,7 @@ public class CommandLineOptions { builder.setField(cl.getOptionValue(FIELD_OPTION)); builder.setLanguage(cl.getOptionValue(LANGUAGE_OPTION)); builder.setDocType(cl.getOptionValue(DOC_TYPE_OPTION)); + builder.setZstCompression(cl.hasOption(ZST_COMPRESSION) ? cl.getOptionValue(ZST_COMPRESSION) : "true"); return builder.build(); } catch (ParseException e) { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java index a60408b1f96..3bd836d7a14 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java @@ -32,8 +32,8 @@ public class Main { if (params.help) { options.printHelp(); } else { + System.setProperty("vespa.replace_invalid_unicode", "true"); SignificanceModelGenerator significanceModelGenerator = createSignificanceModelGenerator(params); - significanceModelGenerator.generate(); } } catch (IllegalArgumentException e) { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java index 7972a70cd10..3f257fc6df0 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java @@ -2,8 +2,6 @@ package com.yahoo.vespasignificance; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; @@ -27,6 +25,8 @@ import com.yahoo.language.process.Tokenizer; import com.yahoo.language.significance.impl.DocumentFrequencyFile; import com.yahoo.language.significance.impl.SignificanceModelFile; import com.yahoo.text.Utf8; +import io.airlift.compress.zstd.ZstdInputStream; +import io.airlift.compress.zstd.ZstdOutputStream; import java.io.IOException; import java.io.BufferedReader; @@ -34,6 +34,9 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.FileOutputStream; +import java.io.FileInputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -41,6 +44,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.List; +import java.util.Arrays; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -53,12 +58,15 @@ public class SignificanceModelGenerator { private final Tokenizer tokenizer; private final TreeMap<String, Long> documentFrequency = new TreeMap<>(); - private final Language language; + private final List<Language> languages; + + private final Language languageTag; private final ObjectMapper objectMapper; private final static JsonFactory parserFactory = new JsonFactory(); final DocumentTypeManager types = new DocumentTypeManager(); final DocumentType docType; + private final boolean useZstCompression; private final static String VERSION = "1.0"; private final static String ID = "1"; private final static String SIGNIFICANCE_DESCRIPTION = "Significance model for input file"; @@ -66,14 +74,29 @@ public class SignificanceModelGenerator { public SignificanceModelGenerator(ClientParameters clientParameters) { this.clientParameters = clientParameters; + + if (clientParameters.zstCompression && !clientParameters.outputFile.endsWith(".zst")) { + throw new IllegalArgumentException("Output file must have .zst extension when using zst compression"); + } + + if (!clientParameters.zstCompression && clientParameters.outputFile.endsWith(".zst")) { + throw new IllegalArgumentException("Output file must not have .zst extension when not using zst compression"); + } + + this.languages = Arrays.stream(clientParameters.language.split(",")) + .map(Language::fromLanguageTag) + .collect(Collectors.toList()); + + this.languageTag = this.languages.get(0); + OpenNlpLinguistics openNlpLinguistics = new OpenNlpLinguistics(); tokenizer = openNlpLinguistics.getTokenizer(); objectMapper = new ObjectMapper(); - language = Language.fromLanguageTag(clientParameters.language); - docType = new DocumentType(clientParameters.docType); docType.addField(new Field(clientParameters.field, DataType.STRING)); + useZstCompression = clientParameters.zstCompression; + types.registerDocumentType(docType); } @@ -90,7 +113,7 @@ public class SignificanceModelGenerator { while (reader.ready()) { String line = reader.readLine(); JsonReader jsonReader = new JsonReader(types, new ByteArrayInputStream(Utf8.toBytes(line)), parserFactory); - String wikimediaId = "id:wikimedia:" + language.languageCode() + "::" + i; + String wikimediaId = "id:wikimedia:" + languageTag.languageCode() + "::" + i; ParsedDocumentOperation operation = jsonReader.readSingleDocumentStreaming(DocumentOperationType.PUT, wikimediaId); DocumentPut put = (DocumentPut) operation.operation(); @@ -103,28 +126,40 @@ public class SignificanceModelGenerator { long pageCount = i - 1; SignificanceModelFile modelFile; - if (Paths.get(clientParameters.outputFile).toFile().exists()) { - modelFile = objectMapper.readValue(new File(clientParameters.outputFile), SignificanceModelFile.class); + File outputFile = Paths.get(clientParameters.outputFile).toFile(); + String languagesKey = String.join(",", this.languages.stream().map(Language::languageCode).toList()); + if (outputFile.exists()) { + + InputStream in = outputFile.toString().endsWith(".zst") ? + new ZstdInputStream(new FileInputStream(outputFile)) : + new FileInputStream(outputFile); - modelFile.addLanguage(clientParameters.language, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); + modelFile = objectMapper.readValue(in, SignificanceModelFile.class); + + modelFile.addLanguage(languagesKey, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); } else { HashMap<String, DocumentFrequencyFile> languages = new HashMap<>() {{ - put(clientParameters.language, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); + put(languagesKey, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); }}; modelFile = new SignificanceModelFile(VERSION, ID, SIGNIFICANCE_DESCRIPTION + clientParameters.inputFile, languages); } try { ObjectWriter writer = objectMapper.writerWithDefaultPrettyPrinter(); - writer.writeValue(new File(clientParameters.outputFile), modelFile); + + OutputStream out = useZstCompression ? + new ZstdOutputStream(new FileOutputStream(clientParameters.outputFile)) : + new FileOutputStream(clientParameters.outputFile); + + writer.writeValue(out, modelFile); } catch (IOException e) { throw new IllegalStateException("Failed to write model to output file", e); } } private void handleTokenization(String field) { - var tokens = tokenizer.tokenize(field, language, StemMode.ALL, false); + var tokens = tokenizer.tokenize(field, languageTag, StemMode.ALL, false); Set<String> uniqueWords = StreamSupport.stream(tokens.spliterator(), false) .filter(t -> t.getType() == TokenType.ALPHABETIC) diff --git a/vespaclient-java/src/main/sh/vespa-significance.sh b/vespaclient-java/src/main/sh/vespa-significance.sh index 0c0bac275e6..dfbf5b90c2e 100755 --- a/vespaclient-java/src/main/sh/vespa-significance.sh +++ b/vespaclient-java/src/main/sh/vespa-significance.sh @@ -79,6 +79,7 @@ export ROOT export MALLOC_ARENA_MAX=1 #Does not need fast allocation exec java \ +-Dvespa.replace_invalid_unicode=true \ -server -enableassertions \ -XX:ThreadStackSize=512 \ -XX:MaxJavaStackTraceDepth=1000000 \ diff --git a/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java b/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java index 8e1e8fd5627..4791d78f0a2 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java @@ -5,11 +5,14 @@ package com.yahoo.vespasignificance; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.language.significance.impl.DocumentFrequencyFile; import com.yahoo.language.significance.impl.SignificanceModelFile; +import io.airlift.compress.zstd.ZstdInputStream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.util.HashMap; @@ -25,7 +28,7 @@ public class SignificanceModelGeneratorTest { @TempDir private Path tempDir; - private ClientParameters.Builder createParameters(String inputPath, String outputPath, String field, String language, String docType) { + private ClientParameters.Builder createParameters(String inputPath, String outputPath, String field, String language, String docType, String zstCompression) { tempDir.toFile().mkdirs(); return new ClientParameters.Builder() @@ -33,7 +36,8 @@ public class SignificanceModelGeneratorTest { .setOutputFile(tempDir.resolve(outputPath).toString()) .setField(field) .setLanguage(language) - .setDocType(docType); + .setDocType(docType) + .setZstCompression(zstCompression); } private SignificanceModelGenerator createSignificanceModelGenerator(ClientParameters params) { @@ -44,7 +48,7 @@ public class SignificanceModelGeneratorTest { void testGenerateSimpleFile() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params = createParameters(inputPath, outputPath, "text", "NB", "nb").build(); + ClientParameters params = createParameters(inputPath, outputPath, "text", "nb", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params); generator.generate(); @@ -56,9 +60,44 @@ public class SignificanceModelGeneratorTest { HashMap<String, DocumentFrequencyFile> languages = modelFile.languages(); assertEquals(1, languages.size()); - assertTrue(languages.containsKey("NB")); + assertTrue(languages.containsKey("nb")); - DocumentFrequencyFile documentFrequencyFile = languages.get("NB"); + DocumentFrequencyFile documentFrequencyFile = languages.get("nb"); + + assertEquals(3, documentFrequencyFile.frequencies().get("fra")); + assertEquals(3, documentFrequencyFile.frequencies().get("skriveform")); + assertEquals(3, documentFrequencyFile.frequencies().get("kategori")); + assertEquals(3, documentFrequencyFile.frequencies().get("eldr")); + + } + + @Test + void testGenerateSimpleFileWithZST() throws IOException { + String inputPath = "no.jsonl"; + ClientParameters params1 = createParameters(inputPath, "output.json", "text", "nb", "nb", "true").build(); + + // Throws exception when outputfile does not have .zst extension when using zst compression + assertThrows(IllegalArgumentException.class, () -> createSignificanceModelGenerator(params1)); + + String outputPath = "output.json.zst"; + ClientParameters params = createParameters(inputPath, outputPath, "text", "nb", "nb", "true").build(); + + SignificanceModelGenerator generator = createSignificanceModelGenerator(params); + generator.generate(); + + File outputFile = new File(tempDir.resolve(outputPath ).toString()); + assertTrue(outputFile.exists()); + + InputStream in = new ZstdInputStream(new FileInputStream(outputFile)); + + SignificanceModelFile modelFile = objectMapper.readValue(in, SignificanceModelFile.class); + + HashMap<String, DocumentFrequencyFile> languages = modelFile.languages(); + assertEquals(1, languages.size()); + + assertTrue(languages.containsKey("nb")); + + DocumentFrequencyFile documentFrequencyFile = languages.get("nb"); assertEquals(3, documentFrequencyFile.frequencies().get("fra")); assertEquals(3, documentFrequencyFile.frequencies().get("skriveform")); @@ -71,7 +110,7 @@ public class SignificanceModelGeneratorTest { void testGenerateFileWithMultipleLanguages() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params1 = createParameters(inputPath, outputPath, "text", "NB", "nb").build(); + ClientParameters params1 = createParameters(inputPath, outputPath, "text", "nb", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params1); generator.generate(); @@ -79,7 +118,7 @@ public class SignificanceModelGeneratorTest { assertTrue(outputFile.exists()); String inputPath2 = "en.jsonl"; - ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "EN", "en").build(); + ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "en", "en", "false").build(); generator = createSignificanceModelGenerator(params2); generator.generate(); @@ -92,11 +131,11 @@ public class SignificanceModelGeneratorTest { assertEquals(2, languages.size()); - assertTrue(languages.containsKey("NB")); - assertTrue(languages.containsKey("EN")); + assertTrue(languages.containsKey("nb")); + assertTrue(languages.containsKey("en")); - DocumentFrequencyFile nb = languages.get("NB"); - DocumentFrequencyFile en = languages.get("EN"); + DocumentFrequencyFile nb = languages.get("nb"); + DocumentFrequencyFile en = languages.get("en"); assertEquals(3, nb.documentCount()); assertEquals(3, en.documentCount()); @@ -113,7 +152,7 @@ public class SignificanceModelGeneratorTest { void testOverwriteExistingDocumentFrequencyLanguage() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params1 = createParameters(inputPath, outputPath, "text", "NB", "nb").build(); + ClientParameters params1 = createParameters(inputPath, outputPath, "text", "nb", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params1); generator.generate(); @@ -125,16 +164,16 @@ public class SignificanceModelGeneratorTest { HashMap<String, DocumentFrequencyFile> oldLanguages = preUpdatedFile.languages(); assertEquals(1, oldLanguages.size()); - assertTrue(oldLanguages.containsKey("NB")); + assertTrue(oldLanguages.containsKey("nb")); - DocumentFrequencyFile oldDf = oldLanguages.get("NB"); + DocumentFrequencyFile oldDf = oldLanguages.get("nb"); assertEquals(3, oldDf.frequencies().get("fra")); assertEquals(3, oldDf.frequencies().get("skriveform")); assertFalse(oldDf.frequencies().containsKey("nytt")); String inputPath2 = "no_2.jsonl"; - ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "NB", "nb").build(); + ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "nb", "nb", "false").build(); SignificanceModelGenerator generator2 = createSignificanceModelGenerator(params2); generator2.generate(); @@ -147,13 +186,39 @@ public class SignificanceModelGeneratorTest { assertEquals(1, languages.size()); - assertTrue(languages.containsKey("NB")); + assertTrue(languages.containsKey("nb")); - DocumentFrequencyFile df = languages.get("NB"); + DocumentFrequencyFile df = languages.get("nb"); assertEquals(2, df.frequencies().get("fra")); assertEquals(3, df.frequencies().get("skriveform")); assertTrue(df.frequencies().containsKey("nytt")); assertEquals(2, df.frequencies().get("nytt")); } + + @Test + void testGenerateFileWithMultipleLanguagesForSingleDocumentFrequency() throws IOException { + String inputPath = "no.jsonl"; + String outputPath = "output.json"; + ClientParameters params = createParameters(inputPath, outputPath, "text", "nb,un", "nb", "false").build(); + SignificanceModelGenerator generator = createSignificanceModelGenerator(params); + generator.generate(); + + File outputFile = new File(tempDir.resolve(outputPath).toString()); + assertTrue(outputFile.exists()); + + SignificanceModelFile modelFile = objectMapper.readValue(outputFile, SignificanceModelFile.class); + + HashMap<String, DocumentFrequencyFile> languages = modelFile.languages(); + assertEquals(1, languages.size()); + + assertTrue(languages.containsKey("nb,un")); + + DocumentFrequencyFile documentFrequencyFile = languages.get("nb,un"); + + assertEquals(3, documentFrequencyFile.frequencies().get("fra")); + assertEquals(3, documentFrequencyFile.frequencies().get("skriveform")); + assertEquals(3, documentFrequencyFile.frequencies().get("kategori")); + assertEquals(3, documentFrequencyFile.frequencies().get("eldr")); + } } diff --git a/vespalib/src/tests/alloc/alloc_test.cpp b/vespalib/src/tests/alloc/alloc_test.cpp index 1436724267d..566ef8d9716 100644 --- a/vespalib/src/tests/alloc/alloc_test.cpp +++ b/vespalib/src/tests/alloc/alloc_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/memory_allocator.h> #include <vespa/vespalib/util/exceptions.h> diff --git a/vespalib/src/tests/array/array_test.cpp b/vespalib/src/tests/array/array_test.cpp index 48e82b56e7d..5446130d5d9 100644 --- a/vespalib/src/tests/array/array_test.cpp +++ b/vespalib/src/tests/array/array_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/util/round_up_to_page_size.h> diff --git a/vespalib/src/tests/barrier/barrier_test.cpp b/vespalib/src/tests/barrier/barrier_test.cpp index eba57f5381f..18032d92ce0 100644 --- a/vespalib/src/tests/barrier/barrier_test.cpp +++ b/vespalib/src/tests/barrier/barrier_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/barrier.h> +#include <vespa/vespalib/util/count_down_latch.h> using namespace vespalib; diff --git a/vespalib/src/tests/component/component.cpp b/vespalib/src/tests/component/component.cpp index 01d006d8aa8..15b381f6ba2 100644 --- a/vespalib/src/tests/component/component.cpp +++ b/vespalib/src/tests/component/component.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("component_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/component/version.h> diff --git a/vespalib/src/tests/compression/compression_test.cpp b/vespalib/src/tests/compression/compression_test.cpp index 264f21aeefe..2257b57dc7e 100644 --- a/vespalib/src/tests/compression/compression_test.cpp +++ b/vespalib/src/tests/compression/compression_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> +#include <atomic> #include <vespa/log/log.h> LOG_SETUP("compression_test"); diff --git a/vespalib/src/tests/data/memory_input/memory_input_test.cpp b/vespalib/src/tests/data/memory_input/memory_input_test.cpp index b3ed505b6e5..418a595eadb 100644 --- a/vespalib/src/tests/data/memory_input/memory_input_test.cpp +++ b/vespalib/src/tests/data/memory_input/memory_input_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/memory_input.h> using namespace vespalib; diff --git a/vespalib/src/tests/data/output_writer/output_writer_test.cpp b/vespalib/src/tests/data/output_writer/output_writer_test.cpp index b3090624336..2fcc77c929f 100644 --- a/vespalib/src/tests/data/output_writer/output_writer_test.cpp +++ b/vespalib/src/tests/data/output_writer/output_writer_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/data/output_writer.h> diff --git a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp index 01233a20fb5..f64287d334f 100644 --- a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp +++ b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/datastore/entryref.h> #include <vespa/vespalib/datastore/array_store_config.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index b3af0c84b2d..d2dcf99081b 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -1,7 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/datastore/buffer_type.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <cassert> using namespace vespalib::datastore; diff --git a/vespalib/src/tests/executor/executor_test.cpp b/vespalib/src/tests/executor/executor_test.cpp index afe37710088..5fcc23fe618 100644 --- a/vespalib/src/tests/executor/executor_test.cpp +++ b/vespalib/src/tests/executor/executor_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/executor_stats.h> diff --git a/vespalib/src/tests/guard/guard_test.cpp b/vespalib/src/tests/guard/guard_test.cpp index 2efe66201f9..c310f99ba3f 100644 --- a/vespalib/src/tests/guard/guard_test.cpp +++ b/vespalib/src/tests/guard/guard_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/guard.h> #include <fcntl.h> #include <unistd.h> diff --git a/vespalib/src/tests/memory/memory_test.cpp b/vespalib/src/tests/memory/memory_test.cpp index 4b6815196c2..be413fd1a95 100644 --- a/vespalib/src/tests/memory/memory_test.cpp +++ b/vespalib/src/tests/memory/memory_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/memory.h> using namespace vespalib; diff --git a/vespalib/src/tests/memorydatastore/memorydatastore.cpp b/vespalib/src/tests/memorydatastore/memorydatastore.cpp index fbbc1e76dff..3efab9e0178 100644 --- a/vespalib/src/tests/memorydatastore/memorydatastore.cpp +++ b/vespalib/src/tests/memorydatastore/memorydatastore.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/memorydatastore.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/vespalib/src/tests/metrics/simple_metrics_test.cpp b/vespalib/src/tests/metrics/simple_metrics_test.cpp index 8d751bf3528..a5b22b6726d 100644 --- a/vespalib/src/tests/metrics/simple_metrics_test.cpp +++ b/vespalib/src/tests/metrics/simple_metrics_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/json_format.h> #include <vespa/vespalib/metrics/simple_metrics.h> diff --git a/vespalib/src/tests/metrics/stable_store_test.cpp b/vespalib/src/tests/metrics/stable_store_test.cpp index 026e6f5dfef..33bff5eec4a 100644 --- a/vespalib/src/tests/metrics/stable_store_test.cpp +++ b/vespalib/src/tests/metrics/stable_store_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/metrics/simple_metrics.h> #include <vespa/vespalib/metrics/simple_metrics_manager.h> #include <vespa/vespalib/metrics/stable_store.h> diff --git a/vespalib/src/tests/net/send_fd/CMakeLists.txt b/vespalib/src/tests/net/send_fd/CMakeLists.txt index 4c46a773f5c..b4baaeb5855 100644 --- a/vespalib/src/tests/net/send_fd/CMakeLists.txt +++ b/vespalib/src/tests/net/send_fd/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(vespalib_send_fd_test_app TEST send_fd_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_send_fd_test_app COMMAND vespalib_send_fd_test_app) diff --git a/vespalib/src/tests/net/send_fd/send_fd_test.cpp b/vespalib/src/tests/net/send_fd/send_fd_test.cpp index 59b9aacea07..8dc1235ff76 100644 --- a/vespalib/src/tests/net/send_fd/send_fd_test.cpp +++ b/vespalib/src/tests/net/send_fd/send_fd_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/selector.h> @@ -7,15 +7,19 @@ #include <vespa/vespalib/net/server_socket.h> #include <vespa/vespalib/net/socket_options.h> #include <vespa/vespalib/net/socket.h> +#include <vespa/vespalib/test/nexus.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/test/socket_options_verifier.h> -#include <thread> -#include <functional> #include <chrono> +#include <functional> +#include <latch> +#include <optional> +#include <thread> #include <unistd.h> #include <sys/stat.h> using namespace vespalib; +using vespalib::test::Nexus; vespalib::string read_bytes(SocketHandle &socket, size_t wanted_bytes) { char tmp[64]; @@ -23,7 +27,9 @@ vespalib::string read_bytes(SocketHandle &socket, size_t wanted_bytes) { while (result.size() < wanted_bytes) { size_t read_size = std::min(sizeof(tmp), wanted_bytes - result.size()); ssize_t read_result = socket.read(tmp, read_size); - ASSERT_GREATER(read_result, 0); + if (read_result <= 0) { + return result; + } result.append(tmp, read_result); } return result; @@ -34,14 +40,14 @@ void verify_socket_io(bool is_server, SocketHandle &socket) { vespalib::string client_message = "please pick up, I need to talk to you"; if(is_server) { ssize_t written = socket.write(server_message.data(), server_message.size()); - EXPECT_EQUAL(written, ssize_t(server_message.size())); + EXPECT_EQ(written, ssize_t(server_message.size())); vespalib::string read = read_bytes(socket, client_message.size()); - EXPECT_EQUAL(client_message, read); + EXPECT_EQ(client_message, read); } else { ssize_t written = socket.write(client_message.data(), client_message.size()); - EXPECT_EQUAL(written, ssize_t(client_message.size())); + EXPECT_EQ(written, ssize_t(client_message.size())); vespalib::string read = read_bytes(socket, server_message.size()); - EXPECT_EQUAL(server_message, read); + EXPECT_EQ(server_message, read); } } @@ -79,10 +85,10 @@ void send_fd(SocketHandle &socket, SocketHandle fd) { int *fd_dst = (int *) (void *) CMSG_DATA(hdr); fd_dst[0] = fd.get(); ssize_t res = sendmsg(socket.get(), &msg, 0); - ASSERT_EQUAL(res, 1); + ASSERT_EQ(res, 1); } -SocketHandle recv_fd(SocketHandle &socket) { +void recv_fd(SocketHandle &socket, std::optional<SocketHandle>& result) { struct msghdr msg = {}; char tag = '*'; struct iovec data; @@ -94,36 +100,62 @@ SocketHandle recv_fd(SocketHandle &socket) { msg.msg_control = buf; msg.msg_controllen = sizeof(buf); ssize_t res = recvmsg(socket.get(), &msg, 0); - ASSERT_EQUAL(res, 1); + ASSERT_EQ(res, 1); struct cmsghdr *hdr = CMSG_FIRSTHDR(&msg); bool type_ok = ((hdr->cmsg_level == SOL_SOCKET) && (hdr->cmsg_type == SCM_RIGHTS)); ASSERT_TRUE(type_ok); int *fd_src = (int *) (void *) CMSG_DATA(hdr); fprintf(stderr, "got fd: %d\n", fd_src[0]); - return SocketHandle(fd_src[0]); + result = SocketHandle(fd_src[0]); } //----------------------------------------------------------------------------- -TEST_MT_FFF("require that an open socket (handle) can be passed over a unix domain socket", 3, - ServerSocket("tcp/0"), ServerSocket("ipc/file:my_socket"), TimeBomb(60)) -{ - if (thread_id == 0) { // server - SocketHandle socket = accept(f1); - TEST_DO(verify_socket_io(true, socket)); // server side - TEST_BARRIER(); - } else if (thread_id == 1) { // proxy - SocketHandle server_socket = connect(f1); - SocketHandle client_socket = accept(f2); - send_fd(client_socket, std::move(server_socket)); - TEST_BARRIER(); - } else { // client - SocketHandle proxy_socket = connect(f2); - SocketHandle socket = recv_fd(proxy_socket); - TEST_DO(verify_socket_io(false, socket)); // client side - TEST_BARRIER(); +namespace { + +class WaitLatch { + std::latch& _latch; +public: + explicit WaitLatch(std::latch& latch) noexcept + : _latch(latch) + { } + ~WaitLatch() { _latch.arrive_and_wait(); } +}; + +} + +TEST(SendFdTest, require_that_an_open_socket_handle_can_be_passed_over_a_unix_domain_socket) +{ + constexpr size_t num_threads = 3; + ServerSocket f1("tcp/0"); + ServerSocket f2("ipc/file:my_socket"); + std::latch latch(num_threads); + TimeBomb f3(60); + auto task = [&f1,&f2,&latch](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + if (thread_id == 0) { // server + SocketHandle socket = accept(f1); + WaitLatch wait(latch); + SCOPED_TRACE("verify socket io server side"); + verify_socket_io(true, socket); // server side + } else if (thread_id == 1) { // proxy + SocketHandle server_socket = connect(f1); + SocketHandle client_socket = accept(f2); + WaitLatch wait(latch); + ASSERT_NO_FATAL_FAILURE(send_fd(client_socket, std::move(server_socket))); + } else { // client + SocketHandle proxy_socket = connect(f2); + std::optional<SocketHandle> socket; + WaitLatch wait(latch); + ASSERT_NO_FATAL_FAILURE(recv_fd(proxy_socket, socket)); + ASSERT_TRUE(socket.has_value()); + SCOPED_TRACE("verify socket io client side"); + verify_socket_io(false, socket.value()); // client side + } + }; + Nexus::run(num_threads, task); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/net/socket/CMakeLists.txt b/vespalib/src/tests/net/socket/CMakeLists.txt index d20720971d2..49c4ca20ba3 100644 --- a/vespalib/src/tests/net/socket/CMakeLists.txt +++ b/vespalib/src/tests/net/socket/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(vespalib_socket_test_app TEST socket_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_socket_test_app COMMAND vespalib_socket_test_app) vespa_add_executable(vespalib_socket_server_app diff --git a/vespalib/src/tests/net/socket/socket_test.cpp b/vespalib/src/tests/net/socket/socket_test.cpp index b0708388cd6..64f337c03f6 100644 --- a/vespalib/src/tests/net/socket/socket_test.cpp +++ b/vespalib/src/tests/net/socket/socket_test.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/selector.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/server_socket.h> #include <vespa/vespalib/net/socket_options.h> #include <vespa/vespalib/net/socket.h> +#include <vespa/vespalib/test/nexus.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/test/socket_options_verifier.h> #include <thread> @@ -14,19 +15,55 @@ #include <sys/stat.h> using namespace vespalib; +using vespalib::test::Nexus; -bool ipv4_enabled = false; -bool ipv6_enabled = false; +class SocketTest : public ::testing::Test { +protected: + static bool ipv4_enabled; + static bool ipv6_enabled; -int my_inet() { + SocketTest(); + ~SocketTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); + static int my_inet(); +}; + +bool SocketTest::ipv4_enabled = false; +bool SocketTest::ipv6_enabled = false; + +SocketTest::SocketTest() + : testing::Test() +{ +} + +SocketTest::~SocketTest() = default; + +void +SocketTest::SetUpTestSuite() +{ + auto list = SocketAddress::resolve(4080); + for (const auto &addr : list) { + (void) addr; + ipv4_enabled |= addr.is_ipv4(); + ipv6_enabled |= addr.is_ipv6(); + } + ASSERT_TRUE(ipv4_enabled || ipv6_enabled) << "tcp/ip support not detected"; +} + +void +SocketTest::TearDownTestSuite() +{ +} + +int +SocketTest::my_inet() +{ if (ipv6_enabled) { return AF_INET6; - } - if (ipv4_enabled) { + } else { return AF_INET; } - TEST_ERROR("tcp/ip support not detected"); - return AF_UNIX; } bool is_socket(const vespalib::string &path) { @@ -52,8 +89,8 @@ void remove_file(const vespalib::string &path) { void replace_file(const vespalib::string &path, const vespalib::string &data) { remove_file(path); int fd = creat(path.c_str(), 0600); - ASSERT_NOT_EQUAL(fd, -1); - ASSERT_EQUAL(write(fd, data.data(), data.size()), ssize_t(data.size())); + ASSERT_NE(fd, -1); + ASSERT_EQ(write(fd, data.data(), data.size()), ssize_t(data.size())); close(fd); } @@ -96,14 +133,14 @@ void verify_socket_io(bool is_server, SocketHandle &socket) { vespalib::string client_message = "please pick up, I need to talk to you"; if(is_server) { ssize_t written = socket.write(server_message.data(), server_message.size()); - EXPECT_EQUAL(written, ssize_t(server_message.size())); + EXPECT_EQ(written, ssize_t(server_message.size())); vespalib::string read = read_bytes(socket, client_message.size()); - EXPECT_EQUAL(client_message, read); + EXPECT_EQ(client_message, read); } else { ssize_t written = socket.write(client_message.data(), client_message.size()); - EXPECT_EQUAL(written, ssize_t(client_message.size())); + EXPECT_EQ(written, ssize_t(client_message.size())); vespalib::string read = read_bytes(socket, server_message.size()); - EXPECT_EQUAL(server_message, read); + EXPECT_EQ(server_message, read); } } @@ -122,22 +159,22 @@ SocketHandle connect_sockets(bool is_server, ServerSocket &server_socket) { //----------------------------------------------------------------------------- -TEST("my local address") { +TEST_F(SocketTest, my_local_address) +{ auto list = SocketAddress::resolve(4080); fprintf(stderr, "resolve(4080):\n"); for (const auto &addr: list) { EXPECT_TRUE(addr.is_wildcard()); EXPECT_TRUE(addr.is_ipv4() || addr.is_ipv6()); - ipv4_enabled |= addr.is_ipv4(); - ipv6_enabled |= addr.is_ipv6(); EXPECT_TRUE(!addr.is_ipc()); EXPECT_TRUE(!addr.is_abstract()); - EXPECT_EQUAL(addr.port(), 4080); + EXPECT_EQ(addr.port(), 4080); fprintf(stderr, " %s (%s)\n", addr.spec().c_str(), get_meta(addr).c_str()); } } -TEST("yahoo.com address") { +TEST_F(SocketTest, yahoo_com_address) +{ auto list = SocketAddress::resolve(80, "yahoo.com"); fprintf(stderr, "resolve(80, 'yahoo.com'):\n"); for (const auto &addr: list) { @@ -145,80 +182,102 @@ TEST("yahoo.com address") { EXPECT_TRUE(addr.is_ipv4() || addr.is_ipv6()); EXPECT_TRUE(!addr.is_ipc()); EXPECT_TRUE(!addr.is_abstract()); - EXPECT_EQUAL(addr.port(), 80); + EXPECT_EQ(addr.port(), 80); fprintf(stderr, " %s (%s)\n", addr.spec().c_str(), get_meta(addr).c_str()); } } -TEST("ipc address (path)") { +TEST_F(SocketTest, ipc_address_with_path) +{ auto addr = SocketAddress::from_path("my_socket"); EXPECT_TRUE(!addr.is_ipv4()); EXPECT_TRUE(!addr.is_ipv6()); EXPECT_TRUE(addr.is_ipc()); EXPECT_TRUE(!addr.is_abstract()); EXPECT_TRUE(!addr.is_wildcard()); - EXPECT_EQUAL(addr.port(), -1); - EXPECT_EQUAL(vespalib::string("my_socket"), addr.path()); + EXPECT_EQ(addr.port(), -1); + EXPECT_EQ(vespalib::string("my_socket"), addr.path()); EXPECT_TRUE(addr.name().empty()); fprintf(stderr, "from_path(my_socket)\n"); fprintf(stderr, " %s (%s)\n", addr.spec().c_str(), get_meta(addr).c_str()); } -TEST("ipc address (name)") { +TEST_F(SocketTest, ipc_address_with_name) +{ auto addr = SocketAddress::from_name("my_socket"); EXPECT_TRUE(!addr.is_ipv4()); EXPECT_TRUE(!addr.is_ipv6()); EXPECT_TRUE(addr.is_ipc()); EXPECT_TRUE(addr.is_abstract()); EXPECT_TRUE(!addr.is_wildcard()); - EXPECT_EQUAL(addr.port(), -1); + EXPECT_EQ(addr.port(), -1); EXPECT_TRUE(addr.path().empty()); - EXPECT_EQUAL(vespalib::string("my_socket"), addr.name()); + EXPECT_EQ(vespalib::string("my_socket"), addr.name()); fprintf(stderr, "from_path(my_socket)\n"); fprintf(stderr, " %s (%s)\n", addr.spec().c_str(), get_meta(addr).c_str()); } -TEST("local client/server addresses") { +TEST_F(SocketTest, local_client_and_server_addresses) { auto spec = SocketSpec("tcp/123"); auto client = spec.client_address(); auto server = spec.server_address(); EXPECT_TRUE(!client.is_wildcard()); - EXPECT_EQUAL(client.port(), 123); + EXPECT_EQ(client.port(), 123); EXPECT_TRUE(server.is_wildcard()); - EXPECT_EQUAL(server.port(), 123); + EXPECT_EQ(server.port(), 123); fprintf(stderr, "client(tcp/123): %s (%s)\n", client.spec().c_str(), get_meta(client).c_str()); fprintf(stderr, "server(tcp/123): %s (%s)\n", server.spec().c_str(), get_meta(server).c_str()); } -TEST_MT_FF("require that basic socket io works", 2, ServerSocket("tcp/0"), TimeBomb(60)) { - bool is_server = (thread_id == 0); - SocketHandle socket = connect_sockets(is_server, f1); - TEST_DO(verify_socket_io(is_server, socket)); -} - -TEST_MT_FF("require that basic unix domain socket io works (path)", 2, - ServerSocket("ipc/file:my_socket"), TimeBomb(60)) +TEST_F(SocketTest, require_that_basic_socket_io_works) +{ + constexpr size_t num_threads = 2; + ServerSocket f1("tcp/0"); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + bool is_server = (ctx.thread_id() == 0); + SocketHandle socket = connect_sockets(is_server, f1); + verify_socket_io(is_server, socket); + }; + Nexus::run(num_threads, task); +} + +TEST_F(SocketTest, require_that_basic_unix_domain_socket_io_works_with_path) +{ + constexpr size_t num_threads = 2; + ServerSocket f1("ipc/file:my_socket"); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + bool is_server = (ctx.thread_id() == 0); + SocketHandle socket = connect_sockets(is_server, f1); + verify_socket_io(is_server, socket); + }; + Nexus::run(num_threads, task); +} + +TEST_F(SocketTest, require_that_server_accept_can_be_interrupted) +{ + constexpr size_t num_threads = 2; + ServerSocket f1("tcp/0"); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + bool is_server = (ctx.thread_id() == 0); + if (is_server) { + fprintf(stderr, "--> calling accept\n"); + SocketHandle socket = f1.accept(); + fprintf(stderr, "<-- accept returned\n"); + EXPECT_TRUE(!socket.valid()); + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + fprintf(stderr, "--- closing server socket\n"); + f1.shutdown(); + } + }; + Nexus::run(num_threads, task); +} + +TEST_F(SocketTest, require_that_socket_file_is_removed_by_server_socket_when_destructed) { - bool is_server = (thread_id == 0); - SocketHandle socket = connect_sockets(is_server, f1); - TEST_DO(verify_socket_io(is_server, socket)); -} - -TEST_MT_FF("require that server accept can be interrupted", 2, ServerSocket("tcp/0"), TimeBomb(60)) { - bool is_server = (thread_id == 0); - if (is_server) { - fprintf(stderr, "--> calling accept\n"); - SocketHandle socket = f1.accept(); - fprintf(stderr, "<-- accept returned\n"); - EXPECT_TRUE(!socket.valid()); - } else { - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - fprintf(stderr, "--- closing server socket\n"); - f1.shutdown(); - } -} - -TEST("require that socket file is removed by server socket when destructed") { remove_file("my_socket"); ServerSocket server("ipc/file:my_socket"); EXPECT_TRUE(server.valid()); @@ -227,7 +286,8 @@ TEST("require that socket file is removed by server socket when destructed") { EXPECT_TRUE(!is_socket("my_socket")); } -TEST("require that socket file is only removed on destruction if it is a socket") { +TEST_F(SocketTest, require_that_socket_file_is_only_removed_on_destruction_if_it_is_a_socket) +{ remove_file("my_socket"); ServerSocket server("ipc/file:my_socket"); EXPECT_TRUE(server.valid()); @@ -238,7 +298,8 @@ TEST("require that socket file is only removed on destruction if it is a socket" remove_file("my_socket"); } -TEST("require that a server socket will fail to listen to a path that is already a regular file") { +TEST_F(SocketTest, require_that_a_server_socket_will_fail_to_listen_to_a_path_that_is_already_a_regular_file) +{ replace_file("my_socket", "hello\n"); ServerSocket server("ipc/file:my_socket"); EXPECT_TRUE(!server.valid()); @@ -247,7 +308,8 @@ TEST("require that a server socket will fail to listen to a path that is already remove_file("my_socket"); } -TEST("require that a server socket will fail to listen to a path that is already taken by another server") { +TEST_F(SocketTest, require_that_a_server_socket_will_fail_to_listen_to_a_path_that_is_already_taken_by_another_server) +{ remove_file("my_socket"); ServerSocket server1("ipc/file:my_socket"); ServerSocket server2("ipc/file:my_socket"); @@ -258,7 +320,8 @@ TEST("require that a server socket will fail to listen to a path that is already EXPECT_TRUE(!is_socket("my_socket")); } -TEST("require that a server socket will remove an old socket file if it cannot be connected to") { +TEST_F(SocketTest, require_that_a_server_socket_will_remove_an_old_socket_file_if_it_cannot_be_connected_to) +{ remove_file("my_socket"); { SocketHandle server_handle = SocketAddress::from_path("my_socket").listen(); @@ -272,15 +335,21 @@ TEST("require that a server socket will remove an old socket file if it cannot b } #ifdef __linux__ -TEST_MT_FF("require that basic unix domain socket io works (name)", 2, - ServerSocket(make_string("ipc/name:my_socket-%d", int(getpid()))), TimeBomb(60)) +TEST_F(SocketTest, require_that_basic_unix_domain_socket_io_works_with_name) +{ + constexpr size_t num_threads = 2; + ServerSocket f1(make_string("ipc/name:my_socket-%d", int(getpid()))); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + bool is_server = (ctx.thread_id() == 0); + SocketHandle socket = connect_sockets(is_server, f1); + verify_socket_io(is_server, socket); + }; + Nexus::run(num_threads, task); +} + +TEST_F(SocketTest, require_that_two_server_sockets_cannot_have_the_same_abstract_unix_domain_socket_name) { - bool is_server = (thread_id == 0); - SocketHandle socket = connect_sockets(is_server, f1); - TEST_DO(verify_socket_io(is_server, socket)); -} - -TEST("require that two server sockets cannot have the same abstract unix domain socket name") { vespalib::string spec = make_string("ipc/name:my_socket-%d", int(getpid())); ServerSocket server1(spec); ServerSocket server2(spec); @@ -288,7 +357,8 @@ TEST("require that two server sockets cannot have the same abstract unix domain EXPECT_TRUE(!server2.valid()); } -TEST("require that abstract socket names are freed when the server socket is destructed") { +TEST_F(SocketTest, require_that_abstract_socket_names_are_freed_when_the_server_socket_is_destructed) +{ vespalib::string spec = make_string("ipc/name:my_socket-%d", int(getpid())); ServerSocket server1(spec); EXPECT_TRUE(server1.valid()); @@ -297,100 +367,162 @@ TEST("require that abstract socket names are freed when the server socket is des EXPECT_TRUE(server2.valid()); } -TEST("require that abstract sockets do not have socket files") { +TEST_F(SocketTest, require_that_abstract_sockets_do_not_have_socket_files) +{ vespalib::string name = make_string("my_socket-%d", int(getpid())); ServerSocket server(SocketSpec::from_name(name)); EXPECT_TRUE(server.valid()); EXPECT_TRUE(!is_socket(name)); - EXPECT_TRUE(!is_file(name)); + EXPECT_TRUE(!is_file(name)); } -TEST_MT_FFF("require that abstract and file-based unix domain sockets are not in conflict", 4, - ServerSocket(make_string("ipc/file:my_socket-%d", int(getpid()))), - ServerSocket(make_string("ipc/name:my_socket-%d", int(getpid()))), TimeBomb(60)) +TEST_F(SocketTest, require_that_abstract_and_file_based_unix_domain_sockets_are_not_in_conflict) { - bool is_server = ((thread_id % 2) == 0); - ServerSocket &server_socket = ((thread_id / 2) == 0) ? f1 : f2; - SocketHandle socket = connect_sockets(is_server, server_socket); - TEST_DO(verify_socket_io(is_server, socket)); + constexpr size_t num_threads = 4; + ServerSocket f1(make_string("ipc/file:my_socket-%d", int(getpid()))); + ServerSocket f2(make_string("ipc/name:my_socket-%d", int(getpid()))); + TimeBomb f3(60); + auto task = [&f1,&f2](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + bool is_server = ((thread_id % 2) == 0); + ServerSocket &server_socket = ((thread_id / 2) == 0) ? f1 : f2; + SocketHandle socket = connect_sockets(is_server, server_socket); + verify_socket_io(is_server, socket); + }; + Nexus::run(num_threads, task); } #endif -TEST("require that sockets can be set blocking and non-blocking") { +TEST_F(SocketTest, require_that_sockets_can_be_set_blocking_and_non_blocking) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_blocking(-1, true)); EXPECT_TRUE(handle.set_blocking(true)); - TEST_DO(verifier.verify_blocking(true)); + { + SCOPED_TRACE("verify blocking true"); + verifier.verify_blocking(true); + } EXPECT_TRUE(handle.set_blocking(false)); - TEST_DO(verifier.verify_blocking(false)); + { + SCOPED_TRACE("verify blocking false"); + verifier.verify_blocking(false); + } } -TEST("require that server sockets use non-blocking underlying socket") { +TEST_F(SocketTest, require_that_server_sockets_use_non_blocking_underlying_socket) +{ ServerSocket tcp_server("tcp/0"); ServerSocket ipc_server("ipc/file:my_socket"); test::SocketOptionsVerifier tcp_verifier(tcp_server.get_fd()); test::SocketOptionsVerifier ipc_verifier(ipc_server.get_fd()); - TEST_DO(tcp_verifier.verify_blocking(false)); - TEST_DO(ipc_verifier.verify_blocking(false)); + { + SCOPED_TRACE("verify tcp nonblocking"); + tcp_verifier.verify_blocking(false); + } + { + SCOPED_TRACE("verify ipc nonblocking"); + ipc_verifier.verify_blocking(false); + } } -TEST("require that tcp nodelay can be enabled and disabled") { +TEST_F(SocketTest, require_that_tcp_nodelay_can_be_enabled_and_disabled) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_nodelay(-1, true)); EXPECT_TRUE(handle.set_nodelay(true)); - TEST_DO(verifier.verify_nodelay(true)); + { + SCOPED_TRACE("verify nodelay true"); + verifier.verify_nodelay(true); + } EXPECT_TRUE(handle.set_nodelay(false)); - TEST_DO(verifier.verify_nodelay(false)); + { + SCOPED_TRACE("verify nodelay false"); + verifier.verify_nodelay(false); + } } -TEST("require that reuse addr can be set and cleared") { +TEST_F(SocketTest, require_that_reuse_addr_can_be_set_and_cleared) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_reuse_addr(-1, true)); EXPECT_TRUE(handle.set_reuse_addr(true)); - TEST_DO(verifier.verify_reuse_addr(true)); + { + SCOPED_TRACE("verify reuse addr true"); + verifier.verify_reuse_addr(true); + } EXPECT_TRUE(handle.set_reuse_addr(false)); - TEST_DO(verifier.verify_reuse_addr(false)); + { + SCOPED_TRACE("verify reuse addr false"); + verifier.verify_reuse_addr(false); + } } -TEST("require that ipv6_only can be set and cleared") { +TEST_F(SocketTest, require_that_ipv6_only_can_be_set_and_cleared) +{ if (ipv6_enabled) { SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_ipv6_only(-1, true)); EXPECT_TRUE(handle.set_ipv6_only(true)); - TEST_DO(verifier.verify_ipv6_only(true)); + { + SCOPED_TRACE("verify ipv6 only true"); + verifier.verify_ipv6_only(true); + } EXPECT_TRUE(handle.set_ipv6_only(false)); - TEST_DO(verifier.verify_ipv6_only(false)); + { + SCOPED_TRACE("verify ipv6 only false"); + verifier.verify_ipv6_only(false); + } } else { fprintf(stderr, "WARNING: skipping ipv6_only test since ipv6 is disabled"); } } -TEST("require that tcp keepalive can be set and cleared") { +TEST_F(SocketTest, require_that_tcp_keepalive_can_be_set_and_cleared) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_keepalive(-1, true)); EXPECT_TRUE(handle.set_keepalive(true)); - TEST_DO(verifier.verify_keepalive(true)); + { + SCOPED_TRACE("verify keepalive true"); + verifier.verify_keepalive(true); + } EXPECT_TRUE(handle.set_keepalive(false)); - TEST_DO(verifier.verify_keepalive(false)); + { + SCOPED_TRACE("verify keepalive false"); + verifier.verify_keepalive(false); + } } -TEST("require that tcp lingering can be adjusted") { +TEST_F(SocketTest, require_that_tcp_lingering_can_be_adjusted) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_linger(-1, true, 0)); EXPECT_TRUE(handle.set_linger(true, 0)); - TEST_DO(verifier.verify_linger(true, 0)); + { + SCOPED_TRACE("verify linger true 0"); + verifier.verify_linger(true, 0); + } EXPECT_TRUE(handle.set_linger(true, 10)); - TEST_DO(verifier.verify_linger(true, 10)); + { + SCOPED_TRACE("verify linger true 10"); + verifier.verify_linger(true, 10); + } EXPECT_TRUE(handle.set_linger(false, 0)); - TEST_DO(verifier.verify_linger(false, 0)); + { + SCOPED_TRACE("verify linger false 0"); + verifier.verify_linger(false, 0); + } EXPECT_TRUE(handle.set_linger(false, 10)); - TEST_DO(verifier.verify_linger(false, 0)); + { + SCOPED_TRACE("verify linger false 0 (overridden)"); + verifier.verify_linger(false, 0); + } } SocketHandle connect_async(const SocketAddress &addr) { @@ -411,7 +543,10 @@ SocketHandle connect_async(const SocketAddress &addr) { ctx.handle = addr.connect_async(); EXPECT_TRUE(ctx.handle.valid()); test::SocketOptionsVerifier verifier(ctx.handle.get()); - TEST_DO(verifier.verify_blocking(false)); + { + SCOPED_TRACE("verify blocking false"); + verifier.verify_blocking(false); + } if (ctx.handle.valid()) { selector.add(ctx.handle.get(), ctx, true, true); while (!ctx.connect_done) { @@ -420,23 +555,30 @@ SocketHandle connect_async(const SocketAddress &addr) { } selector.remove(ctx.handle.get()); } - EXPECT_EQUAL(ctx.error, 0); + EXPECT_EQ(ctx.error, 0); return std::move(ctx.handle); } -TEST_MT_FF("require that async connect pattern works", 2, ServerSocket("tcp/0"), TimeBomb(60)) { - if (thread_id == 0) { - SocketHandle socket = f1.accept(); - EXPECT_TRUE(socket.valid()); - TEST_DO(verify_socket_io(true, socket)); - } else { - SocketAddress addr = SocketSpec::from_port(f1.address().port()).client_address(); - SocketHandle socket = connect_async(addr); - socket.set_blocking(true); - TEST_DO(verify_socket_io(false, socket)); - // TEST_DO(connect_async(SocketAddress::select_remote(80, "www.yahoo.com"))); - // TEST_DO(connect_async(SocketAddress::select_remote(85, "myinternalhost"))); - } -} - -TEST_MAIN() { TEST_RUN_ALL(); } +TEST_F(SocketTest, require_that_async_connect_pattern_works) +{ + constexpr size_t num_threads = 2; + ServerSocket f1("tcp/0"); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + if (ctx.thread_id() == 0) { + SocketHandle socket = f1.accept(); + EXPECT_TRUE(socket.valid()); + SCOPED_TRACE("verify socket io true"); + verify_socket_io(true, socket); + } else { + SocketAddress addr = SocketSpec::from_port(f1.address().port()).client_address(); + SocketHandle socket = connect_async(addr); + socket.set_blocking(true); + SCOPED_TRACE("verify socket io false"); + verify_socket_io(false, socket); + } + }; + Nexus::run(num_threads, task); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/CMakeLists.txt b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/CMakeLists.txt index 6bbf0189862..87e13f14875 100644 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/CMakeLists.txt +++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(vespalib_net_tls_auto_reloading_tls_crypto_engine_test_app auto_reloading_tls_crypto_engine_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_net_tls_auto_reloading_tls_crypto_engine_test_app COMMAND vespalib_net_tls_auto_reloading_tls_crypto_engine_test_app) diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp index b6efb66c8bb..ed20dd6bcf4 100644 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp +++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp @@ -1,15 +1,17 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h> #include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/transport_security_options_reading.h> #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/testkit/time_bomb.h> #include <openssl/ssl.h> #include <filesystem> +#include <fstream> using namespace vespalib; using namespace vespalib::net::tls; @@ -89,6 +91,42 @@ void write_file(vespalib::stringref path, vespalib::stringref data) { f.write(data.data(), data.size(), 0); } +class AutoReloadingTlsCryptoEngineTest : public ::testing::Test +{ +protected: + AutoReloadingTlsCryptoEngineTest(); + ~AutoReloadingTlsCryptoEngineTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +AutoReloadingTlsCryptoEngineTest::AutoReloadingTlsCryptoEngineTest() + : testing::Test() +{ +} + +AutoReloadingTlsCryptoEngineTest::~AutoReloadingTlsCryptoEngineTest() = default; + +void +AutoReloadingTlsCryptoEngineTest::SetUpTestSuite() +{ + std::ofstream test_config("test_config.json"); + test_config << "{\n" << + " \"files\":{\n" << + " \"private-key\": \"" + TEST_PATH("test_key.pem") << "\",\n" << + " \"ca-certificates\": \"" + TEST_PATH("test_ca.pem") << "\",\n" << + " \"certificates\": \"test_cert.pem\"\n" << + " }\n" << + "}" << std::endl; + test_config.close(); +} + +void +AutoReloadingTlsCryptoEngineTest::TearDownTestSuite() +{ + std::filesystem::remove("test_config.json"); +} + struct Fixture { std::unique_ptr<AutoReloadingTlsCryptoEngine> engine; explicit Fixture(AutoReloadingTlsCryptoEngine::TimeInterval reload_interval, @@ -114,9 +152,13 @@ struct Fixture { } }; -TEST_FF("Config reloading transitively loads updated files", Fixture(50ms), TimeBomb(60)) { +TEST_F(AutoReloadingTlsCryptoEngineTest, config_reloading_transitively_loads_updated_files) +{ + Fixture f1(50ms); + TimeBomb f2(60); + auto current_certs = f1.current_cert_chain(); - ASSERT_EQUAL(cert1_pem, current_certs); + ASSERT_EQ(cert1_pem, current_certs); write_file("test_cert.pem.tmp", cert2_pem); std::filesystem::rename(std::filesystem::path("test_cert.pem.tmp"), std::filesystem::path("test_cert.pem")); // We expect this to be an atomic rename under the hood @@ -129,15 +171,25 @@ TEST_FF("Config reloading transitively loads updated files", Fixture(50ms), Time // If the config is never reloaded, test will go boom. } -TEST_FF("Shutting down auto-reloading engine immediately stops background thread", Fixture(600s), TimeBomb(60)) { +TEST_F(AutoReloadingTlsCryptoEngineTest, shutting_down_auto_reloading_engine_immediately_stops_background_thread) +{ + Fixture f1(600s); + TimeBomb f2(60); // This passes just from not having the TimeBomb blow up. } -TEST_FF("Authorization mode is propagated to engine", Fixture(50ms, AuthorizationMode::LogOnly), TimeBomb(60)) { - EXPECT_EQUAL(AuthorizationMode::LogOnly, f1.current_authorization_mode()); +TEST_F(AutoReloadingTlsCryptoEngineTest, authorization_mode_is_propagated_to_engine) +{ + Fixture f1(50ms, AuthorizationMode::LogOnly); + TimeBomb f2(60); + EXPECT_EQ(AuthorizationMode::LogOnly, f1.current_authorization_mode()); } -TEST_FF("Config reload failure increments failure statistic", Fixture(50ms), TimeBomb(60)) { +TEST_F(AutoReloadingTlsCryptoEngineTest, config_reload_failure_increments_failure_statistic) +{ + Fixture f1(50ms); + TimeBomb f2(60); + auto before = ConfigStatistics::get().snapshot(); write_file("test_cert.pem.tmp", "Broken file oh no :("); @@ -148,4 +200,4 @@ TEST_FF("Config reload failure increments failure statistic", Fixture(50ms), Tim } } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/test_config.json b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/test_config.json deleted file mode 100644 index 2b2322d928f..00000000000 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/test_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "files":{ - "private-key": "test_key.pem", - "ca-certificates": "test_ca.pem", - "certificates": "test_cert.pem" - } -} diff --git a/vespalib/src/tests/net/tls/transport_options/CMakeLists.txt b/vespalib/src/tests/net/tls/transport_options/CMakeLists.txt index 3623912bb42..2013879569f 100644 --- a/vespalib/src/tests/net/tls/transport_options/CMakeLists.txt +++ b/vespalib/src/tests/net/tls/transport_options/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(vespalib_net_tls_transport_options_test_app TEST transport_options_reading_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_net_tls_transport_options_test_app COMMAND vespalib_net_tls_transport_options_test_app) diff --git a/vespalib/src/tests/net/tls/transport_options/ok_config.json b/vespalib/src/tests/net/tls/transport_options/ok_config.json deleted file mode 100644 index dd2591661dc..00000000000 --- a/vespalib/src/tests/net/tls/transport_options/ok_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "files":{ - "private-key": "dummy_privkey.txt", - "ca-certificates": "dummy_ca_certs.txt", - "certificates": "dummy_certs.txt" - } -} diff --git a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp index ef0fdac0495..ea499607685 100644 --- a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp +++ b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp @@ -1,113 +1,245 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/transport_security_options_reading.h> #include <vespa/vespalib/test/peer_policy_utils.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/util/exceptions.h> +#include <gmock/gmock.h> +#include <filesystem> +#include <fstream> +#include <optional> +#include <sstream> using namespace vespalib; using namespace vespalib::net::tls; -TEST("can load TLS credentials via config file") { +namespace { + +class ConfigWriter { + std::optional<std::string> _private_key; + std::optional<std::string> _ca_certificates; + std::optional<std::string> _certificates; + std::optional<std::string> _accepted_ciphers; + std::optional<std::string> _authorized_peers; + std::optional<std::string> _disable_hostname_validation; + std::optional<std::string> _flipper_the_dolphin; +public: + ConfigWriter(); + ConfigWriter(ConfigWriter&&); + ~ConfigWriter(); + ConfigWriter private_key(std::optional<std::string> value) && { _private_key = value; return std::move(*this); } + ConfigWriter ca_certificates(std::optional<std::string> value) && { _ca_certificates = value; return std::move(*this); } + ConfigWriter certificates(std::optional<std::string> value) && { _certificates = value; return std::move(*this); } + ConfigWriter accepted_ciphers(std::optional<std::string> value) && { _accepted_ciphers = value; return std::move(*this); } + ConfigWriter authorized_peers(std::optional<std::string> value) && { _authorized_peers = value; return std::move(*this); } + ConfigWriter disable_hostname_validation(std::optional<std::string> value) && { _disable_hostname_validation = value; return std::move(*this); } + ConfigWriter flipper_the_dolphin(std::optional<std::string> value) && { _flipper_the_dolphin = value; return std::move(*this); } + void write(std::ostream& os); + std::string write(); +}; + +ConfigWriter::ConfigWriter() + : _private_key(TEST_PATH("dummy_privkey.txt")), + _ca_certificates(TEST_PATH("dummy_ca_certs.txt")), + _certificates(TEST_PATH("dummy_certs.txt")), + _accepted_ciphers(), + _authorized_peers(), + _disable_hostname_validation(), + _flipper_the_dolphin() +{ +} + +ConfigWriter::ConfigWriter(ConfigWriter&&) = default; + +ConfigWriter::~ConfigWriter() = default; + +void +ConfigWriter::write(std::ostream& os) +{ + os << "{\n" << + R"( "files": {)"; + bool had_files_entry = false; + if (_private_key.has_value()) { + os << "\n" << R"( "private-key": ")" << _private_key.value() << R"(")"; + had_files_entry = true; + } + if (_ca_certificates.has_value()) { + if (had_files_entry) { + os << ","; + } + os << "\n" << R"( "ca-certificates": ")" << _ca_certificates.value() << R"(")"; + had_files_entry = true; + } + if (_certificates.has_value()) { + if (had_files_entry) { + os << ","; + } + os << "\n" << R"( "certificates": ")" << _certificates.value() << R"(")"; + had_files_entry = true; + } + os << "\n }"; + if (_accepted_ciphers.has_value()) { + os << ",\n" << R"( "accepted-ciphers" : )" << _accepted_ciphers.value(); + } + if (_authorized_peers.has_value()) { + os << ",\n" << R"( "authorized-peers": )" << _authorized_peers.value(); + } + if (_disable_hostname_validation.has_value()) { + os << ",\n" << R"( "disable-hostname-validation": )" << _disable_hostname_validation.value(); + } + if (_flipper_the_dolphin.has_value()) { + os << ",\n" << R"( "flipper-the-dolphin": )" << _flipper_the_dolphin.value(); + } + os << "\n}" << std::endl; +} + +std::string +ConfigWriter::write() +{ + std::ostringstream os; + write(os); + return os.str(); +} + +} + +class TransportSecurityOptionsTest : public ::testing::Test { +protected: + TransportSecurityOptionsTest(); + ~TransportSecurityOptionsTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +TransportSecurityOptionsTest::TransportSecurityOptionsTest() + : ::testing::Test() +{ +} + +TransportSecurityOptionsTest::~TransportSecurityOptionsTest() = default; + +void +TransportSecurityOptionsTest::SetUpTestSuite() +{ + std::ofstream ok_config("ok_config.json"); + ConfigWriter().write(ok_config); + ok_config.close(); +} + +void +TransportSecurityOptionsTest::TearDownTestSuite() +{ + std::filesystem::remove("ok_config.json"); +} + +TEST_F(TransportSecurityOptionsTest, can_load_tls_credentials_via_config_file) +{ auto opts = read_options_from_json_file("ok_config.json"); ASSERT_TRUE(opts.get() != nullptr); // Obviously we'd need to change this to actual PEM data if config reading started // actually verifying the _content_ of files, not just reading them. - EXPECT_EQUAL("My private key\n", opts->private_key_pem()); - EXPECT_EQUAL("My CA certificates\n", opts->ca_certs_pem()); - EXPECT_EQUAL("My certificate chain\n", opts->cert_chain_pem()); + EXPECT_EQ("My private key\n", opts->private_key_pem()); + EXPECT_EQ("My CA certificates\n", opts->ca_certs_pem()); + EXPECT_EQ("My certificate chain\n", opts->cert_chain_pem()); } -TEST("copying options without private key does, in fact, not include private key") { +TEST_F(TransportSecurityOptionsTest, copying_options_without_private_key_does_in_fact_not_include_private_key) +{ auto opts = read_options_from_json_file("ok_config.json"); auto cloned = opts->copy_without_private_key(); - EXPECT_EQUAL("", cloned.private_key_pem()); - EXPECT_EQUAL("My CA certificates\n", cloned.ca_certs_pem()); - EXPECT_EQUAL("My certificate chain\n", cloned.cert_chain_pem()); + EXPECT_EQ("", cloned.private_key_pem()); + EXPECT_EQ("My CA certificates\n", cloned.ca_certs_pem()); + EXPECT_EQ("My certificate chain\n", cloned.cert_chain_pem()); } -TEST("missing JSON file throws exception") { - EXPECT_EXCEPTION(read_options_from_json_file("missing_config.json"), IllegalArgumentException, - "TLS config file 'missing_config.json' could not be read"); +TEST_F(TransportSecurityOptionsTest, missing_json_file_throws_exception) +{ + EXPECT_THAT([]() { read_options_from_json_file("missing_config.json"); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config file 'missing_config.json' could not be read"))); } -TEST("bad JSON content throws exception") { +TEST_F(TransportSecurityOptionsTest, bad_json_content_throws_exception) +{ const char* bad_json = "hello world :D"; - EXPECT_EXCEPTION(read_options_from_json_string(bad_json), IllegalArgumentException, - "Provided TLS config file is not valid JSON"); + EXPECT_THAT([bad_json]() { read_options_from_json_string(bad_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("Provided TLS config file is not valid JSON"))); } -TEST("missing 'files' field throws exception") { +TEST_F(TransportSecurityOptionsTest, missing_files_field_throws_exception) +{ const char* incomplete_json = R"({})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "TLS config root field 'files' is missing or empty"); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config root field 'files' is missing or empty"))); } -TEST("missing 'private-key' field throws exception") { - const char* incomplete_json = R"({"files":{"certificates":"dummy_certs.txt","ca-certificates":"dummy_ca_certs.txt"}})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "TLS config field 'private-key' has not been set"); +TEST_F(TransportSecurityOptionsTest, missing_private_key_field_throws_exception) +{ + auto incomplete_json = ConfigWriter().private_key(std::nullopt).write(); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config field 'private-key' has not been set"))); } -TEST("missing 'certificates' field throws exception") { - const char* incomplete_json = R"({"files":{"private-key":"dummy_privkey.txt","ca-certificates":"dummy_ca_certs.txt"}})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "TLS config field 'certificates' has not been set"); +TEST_F(TransportSecurityOptionsTest, missing_certificates_field_throws_exception) +{ + auto incomplete_json = ConfigWriter().certificates(std::nullopt).write(); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config field 'certificates' has not been set"))); } -TEST("missing 'ca-certificates' field throws exception") { - const char* incomplete_json = R"({"files":{"private-key":"dummy_privkey.txt","certificates":"dummy_certs.txt"}})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "TLS config field 'ca-certificates' has not been set"); +TEST_F(TransportSecurityOptionsTest, missing_ca_certificates_field_throws_exception) +{ + auto incomplete_json = ConfigWriter().ca_certificates(std::nullopt).write(); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config field 'ca-certificates' has not been set"))); } -TEST("missing file referenced by field throws exception") { - const char* incomplete_json = R"({"files":{"private-key":"missing_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "File 'missing_privkey.txt' referenced by TLS config does not exist"); +TEST_F(TransportSecurityOptionsTest, missing_file_referenced_by_field_throws_exception) +{ + auto incomplete_json = ConfigWriter().private_key("missing_privkey.txt").write(); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("File 'missing_privkey.txt' referenced by TLS config does not exist"))); } vespalib::string json_with_policies(const vespalib::string& policies) { - const char* fmt = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "authorized-peers":[%s]})"; - return vespalib::make_string(fmt, policies.c_str()); + return ConfigWriter().authorized_peers(std::string("[") + policies + "]").write(); } -TransportSecurityOptions parse_policies(const vespalib::string& policies) { +TransportSecurityOptions parse_policies(const vespalib::string& policies) +{ return *read_options_from_json_string(json_with_policies(policies)); } -TEST("config file without authorized-peers accepts all pre-verified certificates") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}})"; +TEST_F(TransportSecurityOptionsTest, config_file_without_authorized_peers_accepts_all_pre_verified_certificates) +{ + auto json = ConfigWriter().write(); EXPECT_TRUE(read_options_from_json_string(json)->authorized_peers().allows_all_authenticated()); } // Instead of contemplating what the semantics of an empty allow list should be, // we do the easy way out and just say it's not allowed in the first place. -TEST("empty policy array throws exception") { - EXPECT_EXCEPTION(parse_policies(""), vespalib::IllegalArgumentException, - "\"authorized-peers\" must either be not present (allows " - "all peers with valid certificates) or a non-empty array"); +TEST_F(TransportSecurityOptionsTest, empty_policy_array_throws_exception) +{ + EXPECT_THAT([]() { parse_policies(""); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("\"authorized-peers\" must either be not present (allows " + "all peers with valid certificates) or a non-empty array"))); } -TEST("can parse single peer policy with single requirement") { +TEST_F(TransportSecurityOptionsTest, can_parse_single_peer_policy_with_single_requirement) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "hello.world"} ] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("hello.world")})}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("hello.world")})}), + parse_policies(json).authorized_peers()); } -TEST("can parse single peer policy with multiple requirements") { +TEST_F(TransportSecurityOptionsTest, can_parse_single_peer_policy_with_multiple_requirements) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "hello.world"}, @@ -115,57 +247,56 @@ TEST("can parse single peer policy with multiple requirements") { {"field": "CN", "must-match": "goodbye.moon"} ] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("hello.world"), - required_san_uri("foo://bar/baz"), - required_cn("goodbye.moon")})}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("hello.world"), + required_san_uri("foo://bar/baz"), + required_cn("goodbye.moon")})}), + parse_policies(json).authorized_peers()); } -TEST("unknown field type throws exception") { +TEST_F(TransportSecurityOptionsTest, unknown_field_type_throws_exception) +{ const char* json = R"({ "required-credentials":[ {"field": "winnie the pooh", "must-match": "piglet"} ] })"; - EXPECT_EXCEPTION(parse_policies(json), vespalib::IllegalArgumentException, - "Unsupported credential field type: 'winnie the pooh'. Supported are: CN, SAN_DNS"); + EXPECT_THAT([json]() { parse_policies(json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("Unsupported credential field type: 'winnie the pooh'. Supported are: CN, SAN_DNS"))); } -TEST("empty required-credentials array throws exception") { +TEST_F(TransportSecurityOptionsTest, empty_required_credentials_array_throws_exception) +{ const char* json = R"({ "required-credentials":[] })"; - EXPECT_EXCEPTION(parse_policies(json), vespalib::IllegalArgumentException, - "\"required-credentials\" array can't be empty (would allow all peers)"); + EXPECT_THAT([json]() { parse_policies(json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("\"required-credentials\" array can't be empty (would allow all peers)"))); } -TEST("accepted cipher list is empty if not specified") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}})"; +TEST_F(TransportSecurityOptionsTest, accepted_cipher_list_is_empty_if_not_specified) +{ + auto json = ConfigWriter().write(); EXPECT_TRUE(read_options_from_json_string(json)->accepted_ciphers().empty()); } -TEST("accepted cipher list is populated if specified") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "accepted-ciphers":["foo", "bar"]})"; +TEST_F(TransportSecurityOptionsTest, accepted_cipher_list_is_populated_if_specified) +{ + auto json = ConfigWriter().accepted_ciphers(R"(["foo", "bar"])").write(); auto ciphers = read_options_from_json_string(json)->accepted_ciphers(); - ASSERT_EQUAL(2u, ciphers.size()); - EXPECT_EQUAL("foo", ciphers[0]); - EXPECT_EQUAL("bar", ciphers[1]); + ASSERT_EQ(2u, ciphers.size()); + EXPECT_EQ("foo", ciphers[0]); + EXPECT_EQ("bar", ciphers[1]); } // FIXME this is temporary until we know enabling it by default won't break the world! -TEST("hostname validation is DISABLED by default when creating options from config file") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}})"; +TEST_F(TransportSecurityOptionsTest, hostname_validation_is_DISABLED_by_default_when_creating_options_from_config_file) +{ + auto json = ConfigWriter().write(); EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation()); } -TEST("TransportSecurityOptions builder does not disable hostname validation by default") { +TEST_F(TransportSecurityOptionsTest, transport_security_options_builder_does_not_disable_hostname_validation_by_default) +{ auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). ca_certs_pem("foo"). cert_chain_pem("bar"). @@ -174,67 +305,65 @@ TEST("TransportSecurityOptions builder does not disable hostname validation by d EXPECT_FALSE(ts_opts.disable_hostname_validation()); } -TEST("hostname validation can be explicitly disabled") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "disable-hostname-validation": true})"; +TEST_F(TransportSecurityOptionsTest, hostname_validation_can_be_explicitly_disabled) +{ + auto json = ConfigWriter().disable_hostname_validation("true").write(); EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation()); } -TEST("hostname validation can be explicitly enabled") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "disable-hostname-validation": false})"; +TEST_F(TransportSecurityOptionsTest, hostname_validation_can_be_explicitly_enabled) +{ + auto json = ConfigWriter().disable_hostname_validation("false").write(); EXPECT_FALSE(read_options_from_json_string(json)->disable_hostname_validation()); } -TEST("unknown fields are ignored at parse-time") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "flipper-the-dolphin": "*weird dolphin noises*"})"; +TEST_F(TransportSecurityOptionsTest, unknown_fields_are_ignored_at_parse_time) +{ + auto json = ConfigWriter().flipper_the_dolphin(R"("*weird dolphin noises*")").write(); EXPECT_TRUE(read_options_from_json_string(json).get() != nullptr); // And no exception thrown. } -TEST("policy without explicit capabilities implicitly get all capabilities") { +TEST_F(TransportSecurityOptionsTest, policy_without_explicit_capabilities_implicitly_get_all_capabilities) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "hello.world"} ] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("hello.world")}, - CapabilitySet::make_with_all_capabilities())}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("hello.world")}, + CapabilitySet::make_with_all_capabilities())}), + parse_policies(json).authorized_peers()); } -TEST("specifying a capability set adds all its underlying capabilities") { +TEST_F(TransportSecurityOptionsTest, specifying_a_capability_set_adds_all_its_underlying_capabilities) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } ], "capabilities": ["vespa.content_node"] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, - CapabilitySet::content_node())}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, + CapabilitySet::content_node())}), + parse_policies(json).authorized_peers()); } -TEST("can specify single leaf capabilities") { +TEST_F(TransportSecurityOptionsTest, can_specify_single_leaf_capabilities) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } ], "capabilities": ["vespa.content.metrics_api", "vespa.slobrok.api"] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, - CapabilitySet::of({Capability::content_metrics_api(), - Capability::slobrok_api()}))}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, + CapabilitySet::of({Capability::content_metrics_api(), + Capability::slobrok_api()}))}), + parse_policies(json).authorized_peers()); } -TEST("specifying multiple capability sets adds union of underlying capabilities") { +TEST_F(TransportSecurityOptionsTest, specifying_multiple_capability_sets_adds_union_of_underlying_capabilities) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } @@ -244,23 +373,22 @@ TEST("specifying multiple capability sets adds union of underlying capabilities" CapabilitySet caps; caps.add_all(CapabilitySet::content_node()); caps.add_all(CapabilitySet::container_node()); - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, caps)}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, caps)}), + parse_policies(json).authorized_peers()); } -TEST("empty capabilities array is not allowed") { +TEST_F(TransportSecurityOptionsTest, empty_capabilities_array_is_not_allowed) { const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } ], "capabilities": [] })"; - EXPECT_EXCEPTION(parse_policies(json), vespalib::IllegalArgumentException, - "\"capabilities\" array must either be not present (implies " - "all capabilities) or contain at least one capability name"); + EXPECT_THAT([json]() { parse_policies(json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("\"capabilities\" array must either be not present (implies " + "all capabilities) or contain at least one capability name"))); } // TODO test parsing of multiple policies -TEST_MAIN() { TEST_RUN_ALL(); } - +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp index 212ea417524..62b7dc5f179 100644 --- a/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp +++ b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/polymorphicarrays.h> +#include <cassert> using namespace vespalib; diff --git a/vespalib/src/tests/priority_queue/priority_queue_test.cpp b/vespalib/src/tests/priority_queue/priority_queue_test.cpp index ae85dcfa47a..0f422cacb6d 100644 --- a/vespalib/src/tests/priority_queue/priority_queue_test.cpp +++ b/vespalib/src/tests/priority_queue/priority_queue_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("priority_queue_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/priority_queue.h> using namespace vespalib; diff --git a/vespalib/src/tests/rendezvous/rendezvous_test.cpp b/vespalib/src/tests/rendezvous/rendezvous_test.cpp index d2e2ac2fbab..13c4f968db1 100644 --- a/vespalib/src/tests/rendezvous/rendezvous_test.cpp +++ b/vespalib/src/tests/rendezvous/rendezvous_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/rendezvous.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/count_down_latch.h> #include <utility> #include <thread> diff --git a/vespalib/src/tests/rusage/rusage_test.cpp b/vespalib/src/tests/rusage/rusage_test.cpp index 5c08c99de43..28d3db72099 100644 --- a/vespalib/src/tests/rusage/rusage_test.cpp +++ b/vespalib/src/tests/rusage/rusage_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/rusage.h> using namespace vespalib; diff --git a/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp index 66f155f679b..bcd8ddb24f5 100644 --- a/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/adaptive_sequenced_executor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> #include <condition_variable> diff --git a/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp index eb71709ae43..d899b8b4a2c 100644 --- a/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/foregroundtaskexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <condition_variable> #include <unistd.h> diff --git a/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 96cc23ef70e..474df9bdf3b 100644 --- a/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -4,7 +4,7 @@ #include <vespa/vespalib/util/adaptive_sequenced_executor.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/singleexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> #include <condition_variable> diff --git a/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp b/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp index 0f1c6d3a083..18613c0bc79 100644 --- a/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp +++ b/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/barrier.h> #include <thread> +#include <cassert> using vespalib::steady_clock; diff --git a/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp b/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp index 3e0935e85e0..6376386096e 100644 --- a/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp +++ b/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/shutdownguard.h> #include <vespa/vespalib/util/malloc_mmap_guard.h> #include <thread> diff --git a/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp index 3ae0b709e31..224a4a95afe 100644 --- a/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp +++ b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/singleexecutor.h> #include <vespa/vespalib/util/lambdatask.h> diff --git a/vespalib/src/tests/slime/json_slime_benchmark.cpp b/vespalib/src/tests/slime/json_slime_benchmark.cpp index 78000a5a25d..df9492ba46c 100644 --- a/vespalib/src/tests/slime/json_slime_benchmark.cpp +++ b/vespalib/src/tests/slime/json_slime_benchmark.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <iostream> #include <fstream> +#include <cassert> using namespace vespalib::slime::convenience; diff --git a/vespalib/src/tests/slime/slime_test.cpp b/vespalib/src/tests/slime/slime_test.cpp index 591b3bd6465..d2d1e368715 100644 --- a/vespalib/src/tests/slime/slime_test.cpp +++ b/vespalib/src/tests/slime/slime_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/object_value.h> #include <vespa/vespalib/data/slime/array_value.h> diff --git a/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp b/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp index 87ecb42efa8..065984e5ed9 100644 --- a/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp +++ b/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> +#include <cassert> using namespace vespalib; using namespace vespalib::slime::convenience; diff --git a/vespalib/src/tests/stllike/cache_test.cpp b/vespalib/src/tests/stllike/cache_test.cpp index 9171f923ecf..79f8515162d 100644 --- a/vespalib/src/tests/stllike/cache_test.cpp +++ b/vespalib/src/tests/stllike/cache_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/cache.hpp> #include <map> diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index 493ea700fb2..592fc72edeb 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_map_equal.hpp> diff --git a/vespalib/src/tests/stllike/hashtable_test.cpp b/vespalib/src/tests/stllike/hashtable_test.cpp index c7890c79164..4880c7a872d 100644 --- a/vespalib/src/tests/stllike/hashtable_test.cpp +++ b/vespalib/src/tests/stllike/hashtable_test.cpp @@ -4,7 +4,7 @@ #include <vespa/vespalib/stllike/hashtable.hpp> #include <vespa/vespalib/stllike/hash_fun.h> #include <vespa/vespalib/stllike/identity.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <memory> #include <vector> diff --git a/vespalib/src/tests/stllike/lrucache.cpp b/vespalib/src/tests/stllike/lrucache.cpp index 3722845919d..9f81b29394e 100644 --- a/vespalib/src/tests/stllike/lrucache.cpp +++ b/vespalib/src/tests/stllike/lrucache.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/lrucache_map.hpp> diff --git a/vespalib/src/tests/stllike/vector_map_test.cpp b/vespalib/src/tests/stllike/vector_map_test.cpp index 00794bcc1e3..18b573f323b 100644 --- a/vespalib/src/tests/stllike/vector_map_test.cpp +++ b/vespalib/src/tests/stllike/vector_map_test.cpp @@ -5,7 +5,7 @@ LOG_SETUP("vector_map_test"); #include <vespa/vespalib/stllike/vector_map.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using vespalib::vector_map; diff --git a/vespalib/src/tests/sync/sync_test.cpp b/vespalib/src/tests/sync/sync_test.cpp index df781dea423..dd15e2f0057 100644 --- a/vespalib/src/tests/sync/sync_test.cpp +++ b/vespalib/src/tests/sync/sync_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/gate.h> using namespace vespalib; diff --git a/vespalib/src/tests/text/lowercase/lowercase_test.cpp b/vespalib/src/tests/text/lowercase/lowercase_test.cpp index 40803b8f295..75512a733a5 100644 --- a/vespalib/src/tests/text/lowercase/lowercase_test.cpp +++ b/vespalib/src/tests/text/lowercase/lowercase_test.cpp @@ -2,7 +2,6 @@ #include <vespa/log/log.h> LOG_SETUP("lowercase_test"); #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/text/lowercase.h> #include <iostream> #include <fstream> diff --git a/vespalib/src/tests/trace/trace.cpp b/vespalib/src/tests/trace/trace.cpp index 2b35740deb1..5dce0e5cc6b 100644 --- a/vespalib/src/tests/trace/trace.cpp +++ b/vespalib/src/tests/trace/trace.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/trace/trace.h> #include <vespa/vespalib/trace/tracevisitor.h> diff --git a/vespalib/src/tests/valgrind/valgrind_test.cpp b/vespalib/src/tests/valgrind/valgrind_test.cpp index e9fd265cfa3..afecff17eb9 100644 --- a/vespalib/src/tests/valgrind/valgrind_test.cpp +++ b/vespalib/src/tests/valgrind/valgrind_test.cpp @@ -1,20 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/valgrind.h> using namespace vespalib; -class Test : public TestApp -{ - int Main() override; - void testUninitializedUser(); - void testUninitializedSystemCall(); - void testInitializedUser(); - void testInitializedSystemCall(); -}; - -void Test::testUninitializedUser() +void testUninitializedUser() { char buf[7]; buf[0] = 7; @@ -22,45 +13,37 @@ void Test::testUninitializedUser() Valgrind::testUninitialized(buf, sizeof(buf)); } -void Test::testUninitializedSystemCall() +void testUninitializedSystemCall() { char buf[7]; buf[0] = 7; buf[5] = 7; Valgrind::testSystemCall(buf, sizeof(buf)); } -void Test::testInitializedUser() +void testInitializedUser() { char buf[7]; memset(buf, 0, sizeof(buf)); Valgrind::testUninitialized(buf, sizeof(buf)); } -void Test::testInitializedSystemCall() +void testInitializedSystemCall() { char buf[7]; memset(buf, 0, sizeof(buf)); Valgrind::testSystemCall(buf, sizeof(buf)); } -int -Test::Main() -{ - TEST_INIT("valgrind_test"); - - if (strcmp(_argv[1], "testInitializedUser") == 0) { +TEST_MAIN() { + if (strcmp(argv[1], "testInitializedUser") == 0) { testInitializedUser(); - } else if (strcmp(_argv[1], "testInitializedSystemCall") == 0) { + } else if (strcmp(argv[1], "testInitializedSystemCall") == 0) { testInitializedSystemCall(); - } else if (strcmp(_argv[1], "testUninitializedUser") == 0) { + } else if (strcmp(argv[1], "testUninitializedUser") == 0) { testUninitializedUser(); - } else if (strcmp(_argv[1], "testUninitializedSystemCall") == 0) { + } else if (strcmp(argv[1], "testUninitializedSystemCall") == 0) { testUninitializedSystemCall(); } else { testInitializedUser(); } - - TEST_DONE(); } - -TEST_APPHOOK(Test) diff --git a/vespalib/src/tests/zcurve/zcurve_test.cpp b/vespalib/src/tests/zcurve/zcurve_test.cpp index 8ed92304783..453bf6ddeba 100644 --- a/vespalib/src/tests/zcurve/zcurve_test.cpp +++ b/vespalib/src/tests/zcurve/zcurve_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/geo/zcurve.h> #include <algorithm> #include <limits> @@ -10,45 +10,14 @@ #include <vespa/log/log.h> LOG_SETUP("zcurve_test"); -namespace vespalib { +using namespace vespalib; using geo::ZCurve; -class ZCurveTest : public vespalib::TestApp -{ -public: - ZCurveTest(void) - : vespalib::TestApp() - { - } - - void testEncoding(); - - void testDecoding(); - - double ftime(); - - static inline int64_t encodexy3(int32_t x, int32_t y); - #define BMLIMIT 0x1000000 - template <bool decode> - int64_t bm(); - - template <bool decode> - int64_t bm2(); - - template <bool decode> - int64_t bm3(); - - int64_t bmcheck(); - - int Main() override; -}; - - void -ZCurveTest::testEncoding(void) +testEncoding(void) { int32_t x = 0; int32_t y = 0; @@ -83,7 +52,7 @@ ZCurveTest::testEncoding(void) void -ZCurveTest::testDecoding(void) +testDecoding(void) { int32_t x = 0; int32_t y = 0; @@ -134,7 +103,7 @@ ZCurveTest::testDecoding(void) double -ZCurveTest::ftime() +ftime() { struct timeval tv; gettimeofday(&tv, NULL); @@ -142,7 +111,7 @@ ZCurveTest::ftime() } int64_t -ZCurveTest::encodexy3(int32_t x, int32_t y) +encodexy3(int32_t x, int32_t y) { uint32_t resxl; uint32_t resxh; @@ -177,7 +146,7 @@ ZCurveTest::encodexy3(int32_t x, int32_t y) template <bool decode> int64_t -ZCurveTest::bm() +bm() { int64_t res = 0; double before = ftime(); @@ -218,7 +187,7 @@ ZCurveTest::bm() template <bool decode> int64_t -ZCurveTest::bm2(void) +bm2(void) { int64_t res = 0; double before = ftime(); @@ -259,7 +228,7 @@ ZCurveTest::bm2(void) template <bool decode> int64_t -ZCurveTest::bm3() +bm3() { int64_t res = 0; double before = ftime(); @@ -299,7 +268,7 @@ ZCurveTest::bm3() int64_t -ZCurveTest::bmcheck() +bmcheck() { int64_t res = 0; double before = ftime(); @@ -340,12 +309,7 @@ ZCurveTest::bmcheck() return res; } - -int -ZCurveTest::Main() -{ - TEST_INIT("zcurve_test"); - +TEST_MAIN() { for (int32_t x = 0; x < 4; x++) { for (int32_t y = 0; y < 4; y++) { int64_t enc = 0; @@ -368,7 +332,7 @@ ZCurveTest::Main() } testEncoding(); testDecoding(); - if (_argc >= 2) { + if (argc >= 2) { int64_t enc1 = bm<true>(); int64_t enc1b = bm<false>(); int64_t enc2 = bm2<true>(); @@ -383,10 +347,4 @@ ZCurveTest::Main() ASSERT_TRUE(enc1 == enc3b); ASSERT_TRUE(enc1 == enc4); } - - TEST_DONE(); } - -} - -TEST_APPHOOK(vespalib::ZCurveTest); diff --git a/vespalib/src/vespa/vespalib/test/CMakeLists.txt b/vespalib/src/vespa/vespalib/test/CMakeLists.txt index 4a43f9a92dd..b3b872f14c7 100644 --- a/vespalib/src/vespa/vespalib/test/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/test/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(vespalib_vespalib_test OBJECT memory_allocator_observer.cpp nexus.cpp peer_policy_utils.cpp + test_data_base.cpp thread_meets.cpp time_tracer.cpp DEPENDS diff --git a/vespalib/src/vespa/vespalib/test/socket_options_verifier.h b/vespalib/src/vespa/vespalib/test/socket_options_verifier.h index 04b4dea414e..3831c03d629 100644 --- a/vespalib/src/vespa/vespalib/test/socket_options_verifier.h +++ b/vespalib/src/vespa/vespalib/test/socket_options_verifier.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <fcntl.h> #include <unistd.h> #include <netinet/tcp.h> @@ -16,9 +16,9 @@ namespace { void verify_bool_opt(int fd, int level, int name, bool expect) { int data = 0; socklen_t size = sizeof(data); - EXPECT_EQUAL(getsockopt(fd, level, name, &data, &size), 0); - EXPECT_EQUAL(size, sizeof(data)); - EXPECT_EQUAL(data != 0, expect); + EXPECT_EQ(getsockopt(fd, level, name, &data, &size), 0); + EXPECT_EQ(size, sizeof(data)); + EXPECT_EQ(data != 0, expect); } } // namespace vespalib::test::<unnamed> @@ -31,31 +31,35 @@ struct SocketOptionsVerifier { SocketOptionsVerifier(int fd_in) : fd(fd_in) {} void verify_blocking(bool value) { int flags = fcntl(fd, F_GETFL, NULL); - EXPECT_NOT_EQUAL(flags, -1); - EXPECT_EQUAL(((flags & O_NONBLOCK) == 0), value); + EXPECT_NE(flags, -1); + EXPECT_EQ(((flags & O_NONBLOCK) == 0), value); } void verify_nodelay(bool value) { - TEST_DO(verify_bool_opt(fd, IPPROTO_TCP, TCP_NODELAY, value)); + SCOPED_TRACE("verify nodelay"); + verify_bool_opt(fd, IPPROTO_TCP, TCP_NODELAY, value); } void verify_reuse_addr(bool value) { - TEST_DO(verify_bool_opt(fd, SOL_SOCKET, SO_REUSEADDR, value)); + SCOPED_TRACE("verify reuse addr"); + verify_bool_opt(fd, SOL_SOCKET, SO_REUSEADDR, value); } void verify_ipv6_only(bool value) { - TEST_DO(verify_bool_opt(fd, IPPROTO_IPV6, IPV6_V6ONLY, value)); + SCOPED_TRACE("verify ipv6 only"); + verify_bool_opt(fd, IPPROTO_IPV6, IPV6_V6ONLY, value); } void verify_keepalive(bool value) { - TEST_DO(verify_bool_opt(fd, SOL_SOCKET, SO_KEEPALIVE, value)); + SCOPED_TRACE("verify keepalive"); + verify_bool_opt(fd, SOL_SOCKET, SO_KEEPALIVE, value); } void verify_linger(bool enable, int value) { struct linger data; socklen_t size = sizeof(data); memset(&data, 0, sizeof(data)); - EXPECT_EQUAL(getsockopt(fd, SOL_SOCKET, SO_LINGER, &data, &size), 0); - EXPECT_EQUAL(size, sizeof(data)); - EXPECT_EQUAL(enable, data.l_onoff); + EXPECT_EQ(getsockopt(fd, SOL_SOCKET, SO_LINGER, &data, &size), 0); + EXPECT_EQ(size, sizeof(data)); + EXPECT_EQ(enable, data.l_onoff); if (enable) { - EXPECT_EQUAL(value, data.l_linger); + EXPECT_EQ(value, data.l_linger); } } }; diff --git a/vespalib/src/vespa/vespalib/test/test_data.h b/vespalib/src/vespa/vespalib/test/test_data.h new file mode 100644 index 00000000000..1a5c24616b0 --- /dev/null +++ b/vespalib/src/vespa/vespalib/test/test_data.h @@ -0,0 +1,60 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "test_data_base.h" +#include <vespa/vespalib/gtest/gtest.h> +#include <filesystem> + +namespace vespalib::test { + +// Utility class for accessing test data used by unit tests. +template <class Derived> +class TestData : public TestDataBase { +protected: + static std::string _source_testdata; + static std::string _build_testdata; +public: + static void setup_test_data(const std::string& source_testdata_in, const std::string& build_testdata_in); + static void tear_down_test_data(); + static const std::string& source_testdata() noexcept { return _source_testdata; } + static const std::string& build_testdata() noexcept { return _build_testdata; } + void remove_unchanged_build_testdata_file_or_fail(const nbostream& buf, const std::string& file_name); +}; + +template <class Derived> +std::string TestData<Derived>::_source_testdata; + +template <class Derived> +std::string TestData<Derived>::_build_testdata; + +template <class Derived> +void +TestData<Derived>::setup_test_data(const std::string& source_testdata_in, const std::string& build_testdata_in) +{ + _source_testdata = source_testdata_in; + _build_testdata = build_testdata_in; + std::filesystem::create_directory(build_testdata()); +} + +template <class Derived> +void +TestData<Derived>::tear_down_test_data() +{ + std::filesystem::remove(build_testdata()); +} + +template <class Derived> +void +TestData<Derived>::remove_unchanged_build_testdata_file_or_fail(const nbostream& buf, const std::string& file_name) +{ + auto act_path = build_testdata() + "/" + file_name; + auto exp_path = source_testdata() + "/" + file_name; + ASSERT_TRUE(std::filesystem::exists(exp_path)) << "Missing expected contents file " << exp_path; + auto exp_buf = read_buffer_from_file(exp_path); + ASSERT_TRUE(equiv_buffers(exp_buf, buf)) << "Files " << exp_path << " and " << act_path << + " have diferent contents"; + std::filesystem::remove(act_path); +} + +} diff --git a/vespalib/src/vespa/vespalib/test/test_data_base.cpp b/vespalib/src/vespa/vespalib/test/test_data_base.cpp new file mode 100644 index 00000000000..12bfca50d5e --- /dev/null +++ b/vespalib/src/vespa/vespalib/test/test_data_base.cpp @@ -0,0 +1,45 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "test_data.h" +#include <vespa/vespalib/objects/nbostream.h> +#include <cassert> +#include <cstring> +#include <fstream> + +namespace vespalib::test { + +bool +TestDataBase::equiv_buffers(const nbostream& lhs, const nbostream& rhs) +{ + return lhs.size() == rhs.size() && memcmp(lhs.data(), rhs.data(), lhs.size()) == 0; +} + +nbostream +TestDataBase::read_buffer_from_file(const std::string& path) +{ + auto file = std::ifstream(path, std::ios::in | std::ios::binary | std::ios::ate); + auto size = file.tellg(); + file.seekg(0); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(size); + file.read(static_cast<char *>(buf.get()), size); + assert(file.good()); + file.close(); + return nbostream(std::move(buf), size); +} + +void +TestDataBase::write_buffer_to_file(const nbostream& buf, const std::string& path) +{ + write_buffer_to_file(std::string_view{buf.data(), buf.size()}, path); +} + +void +TestDataBase::write_buffer_to_file(std::string_view buf, const std::string& path) +{ + auto file = std::ofstream(path, std::ios::out | std::ios::binary | std::ios::trunc); + file.write(buf.data(), buf.size()); + assert(file.good()); + file.close(); +} + +} diff --git a/vespalib/src/vespa/vespalib/test/test_data_base.h b/vespalib/src/vespa/vespalib/test/test_data_base.h new file mode 100644 index 00000000000..65ea4533f81 --- /dev/null +++ b/vespalib/src/vespa/vespalib/test/test_data_base.h @@ -0,0 +1,20 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <string_view> + +namespace vespalib { class nbostream; } + +namespace vespalib::test { + +// Utility base class for accessing test data used by unit tests. +class TestDataBase { +public: + static bool equiv_buffers(const nbostream& lhs, const nbostream& rhs); + static nbostream read_buffer_from_file(const std::string& path); + static void write_buffer_to_file(const nbostream& buf, const std::string& path); + static void write_buffer_to_file(std::string_view buf, const std::string& path); +}; + +} diff --git a/vespalib/src/vespa/vespalib/testkit/CMakeLists.txt b/vespalib/src/vespa/vespalib/testkit/CMakeLists.txt index 0ec3754b66d..41bb1345f2c 100644 --- a/vespalib/src/vespa/vespalib/testkit/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/testkit/CMakeLists.txt @@ -6,7 +6,6 @@ vespa_add_library(vespalib_vespalib_testkit OBJECT test_master.cpp test_path.cpp test_state_guard.cpp - testapp.cpp time_bomb.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/testkit/test_hook.cpp b/vespalib/src/vespa/vespalib/testkit/test_hook.cpp index 4f7c7cf53bc..b14b575ae93 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_hook.cpp +++ b/vespalib/src/vespa/vespalib/testkit/test_hook.cpp @@ -1,8 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "test_hook.h" +#include <vespa/vespalib/util/count_down_latch.h> +#include <vespa/vespalib/util/barrier.h> +#include <vespa/vespalib/util/thread.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/size_literals.h> +#include <cassert> #include <regex> namespace vespalib { @@ -29,7 +32,7 @@ TestThreadWrapper::threadEntry() master.check(false, __FILE__, __LINE__, "test threw stuff", false); } _barrier.destroy(); - master.setThreadBarrier(0); + master.setThreadBarrier(nullptr); bool fail = (master.getThreadFailCnt() > preThreadFailCnt); master.setThreadUnwind(false); master.setThreadIgnore(false); @@ -39,30 +42,64 @@ TestThreadWrapper::threadEntry() master.setThreadName(oldThreadName.c_str()); } -TestHook *TestHook::_head = 0; -TestHook *TestHook::_tail = 0; +TestHook *TestHook::_head = nullptr; +TestHook *TestHook::_tail = nullptr; + +TestHook::~TestHook() = default; TestHook::TestHook(const std::string &file, const std::string &name, bool ignore) - : _next(0), + : _next(nullptr), _name(name), _tag(make_string("%s:%s", file.c_str(), name.c_str())), _ignore(ignore) { - if (_head == 0) { - assert(_tail == 0); + if (_head == nullptr) { + assert(_tail == nullptr); _head = this; _tail = this; } else { - assert(_tail != 0); - assert(_tail->_next == 0); + assert(_tail != nullptr); + assert(_tail->_next == nullptr); _tail->_next = this; _tail = this; } } +bool TestHook::runMyTest(const FixtureFactory & fixture_factory, size_t num_threads) { + assert(num_threads > 0); + using ThreadUP = std::unique_ptr<TestThreadWrapper>; + using FixtureUP = std::unique_ptr<TestFixtureWrapper>; + std::vector<TestMaster::TraceItem> traceStack = TestMaster::master.getThreadTraceStack(); + CountDownLatch latch(num_threads); + Barrier barrier(num_threads); + std::vector<FixtureUP> fixtures; + std::vector<ThreadUP> threads; + ThreadPool pool; + threads.reserve(num_threads); + fixtures.reserve(num_threads); + for (size_t i = 0; i < num_threads; ++i) { + FixtureUP fixture_up = fixture_factory(); + fixture_up->thread_id = i; + fixture_up->num_threads = num_threads; + threads.emplace_back(new TestThreadWrapper(_ignore, latch, barrier, traceStack, *fixture_up)); + fixtures.push_back(std::move(fixture_up)); + } + for (size_t i = 1; i < num_threads; ++i) { + pool.start([&target = *threads[i]](){ target.threadEntry(); }); + } + threads[0]->threadEntry(); + latch.await(); + pool.join(); + bool result = true; + for (size_t i = 0; i < num_threads; ++i) { + result = result && threads[i]->getResult(); + } + return result; +} + const char *lookup_subset_pattern(const std::string &name) { const char *pattern = getenv("TEST_SUBSET"); - if (pattern != 0) { + if (pattern != nullptr) { fprintf(stderr, "%s: info: only running tests matching '%s'\n", name.c_str(), pattern); } else { @@ -80,7 +117,7 @@ TestHook::runAll() size_t testsFailed = 0; size_t testsIgnored = 0; size_t testsSkipped = 0; - for (TestHook *test = _head; test != 0; test = test->_next) { + for (TestHook *test = _head; test != nullptr; test = test->_next) { if (std::regex_search(test->_tag, pattern)) { bool ignored = test->_ignore; bool failed = !test->run(); diff --git a/vespalib/src/vespa/vespalib/testkit/test_hook.h b/vespalib/src/vespa/vespalib/testkit/test_hook.h index fd9d743ff19..6887ae157d1 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_hook.h +++ b/vespalib/src/vespa/vespalib/testkit/test_hook.h @@ -2,28 +2,24 @@ #pragma once -#include <vespa/vespalib/util/count_down_latch.h> -#include <vespa/vespalib/util/barrier.h> -#include <vespa/vespalib/util/thread.h> -#include <thread> -#include <string> -#include <vector> -#include <cassert> #include "test_master.h" +#include <functional> namespace vespalib { +class CountDownLatch; + struct TestThreadEntry { virtual void threadEntry() = 0; - virtual ~TestThreadEntry() {} + virtual ~TestThreadEntry() = default; }; struct TestFixtureWrapper { size_t thread_id; size_t num_threads; - TestFixtureWrapper() : thread_id(0), num_threads(1) {} + TestFixtureWrapper() noexcept: thread_id(0), num_threads(1) {} virtual void test_entry_point() = 0; - virtual ~TestFixtureWrapper() {} + virtual ~TestFixtureWrapper() = default; }; class TestThreadWrapper : public TestThreadEntry @@ -39,13 +35,13 @@ private: public: TestThreadWrapper(bool ignore, CountDownLatch &l, Barrier &b, const std::vector<TestMaster::TraceItem> &traceStack, - TestFixtureWrapper &fixture) + TestFixtureWrapper &fixture) noexcept : _result(false), _ignore(ignore), _latch(l), _barrier(b), _traceStack(traceStack), _fixture(fixture) {} void threadEntry() override; - bool getResult() const { + bool getResult() const noexcept { return _result; } }; @@ -61,48 +57,21 @@ private: std::string _tag; bool _ignore; - TestHook(const TestHook &); - TestHook &operator=(const TestHook &); - + using FixtureFactory = std::function<std::unique_ptr<TestFixtureWrapper>()>; + bool runMyTest(const FixtureFactory &fixture_factory, size_t num_threads); protected: TestHook(const std::string &file, const std::string &name, bool ignore); - virtual ~TestHook() {} + virtual ~TestHook(); template <typename T> bool runTest(const T &fixture, size_t num_threads) { - assert(num_threads > 0); - using ThreadUP = std::unique_ptr<TestThreadWrapper>; - using FixtureUP = std::unique_ptr<T>; - std::vector<TestMaster::TraceItem> traceStack = TestMaster::master.getThreadTraceStack(); - CountDownLatch latch(num_threads); - Barrier barrier(num_threads); - std::vector<FixtureUP> fixtures; - std::vector<ThreadUP> threads; - ThreadPool pool; - threads.reserve(num_threads); - fixtures.reserve(num_threads); - for (size_t i = 0; i < num_threads; ++i) { - FixtureUP fixture_up(new T(fixture)); - fixture_up->thread_id = i; - fixture_up->num_threads = num_threads; - threads.emplace_back(new TestThreadWrapper(_ignore, latch, barrier, traceStack, *fixture_up)); - fixtures.push_back(std::move(fixture_up)); - } - for (size_t i = 1; i < num_threads; ++i) { - pool.start([&target = *threads[i]](){ target.threadEntry(); }); - } - threads[0]->threadEntry(); - latch.await(); - pool.join(); - bool result = true; - for (size_t i = 0; i < num_threads; ++i) { - result = result && threads[i]->getResult(); - } - return result; + return runMyTest([&fixture]() { return std::make_unique<T>(fixture); }, num_threads); } virtual bool run() = 0; public: + TestHook(const TestHook &) = delete; + TestHook &operator=(const TestHook &) = delete; static void runAll(); }; #endif diff --git a/vespalib/src/vespa/vespalib/testkit/test_macros.h b/vespalib/src/vespa/vespalib/testkit/test_macros.h index 907783ed0e9..cb9b4c580a1 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_macros.h +++ b/vespalib/src/vespa/vespalib/testkit/test_macros.h @@ -11,6 +11,7 @@ #define TEST_CAT(a, b) TEST_CAT_IMPL(a, b) #define TEST_MASTER vespalib::TestMaster::master #define TEST_DEBUG(lhsFile, rhsFile) TEST_MASTER.openDebugFiles(lhsFile, rhsFile) +#define TEST_DEBUG_STOP() TEST_MASTER.close_debug_files() #define TEST_STATE(msg) vespalib::TestStateGuard TEST_CAT(testStateGuard, __LINE__) (__FILE__, __LINE__, msg) #define TEST_DO(doit) do { TEST_STATE(TEST_STR(doit)); doit; } while(false) #define TEST_FLUSH() TEST_MASTER.flush(__FILE__, __LINE__) diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.cpp b/vespalib/src/vespa/vespalib/testkit/test_master.cpp index 20fa5a7e860..4698d40ecc5 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.cpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.cpp @@ -38,6 +38,12 @@ TestMaster::TraceItem::TraceItem(const TraceItem &) = default; TestMaster::TraceItem & TestMaster::TraceItem::operator=(const TraceItem &) = default; TestMaster::TraceItem::~TraceItem() = default; +TestMaster::ThreadState::~ThreadState() = default; +TestMaster::ThreadState::ThreadState(const std::string &n) + : name(n), passCnt(0), failCnt(0), preIgnoreFailCnt(0), + ignore(false), unwind(false), traceStack(), barrier(nullptr) +{} + TestMaster::ThreadState & TestMaster::threadState(const lock_guard &) { @@ -171,6 +177,8 @@ TestMaster::TestMaster() setThreadName("master"); } +TestMaster::~TestMaster() = default; + //----------------------------------------------------------------------------- void @@ -291,6 +299,13 @@ TestMaster::openDebugFiles(const std::string &lhsFile, } void +TestMaster::close_debug_files() +{ + lock_guard guard(_lock); + closeDebugFiles(guard); +} + +void TestMaster::pushState(const char *file, uint32_t line, const char *msg) { threadState().traceStack.emplace_back(skip_path(file), line, msg); diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.h b/vespalib/src/vespa/vespalib/testkit/test_master.h index cd73103c49f..232ef8009c9 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.h +++ b/vespalib/src/vespa/vespalib/testkit/test_master.h @@ -17,10 +17,6 @@ class Barrier; **/ class TestMaster { -private: - TestMaster(const TestMaster &); - TestMaster &operator=(const TestMaster &); - public: struct Progress { size_t passCnt; @@ -47,17 +43,17 @@ public: private: struct ThreadState { std::string name; - bool unwind; size_t passCnt; size_t failCnt; - bool ignore; size_t preIgnoreFailCnt; + bool ignore; + bool unwind; std::vector<TraceItem> traceStack; Barrier *barrier; - ThreadState(const std::string &n) - : name(n), unwind(false), passCnt(0), - failCnt(0), ignore(false), preIgnoreFailCnt(0), traceStack(), - barrier(0) {} + ~ThreadState(); + ThreadState(const std::string &n); + ThreadState(ThreadState &&) noexcept = default; + ThreadState & operator=(ThreadState &&) noexcept = default; }; static __thread ThreadState *_threadState; @@ -66,18 +62,20 @@ private: size_t failCnt; FILE *lhsFile; FILE *rhsFile; - SharedState() : passCnt(0), failCnt(0), - lhsFile(0), rhsFile(0) {} + SharedState() noexcept + : passCnt(0), failCnt(0), + lhsFile(nullptr), rhsFile(nullptr) + {} }; -private: - std::mutex _lock; - std::string _name; - SharedState _state; + std::mutex _lock; + std::string _name; + SharedState _state; std::vector<std::unique_ptr<ThreadState> > _threadStorage; using lock_guard = std::lock_guard<std::mutex>; private: + TestMaster(); ThreadState &threadState(const lock_guard &); ThreadState &threadState(); void checkFailed(const lock_guard &, @@ -90,10 +88,11 @@ private: void importThreads(const lock_guard &); bool reportConclusion(const lock_guard &); -private: - TestMaster(); public: + ~TestMaster(); + TestMaster(const TestMaster &) = delete; + TestMaster &operator=(const TestMaster &) = delete; void init(const char *name); std::string getName(); void setThreadName(const char *name); @@ -107,6 +106,7 @@ public: size_t getThreadFailCnt(); Progress getProgress(); void openDebugFiles(const std::string &lhsFile, const std::string &rhsFile); + void close_debug_files(); void pushState(const char *file, uint32_t line, const char *msg); void popState(); bool check(bool rc, const char *file, uint32_t line, diff --git a/vespalib/src/vespa/vespalib/testkit/testapp.cpp b/vespalib/src/vespa/vespalib/testkit/testapp.cpp deleted file mode 100644 index 6998a6dad17..00000000000 --- a/vespalib/src/vespa/vespalib/testkit/testapp.cpp +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "testapp.h" - -namespace vespalib { - -int -TestApp::Entry(int argc, char **argv) -{ - _argc = argc; - _argv = argv; - return Main(); -} - -TestApp::~TestApp() = default; - -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/testkit/testapp.h b/vespalib/src/vespa/vespalib/testkit/testapp.h deleted file mode 100644 index 24389390fa0..00000000000 --- a/vespalib/src/vespa/vespalib/testkit/testapp.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "test_kit.h" - -#define TEST_INIT(name) do { TEST_MASTER.init(name); } while(false) -#define TEST_DONE() do { return TEST_MASTER.fini() ? 0 : 1; } while(false) - -#define TEST_APPHOOK(app) \ - int main(int argc, char **argv) \ - { \ - app myapp; \ - return myapp.Entry(argc, argv); \ - } - -namespace vespalib { - -class TestApp -{ -public: - int _argc = 0; - char **_argv = nullptr; - - virtual int Main() = 0; - int Entry(int argc, char **argv); - virtual ~TestApp(); -}; - -} // namespace vespalib diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp index 8e5c6b6cf3c..74a45f9a9d0 100644 --- a/vespamalloc/src/tests/test1/new_test.cpp +++ b/vespamalloc/src/tests/test1/new_test.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/optimized.h> #include <vespa/log/log.h> #include <malloc.h> #include <dlfcn.h> #include <functional> +#include <cassert> LOG_SETUP("new_test"); diff --git a/vespamalloc/src/tests/test1/testatomic.cpp b/vespamalloc/src/tests/test1/testatomic.cpp index 4630d58b735..79347bd1ab8 100644 --- a/vespamalloc/src/tests/test1/testatomic.cpp +++ b/vespamalloc/src/tests/test1/testatomic.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespamalloc/malloc/allocchunk.h> #include <vespamalloc/malloc/mmappool.h> #include <unistd.h> diff --git a/vespamalloc/src/tests/thread/racemanythreads.cpp b/vespamalloc/src/tests/thread/racemanythreads.cpp index 3f153825844..d5ef2475e9a 100644 --- a/vespamalloc/src/tests/thread/racemanythreads.cpp +++ b/vespamalloc/src/tests/thread/racemanythreads.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <unistd.h> +#include <cassert> using namespace vespalib; @@ -9,9 +10,9 @@ void * hammer(void * arg) { usleep(4000000); long seconds = * static_cast<const long *>(arg); - long stopTime(time(NULL) + seconds); + long stopTime(time(nullptr) + seconds); pthread_t id = pthread_self(); - while (time(NULL) < stopTime) { + while (time(nullptr) < stopTime) { std::vector<pthread_t *> allocations; for (size_t i(0); i < 2000; i++) { pthread_t *t = new pthread_t[20]; @@ -21,11 +22,11 @@ void * hammer(void * arg) } } - for (size_t i(0); i < allocations.size(); i++) { + for (auto & allocation : allocations) { for (size_t j(0); j < 20; j++) { - assert(allocations[i][j] == id); + assert(allocation[j] == id); } - delete [] allocations[i]; + delete [] allocation; } } return arg; @@ -35,9 +36,9 @@ TEST_MAIN() { size_t threadCount(1024); long seconds(10); if (argc >= 2) { - threadCount = strtoul(argv[1], NULL, 0); + threadCount = strtoul(argv[1], nullptr, 0); if (argc >= 3) { - seconds = strtoul(argv[2], NULL, 0); + seconds = strtoul(argv[2], nullptr, 0); } } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index ba597d177c4..5f4dd62b966 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -71,7 +71,7 @@ public class Curator extends AbstractComponent implements AutoCloseable { private static final int MAX_RETRIES = 4; private static final RetryPolicy DEFAULT_RETRY_POLICY = new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES); // Default value taken from ZookeeperServerConfig - static final long defaultJuteMaxBuffer = Long.parseLong(System.getProperty("jute.maxbuffer", "52428800")); + static final long defaultJuteMaxBuffer = Long.parseLong(System.getProperty("jute.maxbuffer", "104857600")); private final CuratorFramework curatorFramework; private final ConnectionSpec connectionSpec; |