diff options
1464 files changed, 15847 insertions, 10069 deletions
diff --git a/.buildkite/Makefile b/.buildkite/Makefile index 0e7c5ec2d72..b5f730d0133 100644 --- a/.buildkite/Makefile +++ b/.buildkite/Makefile @@ -29,7 +29,7 @@ endif .DEFAULT_GOAL := pr -main: build-rpms cpp-test quick-start-guide publish-container publish-artifacts +main: build-rpms cpp-test quick-start-guide publish-container publish-artifacts upload-test-results pr: build-rpms cpp-test basic-search-test check: @@ -38,9 +38,6 @@ check: prepare: check @$(TOP)/execute.sh $@ -go: prepare - @$(TOP)/execute.sh $@ - bootstrap: prepare @$(TOP)/execute.sh $@ @@ -56,10 +53,10 @@ cpp: bootstrap-cmake cpp-test: cpp @$(TOP)/execute.sh $@ -install: cpp java +install: cpp java cpp-test @$(TOP)/execute.sh $@ -build-rpms: install go +build-rpms: install @$(TOP)/execute.sh $@ basic-search-test: build-rpms @@ -77,11 +74,13 @@ publish-container: build-container publish-artifacts: java build-rpms @$(TOP)/execute.sh $@ +upload-test-results: java cpp-test + @$(TOP)/execute.sh $@ + .PHONY: \ main \ pr \ prepare \ - go \ bootstrap \ bootstrap-cmake \ java \ @@ -94,4 +93,5 @@ publish-artifacts: java build-rpms quick-start-guide \ publish-container \ publish-artifacts \ + upload-test-results \ check diff --git a/.buildkite/bootstrap-cmake.sh b/.buildkite/bootstrap-cmake.sh index b7056140067..75d48142378 100755 --- a/.buildkite/bootstrap-cmake.sh +++ b/.buildkite/bootstrap-cmake.sh @@ -5,12 +5,15 @@ set -euo pipefail source /etc/profile.d/enable-gcc-toolset.sh VESPA_CMAKE_SANITIZERS_OPTION="" +VESPA_CMAKE_CCACHE_OPTION="" if [[ $VESPA_USE_SANITIZER != null ]]; then VESPA_CMAKE_SANITIZERS_OPTION="-DVESPA_USE_SANITIZER=$VESPA_USE_SANITIZER" + VESPA_CMAKE_CCACHE_OPTION="-DVESPA_USE_CCACHE=false" VALGRIND_UNIT_TESTS=false fi if [[ $BUILDKITE_PULL_REQUEST != "false" ]]; then VALGRIND_UNIT_TESTS=false fi -cmake3 -DVESPA_UNPRIVILEGED=no -DVALGRIND_UNIT_TESTS="$VALGRIND_UNIT_TESTS" "$VESPA_CMAKE_SANITIZERS_OPTION" "$SOURCE_DIR" +cmake3 -DVESPA_UNPRIVILEGED=no -DVALGRIND_UNIT_TESTS="$VALGRIND_UNIT_TESTS" \ + "$VESPA_CMAKE_SANITIZERS_OPTION" "$VESPA_CMAKE_CCACHE_OPTION" "$SOURCE_DIR" diff --git a/.buildkite/build-rpms.sh b/.buildkite/build-rpms.sh index 9f4ba7894a5..3dc05e6e6ce 100755 --- a/.buildkite/build-rpms.sh +++ b/.buildkite/build-rpms.sh @@ -9,7 +9,7 @@ make -f .copr/Makefile srpm outdir="$WORKDIR" rpmbuild --rebuild \ --define="_topdir $WORKDIR/vespa-rpmbuild" \ --define "_debugsource_template %{nil}" \ - --define "_binary_payload w10T.zstdio" \ + --define "_binary_payload w10T4.zstdio" \ --define "installdir $WORKDIR/vespa-install" "$WORKDIR"/vespa-"$VESPA_VERSION"-*.src.rpm mv "$WORKDIR"/vespa-rpmbuild/RPMS/*/*.rpm "$WORKDIR/artifacts/$ARCH/rpms" diff --git a/.buildkite/install.sh b/.buildkite/install.sh index d5bf30b5610..42faf5be076 100755 --- a/.buildkite/install.sh +++ b/.buildkite/install.sh @@ -3,3 +3,7 @@ set -euo pipefail make -j "$NUM_CPU_LIMIT" install DESTDIR="$WORKDIR/vespa-install" + +# The cmake install does not handle install into /usr/share/man. Do it explicitly here. +mkdir -p "$WORKDIR/vespa-install/usr/share/man/man1" +"$WORKDIR/vespa-install/opt/vespa/bin/vespa" man "$WORKDIR/vespa-install/usr/share/man/man1" diff --git a/.buildkite/prepare.sh b/.buildkite/prepare.sh index 6989c2fc568..cb48f3de759 100755 --- a/.buildkite/prepare.sh +++ b/.buildkite/prepare.sh @@ -4,5 +4,13 @@ set -euo pipefail "$SOURCE_DIR/screwdriver/replace-vespa-version-in-poms.sh" "$VESPA_VERSION" "$SOURCE_DIR" +# We disable javadoc for all modules not marked as public API +for MODULE in $(comm -2 -3 \ + <(find . -name "*.java" | awk -F/ '{print $2}' | sort -u) + <(find . -name "package-info.java" -exec grep -HnE "@(com.yahoo.api.annotations.)?PublicApi.*" {} \; | awk -F/ '{print $2}' | sort -u)); do + mkdir -p "$MODULE/src/main/javadoc" + echo "No javadoc available for module" > "$MODULE/src/main/javadoc/README" +done + mkdir -p "$WORKDIR/artifacts/$ARCH/rpms" mkdir -p "$WORKDIR/artifacts/$ARCH/maven-repo" diff --git a/.buildkite/quick-start-guide.sh b/.buildkite/quick-start-guide.sh index f3e6e554e5a..05cb0842947 100755 --- a/.buildkite/quick-start-guide.sh +++ b/.buildkite/quick-start-guide.sh @@ -2,4 +2,9 @@ set -euo pipefail +if [[ $VESPA_USE_SANITIZER != null ]]; then + echo "Skipping quick start guide test for sanitizer builds." + exit 0 +fi + "$SOURCE_DIR/screwdriver/test-quick-start-guide.sh" diff --git a/.buildkite/upload-test-results.sh b/.buildkite/upload-test-results.sh new file mode 100755 index 00000000000..e95a9448adf --- /dev/null +++ b/.buildkite/upload-test-results.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $BUILDKITE != true ]]; then + echo "Skipping artifact publishing when not executed by Buildkite." + exit 0 +fi + +if [[ $(arch) == x86_64 ]]; then + JAVA_TEST_TOKEN=$UNIT_TEST_JAVA_AMD64_TOKEN + CPP_TEST_TOKEN=$UNIT_TEST_CPP_AMD64_TOKEN +else + JAVA_TEST_TOKEN=$UNIT_TEST_JAVA_ARM64_TOKEN + CPP_TEST_TOKEN=$UNIT_TEST_CPP_ARM64_TOKEN +fi + +if [[ -z $JAVA_TEST_TOKEN ]]; then + echo "Missing JAVA_TEST_TOKEN. Exiting." + exit 1 +fi +if [[ -z $CPP_TEST_TOKEN ]]; then + echo "Missing CPP_TEST_TOKEN. Exiting." + exit 1 +fi + +upload_result() { + curl \ + -X POST \ + -H "Authorization: Token token=\"$BUILDKITE_ANALYTICS_TOKEN\"" \ + -F "data=@$1" \ + -F "format=junit" \ + -F "run_env[CI]=buildkite" \ + -F "run_env[key]=$BUILDKITE_BUILD_ID" \ + -F "run_env[url]=$BUILDKITE_BUILD_URL" \ + -F "run_env[branch]=$BUILDKITE_BRANCH" \ + -F "run_env[commit_sha]=$BUILDKITE_COMMIT" \ + -F "run_env[number]=$BUILDKITE_BUILD_NUMBER" \ + -F "run_env[job_id]=$BUILDKITE_JOB_ID" \ + -F "run_env[message]=$BUILDKITE_MESSAGE" \ + "https://analytics-api.buildkite.com/v1/uploads" +} + +export -f upload_result + +# Upload all surefire TEST-*.xml reports +cd "$WORKDIR" +export BUILDKITE_ANALYTICS_TOKEN=$JAVA_TEST_TOKEN +# shellcheck disable=2038 +find . -name "TEST-*.xml" -type f | xargs -n 1 -P 50 -I '{}' bash -c "upload_result {}" + +# Upload the cpp test report +export BUILDKITE_ANALYTICS_TOKEN=$CPP_TEST_TOKEN +upload_result "$LOG_DIR/vespa-cpptest-results.xml" + diff --git a/.copr/Makefile b/.copr/Makefile index c70f35f4450..8371f2fbd7b 100644 --- a/.copr/Makefile +++ b/.copr/Makefile @@ -15,7 +15,7 @@ srpm: VESPA_VERSION = $$(git tag --points-at HEAD | grep -oP "\d+\.\d+\.\d+" | s srpm: deps $(TOP)/../dist.sh $(VESPA_VERSION) spectool -g -C $(SOURCEDIR) $(SPECDIR)/vespa-$(VESPA_VERSION).spec - rpmbuild -bs --define "_topdir $(RPMTOPDIR)" $(SPECDIR)/vespa-$(VESPA_VERSION).spec + rpmbuild -bs --define "_topdir $(RPMTOPDIR)" --define "_source_payload w10T4.zstdio" $(SPECDIR)/vespa-$(VESPA_VERSION).spec cp -a $(RPMTOPDIR)/SRPMS/* $(outdir) rpms: srpm diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java index d817071e072..6d28db342e6 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java @@ -16,7 +16,6 @@ public class ServiceType { public static final ServiceType HOST_ADMIN = new ServiceType("hostadmin"); public static final ServiceType CONFIG_SERVER = new ServiceType("configserver"); public static final ServiceType CONTROLLER = new ServiceType("controller"); - public static final ServiceType TRANSACTION_LOG_SERVER = new ServiceType("transactionlogserver"); public static final ServiceType CLUSTER_CONTROLLER = new ServiceType("container-clustercontroller"); public static final ServiceType DISTRIBUTOR = new ServiceType("distributor"); public static final ServiceType SEARCH = new ServiceType("searchnode"); diff --git a/build_settings.cmake b/build_settings.cmake index e046d7f71f3..596192567db 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -21,6 +21,9 @@ set(EXCLUDE_TESTS_FROM_ALL FALSE CACHE BOOL "If TRUE, do not build tests as part # Whether to run unit tests via valgrind set(VALGRIND_UNIT_TESTS FALSE CACHE BOOL "If TRUE, run unit tests via valgrind") +# Whether to use ccache when building +set(VESPA_USE_CCACHE TRUE CACHE BOOL "If TRUE, use ccache (if available) when building") + # Whether to run tests marked as benchmark as part of the test runs set(RUN_BENCHMARKS FALSE CACHE BOOL "If TRUE, benchmarks are run together with the other tests") @@ -185,18 +188,21 @@ if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") endif() endif() -# Find ccache and use it if it is found -find_program(CCACHE_EXECUTABLE ccache) -if(CCACHE_EXECUTABLE) - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_EXECUTABLE}) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_EXECUTABLE}) +# Find ccache and use it if it is found unless disabled +if (VESPA_USE_CCACHE) + find_program(CCACHE_EXECUTABLE ccache) + if(CCACHE_EXECUTABLE) + message("-- Using ccache ${CCACHE_EXECUTABLE}") + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_EXECUTABLE}) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_EXECUTABLE}) + endif() endif() # Check for valgrind and set flags find_program(VALGRIND_EXECUTABLE valgrind) if(VALGRIND_EXECUTABLE) set(VALGRIND_SUPPRESSIONS_FILE "${PROJECT_SOURCE_DIR}/valgrind-suppressions.txt") - set(VALGRIND_OPTIONS "--leak-check=yes --error-exitcode=1 --run-libc-freeres=no --track-origins=yes --suppressions=${VALGRIND_SUPPRESSIONS_FILE}") + set(VALGRIND_OPTIONS "--leak-check=yes --fair-sched=yes --error-exitcode=1 --run-libc-freeres=no --track-origins=yes --suppressions=${VALGRIND_SUPPRESSIONS_FILE}") set(VALGRIND_COMMAND "${VALGRIND_EXECUTABLE} ${VALGRIND_OPTIONS}") endif() # Automatically set sanitizer suppressions file and arguments for unit tests diff --git a/client/go/Makefile b/client/go/Makefile index b2ffdc0feb6..fee92547e73 100644 --- a/client/go/Makefile +++ b/client/go/Makefile @@ -17,6 +17,7 @@ GO_FLAGS := -ldflags "-X github.com/vespa-engine/vespa/client/go/internal/build. PROJECT_ROOT := $(shell realpath $(CURDIR)/../..) GO_TMPDIR := $(PROJECT_ROOT)/build/go DIST_TARGETS := dist-mac dist-mac-arm64 dist-linux dist-linux-arm64 dist-win32 dist-win64 +GOTOOLCHAIN := $(shell go env GOTOOLCHAIN) all: test checkfmt install @@ -111,7 +112,9 @@ install-all: all manpages # Development targets # -ci: +setenv: +# Set GOTOOLCHAIN if its default value has been changed + @test "$(GOTOOLCHAIN)" = auto || go env -w GOTOOLCHAIN="auto" ifdef CI # Ensure that CI systems use a proxy for downloading dependencies go env -w GOPROXY="https://proxy.golang.org,direct" @@ -121,7 +124,7 @@ endif install-brew: brew install vespa-cli -install: ci +install: setenv env GOBIN=$(BIN) go install $(GO_FLAGS) ./... manpages: install @@ -133,7 +136,7 @@ clean: rm -f $(BIN)/vespa $(BIN)/vespa-wrapper $(SHARE)/man/man1/vespa.1 $(SHARE)/man/man1/vespa-*.1 rmdir -p $(BIN) $(SHARE)/man/man1 > /dev/null 2>&1 || true -test: ci +test: setenv # Why custom GOTMPDIR? go builds executables for unit tests and by default these # end up in TMPDIR/GOTMPDIR. In some environments /tmp is mounted noexec so # running test executables will fail diff --git a/client/go/go.mod b/client/go/go.mod index 4fe9108efaf..4b6b0ceef49 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -1,28 +1,27 @@ module github.com/vespa-engine/vespa/client/go -go 1.20 +go 1.22.4 require ( github.com/alessio/shellescape v1.4.2 - github.com/briandowns/spinner v1.23.0 + github.com/briandowns/spinner v1.23.1 github.com/fatih/color v1.17.0 - // This is the most recent version compatible with Go 1.20. Upgrade when we upgrade our Go version - github.com/go-json-experiment/json v0.0.0-20230324203220-04923b7a9528 - github.com/klauspost/compress v1.17.8 + github.com/go-json-experiment/json v0.0.0-20240524174822-2d9f40f7385b + github.com/klauspost/compress v1.17.9 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - github.com/zalando/go-keyring v0.2.4 - golang.org/x/net v0.25.0 - golang.org/x/sys v0.20.0 + github.com/zalando/go-keyring v0.2.5 + golang.org/x/net v0.26.0 + golang.org/x/sys v0.21.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/danieljoos/wincred v1.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect @@ -30,8 +29,7 @@ require ( github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect - golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/client/go/go.sum b/client/go/go.sum index 2af61a8ce98..d8b6a4cacf9 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -1,35 +1,23 @@ github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= -github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Ptu650= +github.com/briandowns/spinner v1.23.1/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/go-json-experiment/json v0.0.0-20230324203220-04923b7a9528 h1:hmpF6G+rHcypt8J6jhBH/rDUx+04Th/L61Y8uCKFb7Q= -github.com/go-json-experiment/json v0.0.0-20230324203220-04923b7a9528/go.mod h1:AHV+bpNGVGD0DCHMBhhTYtT7yeBYD9Yk92XAjB7vOgo= +github.com/go-json-experiment/json v0.0.0-20240524174822-2d9f40f7385b h1:IM96IiRXFcd7l+mU8Sys9pcggoBLbH/dEgzOESrS8F8= +github.com/go-json-experiment/json v0.0.0-20240524174822-2d9f40f7385b/go.mod h1:uDEMZSTQMj7V6Lxdrx4ZwchmHEGdICbjuY+GQd7j9LM= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= -github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= -github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -40,81 +28,33 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= -github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= -github.com/zalando/go-keyring v0.2.4 h1:wi2xxTqdiwMKbM6TWwi+uJCG/Tum2UV0jqaQhCa9/68= -github.com/zalando/go-keyring v0.2.4/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +github.com/zalando/go-keyring v0.2.5 h1:Bc2HHpjALryKD62ppdEzaFG6VxL6Bc+5v0LYpN8Lba8= +github.com/zalando/go-keyring v0.2.5/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/client/go/internal/admin/jvm/mem_options.go b/client/go/internal/admin/jvm/mem_options.go index d3b0d44c677..c78ee80dc80 100644 --- a/client/go/internal/admin/jvm/mem_options.go +++ b/client/go/internal/admin/jvm/mem_options.go @@ -55,8 +55,8 @@ func (opts *Options) MaybeAddHugepages(heapSize AmountOfMemory) { } func adjustAvailableMemory(measured AmountOfMemory) AmountOfMemory { - reserved := 1024 // MB - need_min := 64 // MB + reserved := 700 // MB -- keep in sync with com.yahoo.vespa.model.Host.memoryOverheadGb + need_min := 64 // MB available := measured.ToMB() if available > need_min+2*reserved { return MegaBytesOfMemory(available - reserved) diff --git a/client/go/internal/admin/jvm/mem_options_test.go b/client/go/internal/admin/jvm/mem_options_test.go index 3501e44c723..60cffc824e9 100644 --- a/client/go/internal/admin/jvm/mem_options_test.go +++ b/client/go/internal/admin/jvm/mem_options_test.go @@ -9,11 +9,11 @@ import ( func TestAdjustment(t *testing.T) { lastAdj := 64 - for i := 0; i < 4096; i++ { + for i := range 4096 { adj := adjustAvailableMemory(MegaBytesOfMemory(i)).ToMB() assert.True(t, int(adj) >= lastAdj) lastAdj = int(adj) } - adj := adjustAvailableMemory(MegaBytesOfMemory(31024)).ToMB() + adj := adjustAvailableMemory(MegaBytesOfMemory(30700)).ToMB() assert.Equal(t, 30000, int(adj)) } diff --git a/client/go/internal/admin/jvm/standalone_container.go b/client/go/internal/admin/jvm/standalone_container.go index 20031bc7725..fb615dcf4c2 100644 --- a/client/go/internal/admin/jvm/standalone_container.go +++ b/client/go/internal/admin/jvm/standalone_container.go @@ -37,7 +37,6 @@ func (a *StandaloneContainer) configureOptions() { opts := a.jvmOpts opts.ConfigureCpuCount(0) opts.AddCommonXX() - opts.AddOption("-XX:-OmitStackTraceInFastThrow") opts.AddCommonOpens() opts.AddCommonJdkProperties() a.addJdiscProperties() diff --git a/client/go/internal/admin/jvm/xx_options.go b/client/go/internal/admin/jvm/xx_options.go index 13b69e43dda..abc92f19bf2 100644 --- a/client/go/internal/admin/jvm/xx_options.go +++ b/client/go/internal/admin/jvm/xx_options.go @@ -19,5 +19,5 @@ func (opts *Options) AddCommonXX() { // not common after all: opts.AddOption("-XX:MaxJavaStackTraceDepth=1000000") // Aid debugging for slight cost in performance - opts.AddOption("-XX:+OmitStackTraceInFastThrow") + opts.AddOption("-XX:-OmitStackTraceInFastThrow") } diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go b/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go index ec2c53487be..180f7c84859 100644 --- a/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go +++ b/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go @@ -85,7 +85,7 @@ func (t *unixTail) openTail() { if err != nil { return } - for i := 0; i < n; i++ { + for i := range n { if t.lineBuf[i] == '\n' { sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) if err == nil { diff --git a/client/go/internal/cli/cmd/cert.go b/client/go/internal/cli/cmd/cert.go index 14e4861cec3..9e0b8f6805c 100644 --- a/client/go/internal/cli/cmd/cert.go +++ b/client/go/internal/cli/cmd/cert.go @@ -160,10 +160,10 @@ func doCertAdd(cli *CLI, overwriteCertificate bool, args []string) error { if pkg.HasCertificate() && !overwriteCertificate { return errHint(fmt.Errorf("application package '%s' already contains a certificate", pkg.Path), "Use -f flag to force overwriting") } - return maybeCopyCertificate(true, false, cli, target, pkg) + return requireCertificate(true, false, cli, target, pkg) } -func maybeCopyCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, pkg vespa.ApplicationPackage) error { +func requireCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, pkg vespa.ApplicationPackage) error { if pkg.IsZip() { if ignoreZip { cli.printWarning("Cannot verify existence of "+color.CyanString("security/clients.pem")+" since '"+pkg.Path+"' is compressed", @@ -175,10 +175,31 @@ func maybeCopyCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, return errHint(fmt.Errorf("cannot add certificate to compressed application package: '%s'", pkg.Path), hint) } } + tlsOptions, err := cli.config.readTLSOptions(target.Deployment().Application, target.Type()) + if err != nil { + return err + } if force { - return copyCertificate(cli, target, pkg) + return copyCertificate(tlsOptions, cli, pkg) } if pkg.HasCertificate() { + if cli.isCI() { + return nil // A matching certificate is not required in CI environments + } + if len(tlsOptions.CertificatePEM) == 0 { + return errHint(fmt.Errorf("no certificate exists for %s", target.Deployment().Application.String()), "Try (re)creating the certificate with 'vespa auth cert'") + } + matches, err := pkg.HasMatchingCertificate(tlsOptions.CertificatePEM) + if err != nil { + return err + } + if !matches { + return errHint(fmt.Errorf("certificate in %s does not match the stored key pair for %s", + filepath.Join("security", "clients.pem"), + target.Deployment().Application.String()), + "If this application was deployed using a different application ID in the past, the matching key pair may be stored under a different ID in "+cli.config.homeDir, + "Specify the matching application with --application, or add the current certificate to the package using --add-cert") + } return nil } if cli.isTerminal() { @@ -188,7 +209,7 @@ func maybeCopyCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, return err } if ok { - return copyCertificate(cli, target, pkg) + return copyCertificate(tlsOptions, cli, pkg) } } return errHint(fmt.Errorf("deployment to Vespa Cloud requires certificate in application package"), @@ -196,15 +217,7 @@ func maybeCopyCertificate(force, ignoreZip bool, cli *CLI, target vespa.Target, "Pass --add-cert to use the certificate of the current application") } -func copyCertificate(cli *CLI, target vespa.Target, pkg vespa.ApplicationPackage) error { - tlsOptions, err := cli.config.readTLSOptions(target.Deployment().Application, target.Type()) - if err != nil { - return err - } - hint := "Try generating the certificate with 'vespa auth cert'" - if tlsOptions.CertificateFile == "" { - return errHint(fmt.Errorf("no certificate exists for "+target.Deployment().Application.String()), hint) - } +func copyCertificate(tlsOptions vespa.TLSOptions, cli *CLI, pkg vespa.ApplicationPackage) error { data, err := os.ReadFile(tlsOptions.CertificateFile) if err != nil { return errHint(fmt.Errorf("could not read certificate file: %w", err)) diff --git a/client/go/internal/cli/cmd/clone.go b/client/go/internal/cli/cmd/clone.go index 8fd12eb9e6e..3df490079a6 100644 --- a/client/go/internal/cli/cmd/clone.go +++ b/client/go/internal/cli/cmd/clone.go @@ -32,8 +32,7 @@ func newCloneCmd(cli *CLI) *cobra.Command { cmd := &cobra.Command{ Use: "clone sample-application-path target-directory", Short: "Create files and directory structure from a Vespa sample application", - Long: `Create files and directory structure from a Vespa sample application -from a sample application. + Long: `Create files and directory structure from a Vespa sample application. Sample applications are downloaded from https://github.com/vespa-engine/sample-apps. diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go index fc6d69aab61..dfe48118340 100644 --- a/client/go/internal/cli/cmd/config.go +++ b/client/go/internal/cli/cmd/config.go @@ -35,7 +35,7 @@ func newConfigCmd() *cobra.Command { This command allows setting persistent values for the global flags found in Vespa CLI. On future invocations the flag can then be omitted as it is read - from the config file instead. +from the config file instead. Configuration is written to $HOME/.vespa by default. This path can be overridden by setting the VESPA_CLI_HOME environment variable. @@ -114,8 +114,7 @@ zone Specifies a custom zone to use when connecting to a Vespa Cloud application. This is only relevant for cloud and hosted targets and defaults to a dev zone. See https://cloud.vespa.ai/en/reference/zones for available zones. Examples: -dev.aws-us-east-1c, dev.gcp-us-central1-f, perf.aws-us-east-1c -`, +dev.aws-us-east-1c, dev.gcp-us-central1-f, perf.aws-us-east-1c`, DisableAutoGenTag: true, SilenceUsage: false, Args: cobra.MinimumNArgs(1), @@ -143,8 +142,7 @@ $ vespa config set application my-tenant.my-application.my-instance $ vespa config set instance other-instance # Set an option in local configuration, for the current application only -$ vespa config set --local wait 600 -`, +$ vespa config set --local zone perf.us-north-1`, DisableAutoGenTag: true, SilenceUsage: true, Args: cobra.ExactArgs(2), @@ -180,8 +178,7 @@ Unsetting a configuration option will reset it to its default value, which may b $ vespa config unset target # Stop overriding application option in local config -$ vespa config unset --local application -`, +$ vespa config unset --local application`, DisableAutoGenTag: true, SilenceUsage: true, Args: cobra.ExactArgs(1), @@ -216,8 +213,7 @@ application, i.e. it takes into account any local configuration located in `, Example: `$ vespa config get $ vespa config get target -$ vespa config get --local -`, +$ vespa config get --local`, Args: cobra.MaximumNArgs(1), DisableAutoGenTag: true, SilenceUsage: true, @@ -429,51 +425,76 @@ func (c *Config) privateKeyPath(app vespa.ApplicationID, targetType string) (cre return c.credentialsFile(app, targetType, false) } -func (c *Config) readTLSOptions(app vespa.ApplicationID, targetType string) (vespa.TLSOptions, error) { - _, trustAll := c.environment["VESPA_CLI_DATA_PLANE_TRUST_ALL"] - cert, certOk := c.environment["VESPA_CLI_DATA_PLANE_CERT"] - key, keyOk := c.environment["VESPA_CLI_DATA_PLANE_KEY"] - caCertText, caCertOk := c.environment["VESPA_CLI_DATA_PLANE_CA_CERT"] - options := vespa.TLSOptions{TrustAll: trustAll} - // CA certificate +func (c *Config) caCertificatePEM() ([]byte, string, error) { + envVar := "VESPA_CLI_DATA_PLANE_CA_CERT" + caCertText, caCertOk := c.environment[envVar] if caCertOk { - options.CACertificate = []byte(caCertText) - } else if caCertFile := c.caCertificatePath(); caCertFile != "" { + return []byte(caCertText), envVar, nil + } + if caCertFile := c.caCertificatePath(); caCertFile != "" { b, err := os.ReadFile(caCertFile) if err != nil { - return options, err + return nil, "", err } - options.CACertificate = b - options.CACertificateFile = caCertFile + return b, caCertFile, nil } - // Certificate and private key - if certOk && keyOk { - kp, err := tls.X509KeyPair([]byte(cert), []byte(key)) + return nil, "", nil +} + +func (c *Config) credentialsPEM(envVar string, credentialsFile credentialsFile) ([]byte, string, error) { + if pem, ok := c.environment[envVar]; ok { + return []byte(pem), envVar, nil + } + pem, err := os.ReadFile(credentialsFile.path) + if err != nil { + if os.IsNotExist(err) && credentialsFile.optional { + return nil, "", nil + } + return nil, "", err + } + return []byte(pem), credentialsFile.path, nil +} + +func (c *Config) readTLSOptions(app vespa.ApplicationID, targetType string) (vespa.TLSOptions, error) { + var options vespa.TLSOptions + // Certificate + if certPath, err := c.certificatePath(app, targetType); err == nil { + certPEM, certFile, err := c.credentialsPEM("VESPA_CLI_DATA_PLANE_CERT", certPath) if err != nil { return vespa.TLSOptions{}, err } - options.KeyPair = []tls.Certificate{kp} + options.CertificatePEM = certPEM + options.CertificateFile = certFile } else { - keyFile, err := c.privateKeyPath(app, targetType) + return vespa.TLSOptions{}, err + } + // Private key + if keyPath, err := c.privateKeyPath(app, targetType); err == nil { + keyPEM, keyFile, err := c.credentialsPEM("VESPA_CLI_DATA_PLANE_KEY", keyPath) if err != nil { return vespa.TLSOptions{}, err } - certFile, err := c.certificatePath(app, targetType) + options.PrivateKeyPEM = keyPEM + options.PrivateKeyFile = keyFile + } else { + return vespa.TLSOptions{}, err + + } + // CA certificate + _, options.TrustAll = c.environment["VESPA_CLI_DATA_PLANE_TRUST_ALL"] + caCertificate, caCertificateFile, err := c.caCertificatePEM() + if err != nil { + return vespa.TLSOptions{}, err + } + options.CACertificatePEM = caCertificate + options.CACertificateFile = caCertificateFile + // Key pair + if len(options.CertificatePEM) > 0 && len(options.PrivateKeyPEM) > 0 { + kp, err := tls.X509KeyPair(options.CertificatePEM, options.PrivateKeyPEM) if err != nil { return vespa.TLSOptions{}, err } - kp, err := tls.LoadX509KeyPair(certFile.path, keyFile.path) - allowMissing := os.IsNotExist(err) && keyFile.optional && certFile.optional - if err == nil { - options.KeyPair = []tls.Certificate{kp} - options.PrivateKeyFile = keyFile.path - options.CertificateFile = certFile.path - } else if err != nil && !allowMissing { - return vespa.TLSOptions{}, err - } - } - // If we found a key pair, parse it and check expiry - if options.KeyPair != nil { + options.KeyPair = []tls.Certificate{kp} cert, err := x509.ParseCertificate(options.KeyPair[0].Certificate[0]) if err != nil { return vespa.TLSOptions{}, err @@ -513,7 +534,7 @@ func (c *Config) authConfigPath() string { return filepath.Join(c.homeDir, "auth.json") } -func (c *Config) readAPIKey(cli *CLI, system vespa.System, tenantName string) ([]byte, error) { +func (c *Config) readAPIKey(cli *CLI, tenantName string) ([]byte, error) { if override, ok := c.apiKeyFromEnv(); ok { return override, nil } diff --git a/client/go/internal/cli/cmd/config_test.go b/client/go/internal/cli/cmd/config_test.go index b13f8365f5f..6c9321b3219 100644 --- a/client/go/internal/cli/cmd/config_test.go +++ b/client/go/internal/cli/cmd/config_test.go @@ -156,19 +156,19 @@ func assertConfigCommandErr(t *testing.T, configHome, expected string, args ...s func TestReadAPIKey(t *testing.T) { cli, _, _ := newTestCLI(t) - key, err := cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err := cli.config.readAPIKey(cli, "t1") assert.Nil(t, key) require.NotNil(t, err) // From default path when it exists require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "t1.api-key.pem"), []byte("foo"), 0600)) - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Equal(t, []byte("foo"), key) // Cloud CI never reads key from disk as it's not expected to have any cli, _, _ = newTestCLI(t, "VESPA_CLI_CLOUD_CI=true") - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Nil(t, key) @@ -176,20 +176,20 @@ func TestReadAPIKey(t *testing.T) { keyFile := filepath.Join(t.TempDir(), "key") require.Nil(t, os.WriteFile(keyFile, []byte("bar"), 0600)) cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY_FILE="+keyFile) - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Equal(t, []byte("bar"), key) // From key specified in environment cli, _, _ = newTestCLI(t, "VESPA_CLI_API_KEY=baz") - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Equal(t, []byte("baz"), key) // Prefer Auth0 if we have auth config cli, _, _ = newTestCLI(t) require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "auth.json"), []byte("foo"), 0600)) - key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") + key, err = cli.config.readAPIKey(cli, "t1") require.Nil(t, err) assert.Nil(t, key) } @@ -209,9 +209,14 @@ func TestConfigReadTLSOptions(t *testing.T) { assertTLSOptions(t, homeDir, app, vespa.TargetLocal, vespa.TLSOptions{ - TrustAll: true, - CACertificate: []byte("cacert"), - KeyPair: []tls.Certificate{keyPair}, + TrustAll: true, + KeyPair: []tls.Certificate{keyPair}, + CACertificatePEM: []byte("cacert"), + CertificatePEM: pemCert, + PrivateKeyPEM: pemKey, + CACertificateFile: "VESPA_CLI_DATA_PLANE_CA_CERT", + CertificateFile: "VESPA_CLI_DATA_PLANE_CERT", + PrivateKeyFile: "VESPA_CLI_DATA_PLANE_KEY", }, "VESPA_CLI_DATA_PLANE_TRUST_ALL=true", "VESPA_CLI_DATA_PLANE_CA_CERT=cacert", @@ -230,7 +235,9 @@ func TestConfigReadTLSOptions(t *testing.T) { vespa.TargetLocal, vespa.TLSOptions{ KeyPair: []tls.Certificate{keyPair}, - CACertificate: []byte("cacert"), + CACertificatePEM: []byte("cacert"), + CertificatePEM: pemCert, + PrivateKeyPEM: pemKey, CACertificateFile: caCertFile, CertificateFile: certFile, PrivateKeyFile: keyFile, @@ -249,6 +256,8 @@ func TestConfigReadTLSOptions(t *testing.T) { vespa.TargetLocal, vespa.TLSOptions{ KeyPair: []tls.Certificate{keyPair}, + CertificatePEM: pemCert, + PrivateKeyPEM: pemKey, CertificateFile: defaultCertFile, PrivateKeyFile: defaultKeyFile, }, diff --git a/client/go/internal/cli/cmd/curl.go b/client/go/internal/cli/cmd/curl.go index c7297574a32..63f6ef8c592 100644 --- a/client/go/internal/cli/cmd/curl.go +++ b/client/go/internal/cli/cmd/curl.go @@ -37,7 +37,7 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain if err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) service, err := waiter.Service(target, cli.config.cluster()) if err != nil { return err diff --git a/client/go/internal/cli/cmd/deploy.go b/client/go/internal/cli/cmd/deploy.go index 9ae6676bc17..fac779c8241 100644 --- a/client/go/internal/cli/cmd/deploy.go +++ b/client/go/internal/cli/cmd/deploy.go @@ -5,6 +5,7 @@ package cmd import ( + "errors" "fmt" "io" "log" @@ -29,19 +30,22 @@ func newDeployCmd(cli *CLI) *cobra.Command { Short: "Deploy (prepare and activate) an application package", Long: `Deploy (prepare and activate) an application package. -When this returns successfully the application package has been validated -and activated on config servers. The process of applying it on individual nodes -has started but may not have completed. +An application package defines a deployable Vespa application. See +https://docs.vespa.ai/en/reference/application-packages-reference.html for +details about the files contained in this package. -If application directory is not specified, it defaults to working directory. +To get started, 'vespa clone' can be used to download a sample application. + +This command deploys an application package. When deploy returns successfully +the application package has been validated and activated on config servers. The +process of applying it on individual nodes has started but may not have +completed. -When deploying to Vespa Cloud the system can be overridden by setting the -environment variable VESPA_CLI_CLOUD_SYSTEM. This is intended for internal use -only. +If application directory is not specified, it defaults to working directory. -In Vespa Cloud you may override the Vespa runtime version for your deployment. -This option should only be used if you have a reason for using a specific -version. By default Vespa Cloud chooses a suitable version for you. +In Vespa Cloud you may override the Vespa runtime version (--version) for your +deployment. This option should only be used if you have a reason for using a +specific version. By default Vespa Cloud chooses a suitable version for you. `, Example: `$ vespa deploy . $ vespa deploy -t cloud @@ -59,7 +63,6 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, if err != nil { return err } - timeout := time.Duration(waitSecs) * time.Second opts := vespa.DeploymentOptions{ApplicationPackage: pkg, Target: target} if versionArg != "" { version, err := version.Parse(versionArg) @@ -69,19 +72,26 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, opts.Version = version } if target.Type() == vespa.TargetCloud { - if err := maybeCopyCertificate(copyCert, true, cli, target, pkg); err != nil { + if err := requireCertificate(copyCert, true, cli, target, pkg); err != nil { return err } } - waiter := cli.waiter(timeout) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) if _, err := waiter.DeployService(target); err != nil { return err } var result vespa.PrepareResult - if err := cli.spinner(cli.Stderr, "Uploading application package...", func() error { + err = cli.spinner(cli.Stderr, "Uploading application package...", func() error { result, err = vespa.Deploy(opts) return err - }); err != nil { + }) + if err != nil { + if target.IsCloud() && errors.Is(err, vespa.ErrUnauthorized) { + return errHint(err, + "You do not have access to the tenant "+color.CyanString(target.Deployment().Application.Tenant), + "You may need to create the tenant at "+color.CyanString(target.Deployment().System.ConsoleURL+"/tenant"), + "If the tenant already exists you may need to run 'vespa auth login' to gain access to it") + } return err } log.Println() @@ -95,7 +105,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, log.Printf("\nUse %s for deployment status, or follow this deployment at", color.CyanString("vespa status deployment")) log.Print(color.CyanString(opts.Target.Deployment().System.ConsoleRunURL(opts.Target.Deployment(), result.ID))) } - return waitForDeploymentReady(cli, target, result.ID, timeout) + return waitForVespaReady(target, result.ID, waiter) }, } cmd.Flags().StringVarP(&logLevelArg, "log-level", "l", "error", `Log level for Vespa logs. Must be "error", "warning", "info" or "debug"`) @@ -157,8 +167,7 @@ func newActivateCmd(cli *CLI) *cobra.Command { if err != nil { return err } - timeout := time.Duration(waitSecs) * time.Second - waiter := cli.waiter(timeout) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) if _, err := waiter.DeployService(target); err != nil { return err } @@ -168,23 +177,30 @@ func newActivateCmd(cli *CLI) *cobra.Command { return err } cli.printSuccess("Activated application with session ", sessionID) - return waitForDeploymentReady(cli, target, sessionID, timeout) + return waitForVespaReady(target, sessionID, waiter) }, } cli.bindWaitFlag(cmd, 0, &waitSecs) return cmd } -func waitForDeploymentReady(cli *CLI, target vespa.Target, sessionOrRunID int64, timeout time.Duration) error { - if timeout == 0 { - return nil - } - waiter := cli.waiter(timeout) - if _, err := waiter.Deployment(target, sessionOrRunID); err != nil { - return err +func waitForVespaReady(target vespa.Target, sessionOrRunID int64, waiter *Waiter) error { + fastWait := waiter.FastWaitOn(target) + hasTimeout := waiter.Timeout > 0 + if fastWait || hasTimeout { + // Wait for deployment convergence + if _, err := waiter.Deployment(target, sessionOrRunID); err != nil { + return err + } + // Wait for healthy services where we expect them to be reachable (cloud and local). When using a custom target, + // we do not wait for services as there is no guarantee that they are reachable from the machine executing + // deploy. + if hasTimeout && (target.IsCloud() || target.Type() == vespa.TargetLocal) { + _, err := waiter.Services(target) + return err + } } - _, err := waiter.Services(target) - return err + return nil } func printPrepareLog(stderr io.Writer, result vespa.PrepareResult) { diff --git a/client/go/internal/cli/cmd/deploy_test.go b/client/go/internal/cli/cmd/deploy_test.go index 4e32b9bbd60..3f71a59e682 100644 --- a/client/go/internal/cli/cmd/deploy_test.go +++ b/client/go/internal/cli/cmd/deploy_test.go @@ -5,8 +5,10 @@ package cmd import ( + "archive/zip" "bytes" "io" + "os" "path/filepath" "strconv" "strings" @@ -22,7 +24,7 @@ func TestDeployCloud(t *testing.T) { pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, false, false) - cli, stdout, stderr := newTestCLI(t, "CI=true", "NO_COLOR=true") + cli, stdout, stderr := newTestCLI(t, "NO_COLOR=true") httpClient := &mock.HTTPClient{} httpClient.NextResponseString(200, `ok`) cli.httpClient = httpClient @@ -35,11 +37,12 @@ func TestDeployCloud(t *testing.T) { stderr.Reset() require.NotNil(t, cli.Run("deploy", pkgDir)) + apiKeyWarning := "Warning: Authenticating with API key, intended for use in CI environments.\nHint: Authenticate with 'vespa auth login' instead\n" certError := `Error: deployment to Vespa Cloud requires certificate in application package Hint: See https://cloud.vespa.ai/en/security/guide Hint: Pass --add-cert to use the certificate of the current application ` - assert.Equal(t, certError, stderr.String()) + assert.Equal(t, apiKeyWarning+certError, stderr.String()) require.Nil(t, cli.Run("deploy", "--add-cert", "--wait=0", pkgDir)) assert.Contains(t, stdout.String(), "Success: Triggered deployment") @@ -55,11 +58,89 @@ Hint: Pass --add-cert to use the certificate of the current application buf.WriteString("wat\nthe\nfck\nn\n") cli.Stdin = &buf require.NotNil(t, cli.Run("deploy", "--add-cert=false", "--wait=0", pkgDir2)) - warning := "Warning: Application package does not contain security/clients.pem, which is required for deployments to Vespa Cloud\n" + warning := apiKeyWarning + "Warning: Application package does not contain security/clients.pem, which is required for deployments to Vespa Cloud\n" assert.Equal(t, warning+strings.Repeat("Error: please answer 'y' or 'n'\n", 3)+certError, stderr.String()) buf.WriteString("y\n") require.Nil(t, cli.Run("deploy", "--add-cert=false", "--wait=0", pkgDir2)) assert.Contains(t, stdout.String(), "Success: Triggered deployment") + + // Missing application certificate is detected + stderr.Reset() + require.NotNil(t, cli.Run("deploy", "--application=t1.a2.i2", pkgDir2)) + assert.Equal(t, apiKeyWarning+"Error: no certificate exists for t1.a2.i2\nHint: Try (re)creating the certificate with 'vespa auth cert'\n", stderr.String()) + + // Mismatching certificate is detected + stdout.Reset() + stderr.Reset() + assert.Nil(t, cli.Run("auth", "cert", "--application=t1.a1.i1", "-f", "--no-add")) + require.NotNil(t, cli.Run("deploy", "--application=t1.a1.i1", pkgDir2)) + assert.Equal(t, apiKeyWarning+`Error: certificate in security/clients.pem does not match the stored key pair for t1.a1.i1 +Hint: If this application was deployed using a different application ID in the past, the matching key pair may be stored under a different ID in `+ + cli.config.homeDir+"\nHint: Specify the matching application with --application, or add the current certificate to the package using --add-cert\n", + stderr.String()) +} + +func TestDeployCloudFastWait(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, false) + + cli, stdout, stderr := newTestCLI(t, "CI=true") + httpClient := &mock.HTTPClient{} + cli.httpClient = httpClient + + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + assert.Nil(t, cli.Run("config", "set", "application", app.String())) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + + // Deployment completes quickly + httpClient.NextResponseString(200, `ok`) + httpClient.NextResponseString(200, `{"active": false, "status": "success"}`) + require.Nil(t, cli.Run("deploy", pkgDir)) + assert.Contains(t, stdout.String(), "Success: Triggered deployment") + assert.True(t, httpClient.Consumed()) + + // Deployment fails quickly + stdout.Reset() + stderr.Reset() + httpClient.NextResponseString(200, `ok`) + httpClient.NextResponseString(200, `{"active": false, "status": "unsuccesful"}`) + require.NotNil(t, cli.Run("deploy", pkgDir)) + assert.Equal(t, stderr.String(), "Error: deployment run 0 not yet complete after waiting up to 2s: aborting wait: deployment failed: run 0 ended with unsuccessful status: unsuccesful\n") + assert.True(t, httpClient.Consumed()) + + // Deployment which is running does not return error + stdout.Reset() + stderr.Reset() + httpClient.NextResponseString(200, `ok`) + httpClient.NextResponseString(200, `{"active": true, "status": "running"}`) + require.Nil(t, cli.Run("deploy", pkgDir)) + assert.Contains(t, stdout.String(), "Success: Triggered deployment") + assert.True(t, httpClient.Consumed()) +} + +func TestDeployCloudUnauthorized(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, false) + + cli, _, stderr := newTestCLI(t, "CI=true") + httpClient := &mock.HTTPClient{} + cli.httpClient = httpClient + + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + assert.Nil(t, cli.Run("config", "set", "application", app.String())) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + assert.Nil(t, cli.Run("auth", "cert", pkgDir)) + httpClient.NextResponseString(403, "bugger off") + require.NotNil(t, cli.Run("deploy", pkgDir)) + assert.Equal(t, `Error: deployment failed: unauthorized (status 403) +bugger off +Hint: You do not have access to the tenant t1 +Hint: You may need to create the tenant at https://console.vespa-cloud.com/tenant +Hint: If the tenant already exists you may need to run 'vespa auth login' to gain access to it +`, stderr.String()) } func TestDeployWait(t *testing.T) { @@ -94,8 +175,7 @@ func TestPrepareZip(t *testing.T) { } func TestActivateZip(t *testing.T) { - assertActivate("testdata/applications/withTarget/target/application.zip", - []string{"activate", "--wait=0", "testdata/applications/withTarget/target/application.zip"}, t) + assertActivate([]string{"activate", "--wait=0", "testdata/applications/withTarget/target/application.zip"}, t) } func TestDeployZip(t *testing.T) { @@ -145,14 +225,40 @@ func TestDeployApplicationDirectoryWithPomAndEmptyTarget(t *testing.T) { stderr.String()) } +func TestDeployIncludesExpectedFiles(t *testing.T) { + cli, stdout, _ := newTestCLI(t) + client := &mock.HTTPClient{} + cli.httpClient = client + assert.Nil(t, cli.Run("deploy", "--wait=0", "testdata/applications/withSource")) + applicationPackage := "testdata/applications/withSource/src/main/application" + assert.Equal(t, + "\nSuccess: Deployed '"+applicationPackage+"' with session ID 0\n", + stdout.String()) + + zipName := filepath.Join(t.TempDir(), "tmp.zip") + f, err := os.Create(zipName) + assert.Nil(t, err) + if _, err := io.Copy(f, client.LastRequest.Body); err != nil { + t.Fatal(err) + } + zr, err := zip.OpenReader(zipName) + assert.Nil(t, err) + defer zr.Close() + var zipFiles []string + for _, f := range zr.File { + zipFiles = append(zipFiles, f.Name) + } + assert.Equal(t, []string{".vespaignore", "hosts.xml", "schemas/msmarco.sd", "services.xml"}, zipFiles) +} + func TestDeployApplicationPackageErrorWithUnexpectedNonJson(t *testing.T) { - assertApplicationPackageError(t, "deploy", 401, + assertApplicationPackageError(t, "deploy", 400, "Raw text error", "Raw text error") } func TestDeployApplicationPackageErrorWithUnexpectedJson(t *testing.T) { - assertApplicationPackageError(t, "deploy", 401, + assertApplicationPackageError(t, "deploy", 400, `{ "some-unexpected-json": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here" }`, @@ -211,7 +317,7 @@ func assertPrepare(applicationPackage string, arguments []string, t *testing.T) assert.Equal(t, "PUT", client.Requests[1].Method) } -func assertActivate(applicationPackage string, arguments []string, t *testing.T) { +func assertActivate(arguments []string, t *testing.T) { t.Helper() client := &mock.HTTPClient{} cli, stdout, _ := newTestCLI(t) @@ -264,7 +370,7 @@ func assertApplicationPackageError(t *testing.T, cmd string, status int, expecte args = append(args, "testdata/applications/withTarget/target/application.zip") assert.NotNil(t, cli.Run(args...)) assert.Equal(t, - "Error: invalid application package (Status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n", + "Error: invalid application package (status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n", stderr.String()) } @@ -276,6 +382,6 @@ func assertDeployServerError(t *testing.T, status int, errorMessage string) { cli.httpClient = client assert.NotNil(t, cli.Run("deploy", "--wait=0", "testdata/applications/withTarget/target/application.zip")) assert.Equal(t, - "Error: error from deploy API at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", + "Error: error from deploy API at 127.0.0.1:19071 (status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", stderr.String()) } diff --git a/client/go/internal/cli/cmd/destroy.go b/client/go/internal/cli/cmd/destroy.go index a7beff2e4b4..51040c0af8b 100644 --- a/client/go/internal/cli/cmd/destroy.go +++ b/client/go/internal/cli/cmd/destroy.go @@ -23,14 +23,13 @@ When run interactively, the command will prompt for confirmation before removing the application. When run non-interactively, the command will refuse to remove the application unless the --force option is given. -This command can only be used to remove non-production deployments. See -https://cloud.vespa.ai/en/deleting-applications for how to remove -production deployments. This command can only be used for deployments to -Vespa Cloud, for other systems destroy an application by cleaning up -containers in use by the application, see e.g -https://github.com/vespa-engine/sample-apps/tree/master/examples/operations/multinode-HA#clean-up-after-testing +This command can only be used to remove non-production deployments, in Vespa +Cloud. See https://cloud.vespa.ai/en/deleting-applications for how to remove +production deployments. -`, +For other systems, destroy the application by removing the +containers in use by the application. For example: +https://github.com/vespa-engine/sample-apps/tree/master/examples/operations/multinode-HA#clean-up-after-testing`, Example: `$ vespa destroy $ vespa destroy -a mytenant.myapp.myinstance $ vespa destroy --force`, diff --git a/client/go/internal/cli/cmd/document.go b/client/go/internal/cli/cmd/document.go index 0393a9b2595..c9f0c780be3 100644 --- a/client/go/internal/cli/cmd/document.go +++ b/client/go/internal/cli/cmd/document.go @@ -27,8 +27,8 @@ func addDocumentFlags(cli *CLI, cmd *cobra.Command, printCurl *bool, timeoutSecs cli.bindWaitFlag(cmd, 0, waitSecs) } -func documentClient(cli *CLI, timeoutSecs, waitSecs int, printCurl bool) (*document.Client, *vespa.Service, error) { - docService, err := documentService(cli, waitSecs) +func documentClient(cli *CLI, timeoutSecs int, waiter *Waiter, printCurl bool) (*document.Client, *vespa.Service, error) { + docService, err := documentService(cli, waiter) if err != nil { return nil, nil, err } @@ -47,8 +47,8 @@ func documentClient(cli *CLI, timeoutSecs, waitSecs int, printCurl bool) (*docum return client, docService, nil } -func sendOperation(op document.Operation, args []string, timeoutSecs, waitSecs int, printCurl bool, cli *CLI) error { - client, service, err := documentClient(cli, timeoutSecs, waitSecs, printCurl) +func sendOperation(op document.Operation, args []string, timeoutSecs int, waiter *Waiter, printCurl bool, cli *CLI) error { + client, service, err := documentClient(cli, timeoutSecs, waiter, printCurl) if err != nil { return err } @@ -91,8 +91,8 @@ func sendOperation(op document.Operation, args []string, timeoutSecs, waitSecs i return printResult(cli, operationResult(false, doc, service, result), false) } -func readDocument(id string, timeoutSecs, waitSecs int, printCurl bool, cli *CLI) error { - client, service, err := documentClient(cli, timeoutSecs, waitSecs, printCurl) +func readDocument(id string, timeoutSecs int, waiter *Waiter, printCurl bool, cli *CLI, fieldSet string) error { + client, service, err := documentClient(cli, timeoutSecs, waiter, printCurl) if err != nil { return err } @@ -100,7 +100,7 @@ func readDocument(id string, timeoutSecs, waitSecs int, printCurl bool, cli *CLI if err != nil { return err } - result := client.Get(docId) + result := client.Get(docId, fieldSet) return printResult(cli, operationResult(true, document.Document{Id: docId}, service, result), true) } @@ -146,7 +146,8 @@ should be used instead of this.`, SilenceUsage: true, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return sendOperation(-1, args, timeoutSecs, waitSecs, printCurl, cli) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return sendOperation(-1, args, timeoutSecs, waiter, printCurl, cli) }, } addDocumentFlags(cli, cmd, &printCurl, &timeoutSecs, &waitSecs) @@ -171,7 +172,8 @@ $ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resour DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - return sendOperation(document.OperationPut, args, timeoutSecs, waitSecs, printCurl, cli) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return sendOperation(document.OperationPut, args, timeoutSecs, waiter, printCurl, cli) }, } addDocumentFlags(cli, cmd, &printCurl, &timeoutSecs, &waitSecs) @@ -195,7 +197,8 @@ $ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/res DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - return sendOperation(document.OperationUpdate, args, timeoutSecs, waitSecs, printCurl, cli) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return sendOperation(document.OperationUpdate, args, timeoutSecs, waiter, printCurl, cli) }, } addDocumentFlags(cli, cmd, &printCurl, &timeoutSecs, &waitSecs) @@ -219,8 +222,9 @@ $ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) if strings.HasPrefix(args[0], "id:") { - client, service, err := documentClient(cli, timeoutSecs, waitSecs, printCurl) + client, service, err := documentClient(cli, timeoutSecs, waiter, printCurl) if err != nil { return err } @@ -232,7 +236,7 @@ $ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, result := client.Send(doc) return printResult(cli, operationResult(false, doc, service, result), false) } else { - return sendOperation(document.OperationRemove, args, timeoutSecs, waitSecs, printCurl, cli) + return sendOperation(document.OperationRemove, args, timeoutSecs, waiter, printCurl, cli) } }, } @@ -245,6 +249,7 @@ func newDocumentGetCmd(cli *CLI) *cobra.Command { printCurl bool timeoutSecs int waitSecs int + fieldSet string ) cmd := &cobra.Command{ Use: "get id", @@ -254,19 +259,20 @@ func newDocumentGetCmd(cli *CLI) *cobra.Command { SilenceUsage: true, Example: `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`, RunE: func(cmd *cobra.Command, args []string) error { - return readDocument(args[0], timeoutSecs, waitSecs, printCurl, cli) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return readDocument(args[0], timeoutSecs, waiter, printCurl, cli, fieldSet) }, } + cmd.Flags().StringVar(&fieldSet, "field-set", "", "Fields to include when reading document") addDocumentFlags(cli, cmd, &printCurl, &timeoutSecs, &waitSecs) return cmd } -func documentService(cli *CLI, waitSecs int) (*vespa.Service, error) { +func documentService(cli *CLI, waiter *Waiter) (*vespa.Service, error) { target, err := cli.target(targetOptions{}) if err != nil { return nil, err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) return waiter.Service(target, cli.config.cluster()) } diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index 69b847547a9..d6bc59f5b4f 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -20,6 +20,7 @@ func addFeedFlags(cli *CLI, cmd *cobra.Command, options *feedOptions) { cmd.PersistentFlags().IntVar(&options.connections, "connections", 8, "The number of connections to use") cmd.PersistentFlags().StringVar(&options.compression, "compression", "auto", `Compression mode to use. Default is "auto" which compresses large documents. Must be "auto", "gzip" or "none"`) cmd.PersistentFlags().IntVar(&options.timeoutSecs, "timeout", 0, "Individual feed operation timeout in seconds. 0 to disable (default 0)") + cmd.Flags().StringSliceVarP(&options.headers, "header", "", nil, "Add a header to all HTTP requests, on the format 'Header: Value'. This can be specified multiple times") cmd.PersistentFlags().IntVar(&options.doomSecs, "deadline", 0, "Exit if this number of seconds elapse without any successful operations. 0 to disable (default 0)") cmd.PersistentFlags().BoolVar(&options.verbose, "verbose", false, "Verbose mode. Print successful operations in addition to errors") cmd.PersistentFlags().StringVar(&options.route, "route", "", `Target Vespa route for feed operations (default "default")`) @@ -49,6 +50,7 @@ type feedOptions struct { speedtestBytes int speedtestSecs int waitSecs int + headers []string memprofile string cpuprofile string @@ -108,7 +110,7 @@ $ cat docs.jsonl | vespa feed -`, pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } - err := feed(args, options, cli) + err := feed(args, options, cli, cmd) if options.memprofile != "" { f, err := os.Create(options.memprofile) if err != nil { @@ -124,7 +126,7 @@ $ cat docs.jsonl | vespa feed -`, return cmd } -func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]httputil.Client, string, error) { +func createServices(n int, timeout time.Duration, cli *CLI, waiter *Waiter) ([]httputil.Client, string, error) { if n < 1 { return nil, "", fmt.Errorf("need at least one client") } @@ -134,8 +136,7 @@ func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]htt } services := make([]httputil.Client, 0, n) baseURL := "" - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) - for i := 0; i < n; i++ { + for range n { service, err := waiter.Service(target, cli.config.cluster()) if err != nil { return nil, "", err @@ -144,7 +145,7 @@ func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]htt // Create a separate HTTP client for each service client := cli.httpClientFactory(timeout) // Feeding should always use HTTP/2 - httputil.ForceHTTP2(client, service.TLSOptions.KeyPair, service.TLSOptions.CACertificate, service.TLSOptions.TrustAll) + httputil.ForceHTTP2(client, service.TLSOptions.KeyPair, service.TLSOptions.CACertificatePEM, service.TLSOptions.TrustAll) service.SetClient(client) services = append(services, service) } @@ -228,9 +229,10 @@ func enqueueAndWait(files []string, dispatcher *document.Dispatcher, options fee return fmt.Errorf("at least one file to feed from must specified") } -func feed(files []string, options feedOptions, cli *CLI) error { +func feed(files []string, options feedOptions, cli *CLI, cmd *cobra.Command) error { timeout := time.Duration(options.timeoutSecs) * time.Second - clients, baseURL, err := createServices(options.connections, timeout, options.waitSecs, cli) + waiter := cli.waiter(time.Duration(options.waitSecs)*time.Second, cmd) + clients, baseURL, err := createServices(options.connections, timeout, cli, waiter) if err != nil { return err } @@ -238,12 +240,17 @@ func feed(files []string, options feedOptions, cli *CLI) error { if err != nil { return err } + header, err := httputil.ParseHeader(options.headers) + if err != nil { + return err + } client, err := document.NewClient(document.ClientOptions{ Compression: compression, Timeout: timeout, Route: options.route, TraceLevel: options.traceLevel, BaseURL: baseURL, + Header: header, Speedtest: options.speedtestBytes > 0, NowFunc: cli.now, }, clients) diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index 200a0be7c5d..fc25f8e872c 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -82,13 +82,13 @@ func TestFeed(t *testing.T) { require.Nil(t, cli.Run("feed", "-")) assert.Equal(t, want, stdout.String()) - for i := 0; i < 10; i++ { + for range 10 { httpClient.NextResponseString(503, `{"message":"it's broken yo"}`) } require.Nil(t, cli.Run("feed", jsonFile1)) assert.Equal(t, "feed: got status 503 ({\"message\":\"it's broken yo\"}) for put id:ns:type::doc1: giving up after 10 attempts\n", stderr.String()) stderr.Reset() - for i := 0; i < 10; i++ { + for range 10 { httpClient.NextResponseError(fmt.Errorf("something else is broken")) } require.Nil(t, cli.Run("feed", jsonFile1)) diff --git a/client/go/internal/cli/cmd/fetch.go b/client/go/internal/cli/cmd/fetch.go index b2e7d11ba7b..786fe75f9c2 100644 --- a/client/go/internal/cli/cmd/fetch.go +++ b/client/go/internal/cli/cmd/fetch.go @@ -13,8 +13,7 @@ func newFetchCmd(cli *CLI) *cobra.Command { This command can be used to download an already deployed Vespa application package. The package is written as a ZIP file to the given path, or current -directory if no path is given. -`, +directory if no path is given.`, Example: `$ vespa fetch $ vespa fetch mydir/ $ vespa fetch -t cloud mycloudapp.zip diff --git a/client/go/internal/cli/cmd/log.go b/client/go/internal/cli/cmd/log.go index 77ef7f68130..53b7079f428 100644 --- a/client/go/internal/cli/cmd/log.go +++ b/client/go/internal/cli/cmd/log.go @@ -6,6 +6,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/version" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -34,7 +35,7 @@ $ vespa log --follow`, SilenceUsage: true, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - target, err := cli.target(targetOptions{logLevel: levelArg, supportedType: cloudTargetOnly}) + target, err := cli.target(targetOptions{logLevel: levelArg}) if err != nil { return err } @@ -58,7 +59,12 @@ $ vespa log --follow`, options.To = to } if err := target.PrintLog(options); err != nil { - return fmt.Errorf("could not retrieve logs: %w", err) + versionWithLogContainer := version.MustParse("8.359.0") + var hints []string + if err := target.CompatibleWith(versionWithLogContainer); err != nil { + hints = []string{fmt.Sprintf("This command requires a newer version of the Vespa platform: %s", err)} + } + return errHint(fmt.Errorf("could not retrieve logs: %w", err), hints...) } return nil }, diff --git a/client/go/internal/cli/cmd/log_test.go b/client/go/internal/cli/cmd/log_test.go index c1cab951793..e8e8a76b988 100644 --- a/client/go/internal/cli/cmd/log_test.go +++ b/client/go/internal/cli/cmd/log_test.go @@ -9,7 +9,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/version" ) -func TestLog(t *testing.T) { +func TestLogCloud(t *testing.T) { _, pkgDir := mock.ApplicationPackageDir(t, false, false) httpClient := &mock.HTTPClient{} httpClient.NextResponseString(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) @@ -30,14 +30,13 @@ func TestLog(t *testing.T) { assert.Contains(t, stderr.String(), "Error: invalid period: cannot combine --from/--to with relative value: 1h\n") } -func TestLogOldClient(t *testing.T) { +func TestLogCloudIncompatible(t *testing.T) { cli, _, stderr := newTestCLI(t) cli.version = version.MustParse("7.0.0") _, pkgDir := mock.ApplicationPackageDir(t, false, false) httpClient := &mock.HTTPClient{} httpClient.NextResponseString(200, `{"minVersion": "8.0.0"}`) - httpClient.NextResponseString(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) cli.httpClient = httpClient assert.Nil(t, cli.Run("config", "set", "application", "t1.a1.i1")) @@ -46,6 +45,37 @@ func TestLogOldClient(t *testing.T) { assert.Nil(t, cli.Run("auth", "cert", pkgDir)) assert.Nil(t, cli.Run("log")) - expected := "Warning: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This version may not work as expected\nHint: Try 'vespa version' to check for a new version\n" + expected := "Warning: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This version of CLI may not work as expected\nHint: Try 'vespa version' to check for a new version\n" assert.Contains(t, stderr.String(), expected) } + +func TestLogLocal(t *testing.T) { + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(200, `1632738690.905535 localhost 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = httpClient + + assert.Nil(t, cli.Run("log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z")) + expected := "[2021-09-27 10:31:30.905535] localhost info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication Switching to the latest deployed set of configurations and components. Application config generation: 52532\n" + assert.Equal(t, expected, stdout.String()) + + assert.NotNil(t, cli.Run("log", "--from", "2021-09-27T13:12:49Z", "--to", "2021-09-27T13:15:00", "1h")) + assert.Contains(t, stderr.String(), "Error: invalid period: cannot combine --from/--to with relative value: 1h\n") +} + +func TestLogLocalIncompatible(t *testing.T) { + httpClient := &mock.HTTPClient{} + httpClient.NextResponseString(404, `not found`) + httpClient.NextResponse(mock.HTTPResponse{ + URI: "/state/v1/version", + Status: 200, + Body: []byte(`{"version": "8.358.0"}`), + }) + cli, _, stderr := newTestCLI(t) + cli.httpClient = httpClient + + assert.NotNil(t, cli.Run("log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z")) + assert.Equal(t, `Error: could not retrieve logs: failed to read logs: aborting wait: got status 404 +Hint: This command requires a newer version of the Vespa platform: platform version is older than required version: 8.358.0 < 8.359.0 +`, stderr.String()) +} diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go index 620ec055a1d..139e4690ed2 100644 --- a/client/go/internal/cli/cmd/prod.go +++ b/client/go/internal/cli/cmd/prod.go @@ -154,7 +154,7 @@ $ vespa prod deploy`, if err := verifyTests(cli, pkg); err != nil { return err } - if err := maybeCopyCertificate(options.copyCert, true, cli, target, pkg); err != nil { + if err := requireCertificate(options.copyCert, true, cli, target, pkg); err != nil { return err } deployment := vespa.DeploymentOptions{ApplicationPackage: pkg, Target: target} @@ -430,6 +430,6 @@ func verifyTest(cli *CLI, testsParent string, suite string, required bool) error } return nil } - _, _, err = runTests(cli, testDirectory, true, 0) + _, _, err = runTests(cli, testDirectory, true, nil) return err } diff --git a/client/go/internal/cli/cmd/prod_test.go b/client/go/internal/cli/cmd/prod_test.go index e2b0b3b88de..8ea20b3bbe5 100644 --- a/client/go/internal/cli/cmd/prod_test.go +++ b/client/go/internal/cli/cmd/prod_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/mock" "github.com/vespa-engine/vespa/client/go/internal/vespa" @@ -157,6 +158,29 @@ func TestProdDeploy(t *testing.T) { prodDeploy(pkgDir, t) } +func TestProdDeployWithoutCertificate(t *testing.T) { + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, false, false) + + httpClient := &mock.HTTPClient{} + cli, stdout, _ := newTestCLI(t, "CI=true") + cli.httpClient = httpClient + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + assert.Nil(t, cli.Run("config", "set", "application", app.String())) + assert.Nil(t, cli.Run("config", "set", "target", "cloud")) + assert.Nil(t, cli.Run("auth", "api-key")) + stdout.Reset() + cli.Environment["VESPA_CLI_API_KEY_FILE"] = filepath.Join(cli.config.homeDir, "t1.api-key.pem") + + // We have clients.pem, but no key pair for the application + require.Nil(t, os.MkdirAll(filepath.Join(pkgDir, "security"), 0755)) + require.Nil(t, os.WriteFile(filepath.Join(pkgDir, "security", "clients.pem"), []byte{}, 0644)) + httpClient.NextResponseString(200, `{"build": 42}`) + assert.Nil(t, cli.Run("prod", "deploy", pkgDir)) + assert.Contains(t, stdout.String(), "Success: Deployed '"+pkgDir+"' with build number 42") + assert.Contains(t, stdout.String(), "See https://console.vespa-cloud.com/tenant/t1/application/a1/prod/deployment for deployment progress") +} + func TestProdDeployWithoutTests(t *testing.T) { pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, false, true) diff --git a/client/go/internal/cli/cmd/query.go b/client/go/internal/cli/cmd/query.go index db6dfa0a158..5fa225777f0 100644 --- a/client/go/internal/cli/cmd/query.go +++ b/client/go/internal/cli/cmd/query.go @@ -5,7 +5,6 @@ package cmd import ( - "bufio" "encoding/json" "fmt" "io" @@ -17,6 +16,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/internal/curl" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/sse" "github.com/vespa-engine/vespa/client/go/internal/vespa" @@ -45,7 +45,8 @@ can be set by the syntax [parameter-name]=[value].`, SilenceUsage: true, Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return query(cli, args, queryTimeoutSecs, waitSecs, printCurl, format, headers) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + return query(cli, args, queryTimeoutSecs, printCurl, format, headers, waiter) }, } cmd.Flags().BoolVarP(&printCurl, "verbose", "v", false, "Print the equivalent curl command for the query") @@ -67,26 +68,11 @@ func printCurl(stderr io.Writer, url string, service *vespa.Service) error { return err } -func parseHeaders(headers []string) (http.Header, error) { - h := make(http.Header) - for _, header := range headers { - kv := strings.SplitN(header, ":", 2) - if len(kv) < 2 { - return nil, fmt.Errorf("invalid header %q: missing colon separator", header) - } - k := kv[0] - v := strings.TrimSpace(kv[1]) - h.Add(k, v) - } - return h, nil -} - -func query(cli *CLI, arguments []string, timeoutSecs, waitSecs int, curl bool, format string, headers []string) error { +func query(cli *CLI, arguments []string, timeoutSecs int, curl bool, format string, headers []string, waiter *Waiter) error { target, err := cli.target(targetOptions{}) if err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) service, err := waiter.Service(target, cli.config.cluster()) if err != nil { return err @@ -98,7 +84,7 @@ func query(cli *CLI, arguments []string, timeoutSecs, waitSecs int, curl bool, f } url, _ := url.Parse(service.BaseURL + "/search/") urlQuery := url.Query() - for i := 0; i < len(arguments); i++ { + for i := range len(arguments) { key, value := splitArg(arguments[i]) urlQuery.Set(key, value) } @@ -118,7 +104,7 @@ func query(cli *CLI, arguments []string, timeoutSecs, waitSecs int, curl bool, f return err } } - header, err := parseHeaders(headers) + header, err := httputil.ParseHeader(headers) if err != nil { return err } @@ -159,13 +145,11 @@ type printOptions struct { func printResponseBody(body io.Reader, options printOptions, cli *CLI) error { if options.plainStream { - scanner := bufio.NewScanner(body) - for scanner.Scan() { - fmt.Fprintln(cli.Stdout, scanner.Text()) - } - return scanner.Err() + _, err := io.Copy(cli.Stdout, body) + return err } else if options.tokenStream { - dec := sse.NewDecoder(body) + bufSize := 1024 * 1024 // Handle events up to this size + dec := sse.NewDecoderSize(body, bufSize) writingLine := false for { event, err := dec.Decode() diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 8e0f3de4f72..c0f6f3af51e 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -54,6 +54,7 @@ type CLI struct { now func() time.Time retryInterval time.Duration + waitTimeout *time.Duration cmd *cobra.Command config *Config @@ -141,7 +142,12 @@ func New(stdout, stderr io.Writer, environment []string) (*CLI, error) { Use it on Vespa instances running locally, remotely or in Vespa Cloud. -Vespa documentation: https://docs.vespa.ai +To get started, see the following quick start guides: + +- Local Vespa instance: https://docs.vespa.ai/en/vespa-quick-start.html +- Vespa Cloud: https://cloud.vespa.ai/en/getting-started + +The complete Vespa documentation is available at https://docs.vespa.ai. For detailed description of flags and configuration, see 'vespa help config'. `, @@ -153,6 +159,7 @@ For detailed description of flags and configuration, see 'vespa help config'. return fmt.Errorf("invalid command: %s", args[0]) }, } + cmd.CompletionOptions.HiddenDefaultCmd = true // Do not show the 'completion' command in help output env := make(map[string]string) for _, entry := range environment { parts := strings.SplitN(entry, "=", 2) @@ -376,7 +383,9 @@ func (c *CLI) confirm(question string, confirmByDefault bool) (bool, error) { } } -func (c *CLI) waiter(timeout time.Duration) *Waiter { return &Waiter{Timeout: timeout, cli: c} } +func (c *CLI) waiter(timeout time.Duration, cmd *cobra.Command) *Waiter { + return &Waiter{Timeout: timeout, cli: c, cmd: cmd} +} // target creates a target according the configuration of this CLI and given opts. func (c *CLI) target(opts targetOptions) (vespa.Target, error) { @@ -396,9 +405,9 @@ func (c *CLI) target(opts targetOptions) (vespa.Target, error) { if err != nil { return nil, err } - if !c.isCloudCI() { // Vespa Cloud always runs an up-to-date version - if err := target.CheckVersion(c.version); err != nil { - c.printWarning(err, "This version may not work as expected", "Try 'vespa version' to check for a new version") + if target.IsCloud() && !c.isCloudCI() { // Vespa Cloud always runs an up-to-date version + if err := target.CompatibleWith(c.version); err != nil { + c.printWarning(err, "This version of CLI may not work as expected", "Try 'vespa version' to check for a new version") } } return target, nil @@ -460,7 +469,7 @@ func (c *CLI) createCustomTarget(targetType, customURL string) (vespa.Target, er } func (c *CLI) cloudApiAuthenticator(deployment vespa.Deployment, system vespa.System) (vespa.Authenticator, error) { - apiKey, err := c.config.readAPIKey(c, system, deployment.Application.Tenant) + apiKey, err := c.config.readAPIKey(c, deployment.Application.Tenant) if err != nil { return nil, err } diff --git a/client/go/internal/cli/cmd/status.go b/client/go/internal/cli/cmd/status.go index 6056ee439b2..dff0c2d24c6 100644 --- a/client/go/internal/cli/cmd/status.go +++ b/client/go/internal/cli/cmd/status.go @@ -5,6 +5,7 @@ package cmd import ( + "errors" "fmt" "log" "strconv" @@ -49,7 +50,7 @@ $ vepsa status --format plain --cluster mycluster`, if err := verifyFormat(format); err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) var failingContainers []*vespa.Service if cluster == "" { services, err := waiter.Services(t) @@ -125,7 +126,7 @@ func newStatusDeployCmd(cli *CLI) *cobra.Command { if err := verifyFormat(format); err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) s, err := waiter.DeployService(t) if err != nil { return err @@ -149,7 +150,7 @@ func newStatusDeploymentCmd(cli *CLI) *cobra.Command { Long: `Show status of a Vespa deployment. This commands shows whether a Vespa deployment has converged on the latest run - (Vespa Cloud) or config generation (self-hosted). If an argument is given, +(Vespa Cloud) or config generation (self-hosted). If an argument is given, show the convergence status of that particular run or generation. `, Example: `$ vespa status deployment @@ -173,11 +174,11 @@ $ vespa status deployment -t local [session-id] --wait 600 if err != nil { return err } - waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) id, err := waiter.Deployment(t, wantedID) if err != nil { var hints []string - if waiter.Timeout == 0 { + if waiter.Timeout == 0 && !errors.Is(err, vespa.ErrDeployment) { hints = []string{"Consider using the --wait flag to wait for completion"} } return ErrCLI{Status: 1, warn: true, hints: hints, error: err} diff --git a/client/go/internal/cli/cmd/status_test.go b/client/go/internal/cli/cmd/status_test.go index 5ef96c462d8..6db27fd2778 100644 --- a/client/go/internal/cli/cmd/status_test.go +++ b/client/go/internal/cli/cmd/status_test.go @@ -85,7 +85,7 @@ func TestStatusError(t *testing.T) { cli.httpClient = client assert.NotNil(t, cli.Run("status", "container")) assert.Equal(t, - "Container default at http://127.0.0.1:8080 is not ready: unhealthy container default: status 500 at http://127.0.0.1:8080/status.html: giving up\n", + "Container default at http://127.0.0.1:8080 is not ready: unhealthy container default: status 500 at http://127.0.0.1:8080/status.html: wait deadline reached\n", stdout.String()) assert.Equal(t, "Error: services not ready: default\n", @@ -122,13 +122,13 @@ func TestStatusLocalDeployment(t *testing.T) { resp.Body = []byte(`{"currentGeneration": 42, "converged": false}`) client.NextResponse(resp) assert.NotNil(t, cli.Run("status", "deployment")) - assert.Equal(t, "Warning: deployment not converged on latest generation: giving up\nHint: Consider using the --wait flag to wait for completion\n", stderr.String()) + assert.Equal(t, "Warning: deployment not converged on latest generation: wait deadline reached\nHint: Consider using the --wait flag to wait for completion\n", stderr.String()) // Explicit generation stderr.Reset() client.NextResponse(resp) assert.NotNil(t, cli.Run("status", "deployment", "41")) - assert.Equal(t, "Warning: deployment not converged on generation 41: giving up\nHint: Consider using the --wait flag to wait for completion\n", stderr.String()) + assert.Equal(t, "Warning: deployment not converged on generation 41: wait deadline reached\nHint: Consider using the --wait flag to wait for completion\n", stderr.String()) } func TestStatusCloudDeployment(t *testing.T) { @@ -164,11 +164,11 @@ func TestStatusCloudDeployment(t *testing.T) { Body: []byte(`{"active": false, "status": "failure"}`), }) assert.NotNil(t, cli.Run("status", "deployment", "42", "-w", "10")) - assert.Equal(t, "Waiting up to 10s for deployment to converge...\nWarning: deployment run 42 incomplete after waiting up to 10s: aborting wait: run 42 ended with unsuccessful status: failure\n", stderr.String()) + assert.Equal(t, "Waiting up to 10s for deployment to converge...\nWarning: deployment run 42 not yet complete after waiting up to 10s: aborting wait: deployment failed: run 42 ended with unsuccessful status: failure\n", stderr.String()) } func isLocalTarget(args []string) bool { - for i := 0; i < len(args)-1; i++ { + for i := range len(args) - 1 { if args[i] == "-t" { return args[i+1] == "local" } @@ -197,7 +197,7 @@ func assertStatus(expectedTarget string, args []string, t *testing.T) { t.Helper() client := &mock.HTTPClient{} clusterName := "" - for i := 0; i < 3; i++ { + for range 3 { if isLocalTarget(args) { clusterName = "foo" mockServiceStatus(client, clusterName) diff --git a/client/go/internal/cli/cmd/test.go b/client/go/internal/cli/cmd/test.go index 3bc78fc91c8..5144abe95b4 100644 --- a/client/go/internal/cli/cmd/test.go +++ b/client/go/internal/cli/cmd/test.go @@ -42,7 +42,8 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - count, failed, err := runTests(cli, args[0], false, waitSecs) + waiter := cli.waiter(time.Duration(waitSecs)*time.Second, cmd) + count, failed, err := runTests(cli, args[0], false, waiter) if err != nil { return err } @@ -70,7 +71,7 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, return testCmd } -func runTests(cli *CLI, rootPath string, dryRun bool, waitSecs int) (int, []string, error) { +func runTests(cli *CLI, rootPath string, dryRun bool, waiter *Waiter) (int, []string, error) { count := 0 failed := make([]string, 0) if stat, err := os.Stat(rootPath); err != nil { @@ -89,7 +90,7 @@ func runTests(cli *CLI, rootPath string, dryRun bool, waitSecs int) (int, []stri fmt.Fprintln(cli.Stdout, "") previousFailed = false } - failure, err := runTest(testPath, context, waitSecs) + failure, err := runTest(testPath, context, waiter) if err != nil { return 0, nil, err } @@ -101,7 +102,7 @@ func runTests(cli *CLI, rootPath string, dryRun bool, waitSecs int) (int, []stri } } } else if strings.HasSuffix(stat.Name(), ".json") { - failure, err := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun, cli: cli, clusters: map[string]*vespa.Service{}}, waitSecs) + failure, err := runTest(rootPath, testContext{testsPath: filepath.Dir(rootPath), dryRun: dryRun, cli: cli, clusters: map[string]*vespa.Service{}}, waiter) if err != nil { return 0, nil, err } @@ -117,7 +118,7 @@ func runTests(cli *CLI, rootPath string, dryRun bool, waitSecs int) (int, []stri } // Runs the test at the given path, and returns the specified test name if the test fails -func runTest(testPath string, context testContext, waitSecs int) (string, error) { +func runTest(testPath string, context testContext, waiter *Waiter) (string, error) { var test test testBytes, err := os.ReadFile(testPath) if err != nil { @@ -150,7 +151,7 @@ func runTest(testPath string, context testContext, waitSecs int) (string, error) if step.Name != "" { stepName += ": " + step.Name } - failure, longFailure, err := verify(step, test.Defaults.Cluster, defaultParameters, context, waitSecs) + failure, longFailure, err := verify(step, test.Defaults.Cluster, defaultParameters, context, waiter) if err != nil { fmt.Fprintln(context.cli.Stderr) return "", errHint(fmt.Errorf("error in %s: %w", stepName, err), "See https://docs.vespa.ai/en/reference/testing") @@ -173,7 +174,7 @@ func runTest(testPath string, context testContext, waitSecs int) (string, error) } // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails -func verify(step step, defaultCluster string, defaultParameters map[string]string, context testContext, waitSecs int) (string, string, error) { +func verify(step step, defaultCluster string, defaultParameters map[string]string, context testContext, waiter *Waiter) (string, string, error) { requestBody, err := getBody(step.Request.BodyRaw, context.testsPath) if err != nil { return "", "", err @@ -227,9 +228,8 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin } ok := false service, ok = context.clusters[cluster] - if !ok { + if !ok && waiter != nil { // Cache service so we don't have to discover it for every step - waiter := context.cli.waiter(time.Duration(waitSecs) * time.Second) service, err = waiter.Service(target, cluster) if err != nil { return "", "", err diff --git a/client/go/internal/cli/cmd/test_test.go b/client/go/internal/cli/cmd/test_test.go index 728e8c29691..3479e057e45 100644 --- a/client/go/internal/cli/cmd/test_test.go +++ b/client/go/internal/cli/cmd/test_test.go @@ -26,11 +26,11 @@ func TestSuite(t *testing.T) { mockServiceStatus(client, "container") client.NextStatus(200) client.NextStatus(200) - for i := 0; i < 2; i++ { + for range 2 { client.NextResponseString(200, string(searchResponse)) } mockServiceStatus(client, "container") // Some tests do not specify cluster, which is fine since we only have one, but this causes a cache miss - for i := 0; i < 9; i++ { + for range 9 { client.NextResponseString(200, string(searchResponse)) } expectedBytes, _ := os.ReadFile("testdata/tests/expected-suite.out") @@ -45,7 +45,7 @@ func TestSuite(t *testing.T) { requests = append(requests, discoveryRequest) requests = append(requests, createSearchRequest(baseUrl+"/search/")) requests = append(requests, createSearchRequest(baseUrl+"/search/?foo=%2F")) - for i := 0; i < 7; i++ { + for range 7 { requests = append(requests, createSearchRequest(baseUrl+"/search/")) } assertRequests(requests, client, t) diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/.vespaignore b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/.vespaignore new file mode 100644 index 00000000000..24ca95c5b1e --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/.vespaignore @@ -0,0 +1,2 @@ +ignored-dir/ +ignored-file diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-dir/file b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-dir/file new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-dir/file @@ -0,0 +1 @@ + diff --git a/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-file b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-file new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/applications/withSource/src/main/application/ignored-file diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index 963833337c2..7f99df2038e 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -109,7 +109,8 @@ $ vespa visit --field-set "[id]" # list document IDs if !result.Success { return fmt.Errorf("argument error: %s", result.Message) } - service, err := documentService(cli, vArgs.waitSecs) + waiter := cli.waiter(time.Duration(vArgs.waitSecs)*time.Second, cmd) + service, err := documentService(cli, waiter) if err != nil { return err } diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index 3053b987838..85594912da2 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -41,7 +41,7 @@ func TestQuoteFunc(t *testing.T) { var buf []byte = make([]byte, 3) buf[0] = 'a' buf[2] = 'z' - for i := 0; i < 256; i++ { + for i := range 256 { buf[1] = byte(i) s := string(buf) res := quoteArgForUrl(s) @@ -97,7 +97,7 @@ func withMockClient(t *testing.T, prepCli func(*mock.HTTPClient), runOp func(*ve prepCli(client) cli, _, _ := newTestCLI(t) cli.httpClient = client - service, err := documentService(cli, 0) + service, err := documentService(cli, &Waiter{cli: cli}) if err != nil { t.Fatal(err) } diff --git a/client/go/internal/cli/cmd/waiter.go b/client/go/internal/cli/cmd/waiter.go index d818359e61c..8a25e18cd1e 100644 --- a/client/go/internal/cli/cmd/waiter.go +++ b/client/go/internal/cli/cmd/waiter.go @@ -2,10 +2,12 @@ package cmd import ( + "errors" "fmt" "time" "github.com/fatih/color" + "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -15,6 +17,7 @@ type Waiter struct { Timeout time.Duration // TODO(mpolden): Consider making this a budget cli *CLI + cmd *cobra.Command } // DeployService returns the service providing the deploy API on given target, @@ -81,10 +84,30 @@ func (w *Waiter) services(target vespa.Target) ([]*vespa.Service, error) { return target.ContainerServices(w.Timeout) } +// FastWaitOn returns whether we should use a short default timeout for given target. +func (w *Waiter) FastWaitOn(target vespa.Target) bool { + return target.IsCloud() && w.Timeout == 0 && !w.cmd.PersistentFlags().Changed("wait") +} + // Deployment waits for a deployment to become ready, returning the ID of the converged deployment. -func (w *Waiter) Deployment(target vespa.Target, id int64) (int64, error) { - if w.Timeout > 0 { - w.cli.printInfo("Waiting up to ", color.CyanString(w.Timeout.String()), " for deployment to converge...") +func (w *Waiter) Deployment(target vespa.Target, wantedID int64) (int64, error) { + timeout := w.Timeout + fastWait := w.FastWaitOn(target) + if timeout > 0 { + w.cli.printInfo("Waiting up to ", color.CyanString(timeout.String()), " for deployment to converge...") + } else if fastWait { + // If --wait is not explicitly given, we always wait a few seconds in Cloud to catch fast failures, e.g. + // invalid application package + timeout = 2 * time.Second + } + id, err := target.AwaitDeployment(wantedID, timeout) + if errors.Is(err, vespa.ErrWaitTimeout) { + if fastWait { + return id, nil // Do not report fast wait timeout as an error + } + if target.IsCloud() { + w.cli.printInfo("Timed out waiting for deployment to converge. See ", color.CyanString(target.Deployment().System.ConsoleRunURL(target.Deployment(), wantedID)), " for more details") + } } - return target.AwaitDeployment(id, w.Timeout) + return id, err } diff --git a/client/go/internal/httputil/httputil.go b/client/go/internal/httputil/httputil.go index e1e27de5523..56ac31d93a8 100644 --- a/client/go/internal/httputil/httputil.go +++ b/client/go/internal/httputil/httputil.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/http" + "strings" "time" "github.com/vespa-engine/vespa/client/go/internal/build" @@ -99,3 +100,19 @@ func NewClient(timeout time.Duration) Client { }, } } + +// ParseHeader parses headers slice into a http.Header. Each element in the slice is expected to contain a string on +// the format "Header: Value". +func ParseHeader(headers []string) (http.Header, error) { + h := make(http.Header) + for _, header := range headers { + kv := strings.SplitN(header, ":", 2) + if len(kv) < 2 { + return nil, fmt.Errorf("invalid header %q: missing colon separator", header) + } + k := kv[0] + v := strings.TrimSpace(kv[1]) + h.Add(k, v) + } + return h, nil +} diff --git a/client/go/internal/mock/http.go b/client/go/internal/mock/http.go index 2fbaa85ecca..8274f4113d3 100644 --- a/client/go/internal/mock/http.go +++ b/client/go/internal/mock/http.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "os" "strconv" "time" ) @@ -17,6 +18,9 @@ type HTTPClient struct { // The errors to return for future requests. If non-nil, these errors are returned before any responses in nextResponses. nextErrors []error + // LogRequests enables logging of all requests made through this client. + LogRequests bool + // LastRequest is the last HTTP request made through this. LastRequest *http.Request @@ -56,7 +60,12 @@ func (c *HTTPClient) NextResponseError(err error) { c.nextErrors = append(c.nextErrors, err) } +func (c *HTTPClient) Consumed() bool { return len(c.nextResponses) == 0 } + func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { + if c.LogRequests { + fmt.Fprintf(os.Stderr, "Sending request %+v\n", request) + } c.LastRequest = request if len(c.nextErrors) > 0 { err := c.nextErrors[0] diff --git a/client/go/internal/sse/sse.go b/client/go/internal/sse/sse.go index 9a120944eec..caccc90d354 100644 --- a/client/go/internal/sse/sse.go +++ b/client/go/internal/sse/sse.go @@ -87,6 +87,15 @@ func NewDecoder(r io.Reader) *Decoder { return &Decoder{scanner: bufio.NewScanner(r)} } +// NewDecoderSize creates a new Decoder that reads from r. The size argument specifies of the size of the buffer that +// decoder will use when decoding events. Size must be large enough to fit the largest expected event. +func NewDecoderSize(r io.Reader, size int) *Decoder { + scanner := bufio.NewScanner(r) + buf := make([]byte, 0, size) + scanner.Buffer(buf, size) + return &Decoder{scanner: scanner} +} + // IsEnd returns whether this event indicates that the stream has ended. func (e Event) IsEnd() bool { return e.Name == "end" } diff --git a/client/go/internal/sse/sse_test.go b/client/go/internal/sse/sse_test.go index 0e0d6929c75..3e3decaacec 100644 --- a/client/go/internal/sse/sse_test.go +++ b/client/go/internal/sse/sse_test.go @@ -41,6 +41,13 @@ event: end assertDecodeErr(io.EOF, dec, t) } +func TestDecoderLarge(t *testing.T) { + data := strings.Repeat("c", (256*1024)-50) + r := strings.NewReader("event: foo\nid: 42\ndata: " + data + "\n") + dec := NewDecoderSize(r, 256*1024) + assertDecode(&Event{Name: "foo", ID: "42", Data: data}, dec, t) +} + func TestDecoderInvalid(t *testing.T) { r := strings.NewReader(` event: foo diff --git a/client/go/internal/vespa/application.go b/client/go/internal/vespa/application.go index a65fb41d783..d499006c982 100644 --- a/client/go/internal/vespa/application.go +++ b/client/go/internal/vespa/application.go @@ -3,6 +3,7 @@ package vespa import ( "archive/zip" + "bytes" "errors" "fmt" "io" @@ -11,6 +12,7 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/ioutil" + "github.com/vespa-engine/vespa/client/go/internal/vespa/ignore" ) type ApplicationPackage struct { @@ -20,6 +22,14 @@ type ApplicationPackage struct { func (ap *ApplicationPackage) HasCertificate() bool { return ap.hasFile("security", "clients.pem") } +func (ap *ApplicationPackage) HasMatchingCertificate(certificatePEM []byte) (bool, error) { + clientsPEM, err := os.ReadFile(filepath.Join(ap.Path, "security", "clients.pem")) + if err != nil { + return false, err + } + return bytes.Equal(clientsPEM, certificatePEM), nil +} + func (ap *ApplicationPackage) HasDeploymentSpec() bool { return ap.hasFile("deployment.xml", "") } func (ap *ApplicationPackage) hasFile(pathSegment ...string) bool { @@ -73,7 +83,7 @@ func (ap *ApplicationPackage) Validate() error { func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" } -func zipDir(dir string, destination string) error { +func zipDir(dir string, destination string, ignores *ignore.List) error { if !ioutil.Exists(dir) { message := "'" + dir + "' should be an application package zip or dir, but does not exist" return errors.New(message) @@ -82,22 +92,23 @@ func zipDir(dir string, destination string) error { message := "'" + dir + "' should be an application package dir, but is a (non-zip) file" return errors.New(message) } - file, err := os.Create(destination) if err != nil { message := "Could not create a temporary zip file for the application package: " + err.Error() return errors.New(message) } defer file.Close() - w := zip.NewWriter(file) defer w.Close() - walker := func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if ignorePackageFile(filepath.Base(path)) { + zipPath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if ignores.Match(zipPath) { if info.IsDir() { return filepath.SkipDir } @@ -106,22 +117,16 @@ func zipDir(dir string, destination string) error { if info.IsDir() { return nil } - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - zippath, err := filepath.Rel(dir, path) + srcFile, err := os.Open(path) if err != nil { return err } - zipfile, err := w.Create(zippath) + defer srcFile.Close() + zipFile, err := w.Create(zipPath) if err != nil { return err } - - _, err = io.Copy(zipfile, file) + _, err = io.Copy(zipFile, srcFile) if err != nil { return err } @@ -130,39 +135,38 @@ func zipDir(dir string, destination string) error { return filepath.Walk(dir, walker) } -func ignorePackageFile(name string) bool { - switch name { - case ".DS_Store": - return true +func (ap *ApplicationPackage) openZip(name string) (io.ReadCloser, error) { + f, err := os.Open(name) + if err != nil { + return nil, fmt.Errorf("could not open application package at '%s': %w", ap.Path, err) } - return false + return f, nil } func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { - zipFile := ap.Path + path := ap.Path if test { - zipFile = ap.TestPath + path = ap.TestPath } - if !ap.IsZip() { - tempZip, err := os.CreateTemp("", "vespa") - if err != nil { - return nil, fmt.Errorf("could not create a temporary zip file for the application package: %w", err) - } - defer func() { - tempZip.Close() - os.Remove(tempZip.Name()) - // TODO: Caller must remove temporary file - }() - if err := zipDir(zipFile, tempZip.Name()); err != nil { - return nil, err - } - zipFile = tempZip.Name() + if ap.IsZip() { + return ap.openZip(path) } - f, err := os.Open(zipFile) + tmp, err := os.CreateTemp("", "vespa") if err != nil { - return nil, fmt.Errorf("could not open application package at '%s': %w", ap.Path, err) + return nil, fmt.Errorf("could not create a temporary zip file for the application package: %w", err) } - return f, nil + defer func() { + tmp.Close() + os.Remove(tmp.Name()) + }() + ignores, err := ignore.ReadFile(filepath.Join(path, ".vespaignore")) + if err != nil { + return nil, fmt.Errorf("could not read .vespaignore: %w", err) + } + if err := zipDir(path, tmp.Name(), ignores); err != nil { + return nil, err + } + return ap.openZip(tmp.Name()) } func (ap *ApplicationPackage) Unzip(test bool) (string, error) { @@ -283,7 +287,7 @@ func findApplicationPackage(zipOrDir string, options PackageOptions) (Applicatio testPath := existingPath(filepath.Join(zipOrDir, "src", "test", "application")) return ApplicationPackage{Path: path, TestPath: testPath}, nil } - // Application without Java components + // Application without Maven/Java if ioutil.Exists(filepath.Join(zipOrDir, "services.xml")) { testPath := "" if ioutil.Exists(filepath.Join(zipOrDir, "tests")) { diff --git a/client/go/internal/vespa/deploy.go b/client/go/internal/vespa/deploy.go index df74d1def8b..2c96b8b0935 100644 --- a/client/go/internal/vespa/deploy.go +++ b/client/go/internal/vespa/deploy.go @@ -7,6 +7,7 @@ package vespa import ( "bytes" "encoding/json" + "errors" "fmt" "io" "mime/multipart" @@ -20,12 +21,14 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/version" + "github.com/vespa-engine/vespa/client/go/internal/vespa/ignore" ) var ( DefaultApplication = ApplicationID{Tenant: "default", Application: "application", Instance: "default"} DefaultZone = ZoneID{Environment: "prod", Region: "default"} DefaultDeployment = Deployment{Application: DefaultApplication, Zone: DefaultZone} + ErrUnauthorized = errors.New("unauthorized") ) type ApplicationID struct { @@ -197,7 +200,7 @@ func fetchFromConfigServer(deployment DeploymentOptions, path string) error { return err } zipFile := filepath.Join(tmpDir, "application.zip") - if err := zipDir(dir, zipFile); err != nil { + if err := zipDir(dir, zipFile, &ignore.List{}); err != nil { return err } return os.Rename(zipFile, path) @@ -537,10 +540,12 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (PrepareResu } func checkResponse(req *http.Request, response *http.Response) error { - if response.StatusCode/100 == 4 { - return fmt.Errorf("invalid application package (%s)\n%s", response.Status, extractError(response.Body)) + if response.StatusCode == 401 || response.StatusCode == 403 { + return fmt.Errorf("deployment failed: %w (status %d)\n%s", ErrUnauthorized, response.StatusCode, ioutil.ReaderToJSON(response.Body)) + } else if response.StatusCode/100 == 4 { + return fmt.Errorf("invalid application package (status %d)\n%s", response.StatusCode, extractError(response.Body)) } else if response.StatusCode != 200 { - return fmt.Errorf("error from deploy API at %s (%s):\n%s", req.URL.Host, response.Status, ioutil.ReaderToJSON(response.Body)) + return fmt.Errorf("error from deploy API at %s (status %d):\n%s", req.URL.Host, response.StatusCode, ioutil.ReaderToJSON(response.Body)) } return nil } diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index e2a77f7b126..9c301cd7990 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" "sync" - "time" // Why do we use an experimental parser? This appears to be the only JSON library that satisfies the following @@ -19,7 +18,7 @@ import ( // - Supports parsing from a io.Reader // - Supports parsing token-by-token // - Few allocations during parsing (especially for large objects) - "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" ) type Operation int @@ -29,11 +28,11 @@ const ( OperationUpdate OperationRemove - jsonArrayStart json.Kind = '[' - jsonArrayEnd json.Kind = ']' - jsonObjectStart json.Kind = '{' - jsonObjectEnd json.Kind = '}' - jsonString json.Kind = '"' + jsonArrayStart jsontext.Kind = '[' + jsonArrayEnd jsontext.Kind = ']' + jsonObjectStart jsontext.Kind = '{' + jsonObjectEnd jsontext.Kind = '}' + jsonString jsontext.Kind = '"' ) var ( @@ -153,7 +152,7 @@ func (d *Document) Reset() { // Decoder decodes documents from a JSON structure which is either an array of objects, or objects separated by newline. type Decoder struct { - dec *json.Decoder + dec *jsontext.Decoder buf bytes.Buffer array bool @@ -202,13 +201,13 @@ func (d *Decoder) guessMode() error { return nil } -func (d *Decoder) readNext(kind json.Kind) (json.Token, error) { +func (d *Decoder) readNext(kind jsontext.Kind) (jsontext.Token, error) { t, err := d.dec.ReadToken() if err != nil { - return json.Token{}, err + return jsontext.Token{}, err } if t.Kind() != kind { - return json.Token{}, fmt.Errorf("unexpected json kind: %q: want %q", t, kind) + return jsontext.Token{}, fmt.Errorf("unexpected json kind: %q: want %q", t, kind) } return t, nil } @@ -364,7 +363,7 @@ loop: func NewDecoder(r io.Reader) *Decoder { d := &Decoder{} d.documentBuffers.New = func() any { return &bytes.Buffer{} } - d.dec = json.NewDecoder(io.TeeReader(r, &d.buf)) + d.dec = jsontext.NewDecoder(io.TeeReader(r, &d.buf)) return d } diff --git a/client/go/internal/vespa/document/document_test.go b/client/go/internal/vespa/document/document_test.go index 3fcdbd3b292..8875ad83291 100644 --- a/client/go/internal/vespa/document/document_test.go +++ b/client/go/internal/vespa/document/document_test.go @@ -176,7 +176,7 @@ func testDocumentDecoder(t *testing.T, jsonLike string) { if len(docs) != len(result) { t.Errorf("len(result) = %d, want %d", len(result), len(docs)) } - for i := 0; i < len(docs); i++ { + for i := range len(docs) { got := result[i] want := docs[i] if !got.Equal(want) { @@ -206,7 +206,7 @@ func TestDocumentDecoderInvalid(t *testing.T) { t.Errorf("unexpected error: %s", err) } _, err = dec.Decode() - wantErr := "invalid operation at byte offset 110: json: invalid character '\\n' within string (expecting non-control character)" + wantErr := "invalid operation at byte offset 110: jsontext: invalid character '\\n' within string (expecting non-control character)" if err.Error() != wantErr { t.Errorf("want error %q, got %q", wantErr, err.Error()) } diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 3871ab19edd..df4d97e2a82 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -3,6 +3,7 @@ package document import ( "bytes" + "encoding/json" "fmt" "io" "math" @@ -15,7 +16,6 @@ import ( "sync/atomic" "time" - "github.com/go-json-experiment/json" "github.com/klauspost/compress/gzip" "github.com/vespa-engine/vespa/client/go/internal/httputil" @@ -43,6 +43,7 @@ type Client struct { // ClientOptions specifices the configuration options of a feed client. type ClientOptions struct { BaseURL string + Header http.Header Timeout time.Duration Route string TraceLevel int @@ -94,48 +95,48 @@ func NewClient(options ClientOptions, httpClients []httputil.Client) (*Client, e } c.gzippers.New = func() any { return gzip.NewWriter(io.Discard) } c.buffers.New = func() any { return &bytes.Buffer{} } - for i := 0; i < runtime.NumCPU(); i++ { + for range runtime.NumCPU() { go c.preparePending() } return c, nil } -func writeQueryParam(sb *bytes.Buffer, start int, escape bool, k, v string) { - if sb.Len() == start { - sb.WriteString("?") +func writeQueryParam(buf *bytes.Buffer, start int, escape bool, k, v string) { + if buf.Len() == start { + buf.WriteString("?") } else { - sb.WriteString("&") + buf.WriteString("&") } - sb.WriteString(k) - sb.WriteString("=") + buf.WriteString(k) + buf.WriteString("=") if escape { - sb.WriteString(url.QueryEscape(v)) + buf.WriteString(url.QueryEscape(v)) } else { - sb.WriteString(v) + buf.WriteString(v) } } -func (c *Client) writeDocumentPath(id Id, sb *bytes.Buffer) { - sb.WriteString(strings.TrimSuffix(c.options.BaseURL, "/")) - sb.WriteString("/document/v1/") - sb.WriteString(url.PathEscape(id.Namespace)) - sb.WriteString("/") - sb.WriteString(url.PathEscape(id.Type)) +func (c *Client) writeDocumentPath(id Id, buf *bytes.Buffer) { + buf.WriteString(strings.TrimSuffix(c.options.BaseURL, "/")) + buf.WriteString("/document/v1/") + buf.WriteString(url.PathEscape(id.Namespace)) + buf.WriteString("/") + buf.WriteString(url.PathEscape(id.Type)) if id.Number != nil { - sb.WriteString("/number/") + buf.WriteString("/number/") n := uint64(*id.Number) - sb.WriteString(strconv.FormatUint(n, 10)) + buf.WriteString(strconv.FormatUint(n, 10)) } else if id.Group != "" { - sb.WriteString("/group/") - sb.WriteString(url.PathEscape(id.Group)) + buf.WriteString("/group/") + buf.WriteString(url.PathEscape(id.Group)) } else { - sb.WriteString("/docid") + buf.WriteString("/docid") } - sb.WriteString("/") - sb.WriteString(url.PathEscape(id.UserSpecific)) + buf.WriteString("/") + buf.WriteString(url.PathEscape(id.UserSpecific)) } -func (c *Client) methodAndURL(d Document, sb *bytes.Buffer) (string, string) { +func (c *Client) methodAndURL(d Document, buf *bytes.Buffer) (string, string) { httpMethod := "" switch d.Operation { case OperationPut: @@ -146,28 +147,28 @@ func (c *Client) methodAndURL(d Document, sb *bytes.Buffer) (string, string) { httpMethod = http.MethodDelete } // Base URL and path - c.writeDocumentPath(d.Id, sb) + c.writeDocumentPath(d.Id, buf) // Query part - queryStart := sb.Len() + queryStart := buf.Len() if c.options.Timeout > 0 { - writeQueryParam(sb, queryStart, false, "timeout", strconv.FormatInt(c.options.Timeout.Milliseconds(), 10)+"ms") + writeQueryParam(buf, queryStart, false, "timeout", strconv.FormatInt(c.options.Timeout.Milliseconds(), 10)+"ms") } if c.options.Route != "" { - writeQueryParam(sb, queryStart, true, "route", c.options.Route) + writeQueryParam(buf, queryStart, true, "route", c.options.Route) } if c.options.TraceLevel > 0 { - writeQueryParam(sb, queryStart, false, "tracelevel", strconv.Itoa(c.options.TraceLevel)) + writeQueryParam(buf, queryStart, false, "tracelevel", strconv.Itoa(c.options.TraceLevel)) } if c.options.Speedtest { - writeQueryParam(sb, queryStart, false, "dryRun", "true") + writeQueryParam(buf, queryStart, false, "dryRun", "true") } if d.Condition != "" { - writeQueryParam(sb, queryStart, true, "condition", d.Condition) + writeQueryParam(buf, queryStart, true, "condition", d.Condition) } if d.Create { - writeQueryParam(sb, queryStart, false, "create", "true") + writeQueryParam(buf, queryStart, false, "create", "true") } - return httpMethod, sb.String() + return httpMethod, buf.String() } func (c *Client) leastBusyClient() *countingHTTPClient { @@ -216,11 +217,14 @@ func (c *Client) prepare(document Document) (*http.Request, *bytes.Buffer, error return pd.request, pd.buf, pd.err } -func newRequest(method, url string, body io.Reader, gzipped bool) (*http.Request, error) { +func (c *Client) newRequest(method, url string, body io.Reader, gzipped bool) (*http.Request, error) { req, err := http.NewRequest(method, url, body) if err != nil { return nil, err } + for k, v := range c.options.Header { + req.Header[k] = v + } req.Header.Set("Content-Type", "application/json; charset=utf-8") if gzipped { req.Header.Set("Content-Encoding", "gzip") @@ -231,7 +235,7 @@ func newRequest(method, url string, body io.Reader, gzipped bool) (*http.Request func (c *Client) createRequest(method, url string, body []byte, buf *bytes.Buffer) (*http.Request, error) { buf.Reset() if len(body) == 0 { - return newRequest(method, url, nil, false) + return c.newRequest(method, url, nil, false) } useGzip := c.options.Compression == CompressionGzip || (c.options.Compression == CompressionAuto && len(body) > 512) var r io.Reader @@ -249,7 +253,7 @@ func (c *Client) createRequest(method, url string, body []byte, buf *bytes.Buffe } else { r = bytes.NewReader(body) } - return newRequest(method, url, r, useGzip) + return c.newRequest(method, url, r, useGzip) } func (c *Client) clientTimeout() time.Duration { @@ -282,11 +286,14 @@ func (c *Client) Send(document Document) Result { } // Get retrieves document with given ID. -func (c *Client) Get(id Id) Result { +func (c *Client) Get(id Id, fieldSet string) Result { start := c.now() buf := c.buffer() defer c.buffers.Put(buf) c.writeDocumentPath(id, buf) + if fieldSet != "" { + writeQueryParam(buf, buf.Len(), true, "fieldSet", fieldSet) + } url := buf.String() result := Result{Id: id} req, err := http.NewRequest(http.MethodGet, url, nil) @@ -328,7 +335,7 @@ func (c *Client) resultWithResponse(resp *http.Response, sentBytes int, result R } else { if result.Success() && c.options.TraceLevel > 0 { var jsonResponse struct { - Trace json.RawValue `json:"trace"` + Trace json.RawMessage `json:"trace"` } if err := json.Unmarshal(buf.Bytes(), &jsonResponse); err != nil { result = resultWithErr(result, fmt.Errorf("failed to decode json response: %w", err), elapsed) diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 89e9e96064b..878b7a98be3 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -34,7 +34,7 @@ type mockHTTPClient struct { func TestLeastBusyClient(t *testing.T) { httpClient := mock.HTTPClient{} var httpClients []httputil.Client - for i := 0; i < 4; i++ { + for i := range 4 { httpClients = append(httpClients, &mockHTTPClient{i, &httpClient}) } client, _ := NewClient(ClientOptions{}, httpClients) @@ -177,7 +177,7 @@ func TestClientGet(t *testing.T) { }` id := Id{Namespace: "mynamespace", Type: "music", UserSpecific: "doc1"} httpClient.NextResponseString(200, doc) - result := client.Get(id) + result := client.Get(id, "") want := Result{ Id: id, Body: []byte(doc), @@ -189,6 +189,17 @@ func TestClientGet(t *testing.T) { if !reflect.DeepEqual(want, result) { t.Errorf("got %+v, want %+v", result, want) } + gotURL := httpClient.LastRequest.URL.String() + wantURL := "https://example.com:1337/document/v1/mynamespace/music/docid/doc1" + if gotURL != wantURL { + t.Errorf("got URL=%s, want %s", gotURL, wantURL) + } + client.Get(id, "[all]") + gotURL = httpClient.LastRequest.URL.String() + wantURL = "https://example.com:1337/document/v1/mynamespace/music/docid/doc1?fieldSet=%5Ball%5D" + if gotURL != wantURL { + t.Errorf("got URL=%s, want %s", gotURL, wantURL) + } } func TestClientSendCompressed(t *testing.T) { diff --git a/client/go/internal/vespa/document/throttler_test.go b/client/go/internal/vespa/document/throttler_test.go index eba8cbd2972..3cdecb22be4 100644 --- a/client/go/internal/vespa/document/throttler_test.go +++ b/client/go/internal/vespa/document/throttler_test.go @@ -13,7 +13,7 @@ func TestThrottler(t *testing.T) { if got, want := tr.TargetInflight(), int64(16); got != want { t.Errorf("got TargetInflight() = %d, but want %d", got, want) } - for i := 0; i < 65; i++ { + for range 65 { tr.Sent() tr.Success() } diff --git a/client/go/internal/vespa/ignore/ignore.go b/client/go/internal/vespa/ignore/ignore.go new file mode 100644 index 00000000000..ea045ec326b --- /dev/null +++ b/client/go/internal/vespa/ignore/ignore.go @@ -0,0 +1,62 @@ +package ignore + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// List is a list of ignore patterns. +type List struct{ patterns []string } + +// Match returns whether path matches any pattern in this list. +func (l *List) Match(path string) bool { + for _, pattern := range l.patterns { + if ok, _ := filepath.Match(pattern, path); ok { + return true + } + // A directory exclude applies to all subpaths + if strings.HasSuffix(pattern, string(filepath.Separator)) && strings.HasPrefix(path, pattern) { + return true + } + } + return false +} + +// Read reads an ignore list from reader r. +func Read(r io.Reader) (*List, error) { + scanner := bufio.NewScanner(r) + ignore := List{} + line := 0 + for scanner.Scan() { + line++ + pattern := strings.TrimSpace(scanner.Text()) + if pattern == "" || strings.HasPrefix(pattern, "#") { + continue + } + if _, err := filepath.Match(pattern, ""); err != nil { + return nil, fmt.Errorf("line %d: bad pattern: %s: %w", line, pattern, err) + } + ignore.patterns = append(ignore.patterns, pattern) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return &ignore, nil +} + +// ReadFile reads an ignore list from the named file. Reading a non-existent file returns an empty list, and no error. +func ReadFile(name string) (*List, error) { + f, err := os.Open(name) + if err != nil { + if os.IsNotExist(err) { + return &List{}, nil + } + return nil, err + } + defer f.Close() + return Read(f) +} diff --git a/client/go/internal/vespa/ignore/ignore_test.go b/client/go/internal/vespa/ignore/ignore_test.go new file mode 100644 index 00000000000..79ed437cc2d --- /dev/null +++ b/client/go/internal/vespa/ignore/ignore_test.go @@ -0,0 +1,50 @@ +package ignore + +import ( + "strings" + "testing" +) + +func TestRead(t *testing.T) { + f := ` +# files + +foo +foob* +???.tmp + +# directories + + foo/bar +foo/*/baz +bar/ +` + list, err := Read(strings.NewReader(f)) + if err != nil { + t.Fatal(err) + } + assertMatch(t, list, "", false) + assertMatch(t, list, "\n", false) + assertMatch(t, list, "foo1", false) + assertMatch(t, list, "foo", true) + assertMatch(t, list, "foobar", true) + assertMatch(t, list, "foo/bar", true) + assertMatch(t, list, "foo/bar/baz", true) + assertMatch(t, list, "foo/bar/bax", false) + assertMatch(t, list, "bar", false) + assertMatch(t, list, "bar/", true) + assertMatch(t, list, "bar/x", true) + assertMatch(t, list, "foo.tmp", true) + assertMatch(t, list, "fooo.tmp", false) + + _, err = Read(strings.NewReader("myfile[")) + if err == nil { + t.Fatal("want error") + } +} + +func assertMatch(t *testing.T, list *List, name string, match bool) { + if got := list.Match(name); got != match { + t.Errorf("Match(%q) = %t, want %t", name, got, match) + } +} diff --git a/client/go/internal/vespa/load_env.go b/client/go/internal/vespa/load_env.go index 5cae03694bc..4eb1297e711 100644 --- a/client/go/internal/vespa/load_env.go +++ b/client/go/internal/vespa/load_env.go @@ -151,7 +151,7 @@ func nSpacedFields(s string, n int) []string { // pretty strict for now, can be more lenient if needed func isValidShellVariableName(s string) bool { - for i := 0; i < len(s); i++ { + for i := range len(s) { b := s[i] switch { case (b >= 'A' && b <= 'Z'): // ok diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 94eb2cbafe4..674bedc9343 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -3,11 +3,14 @@ package vespa import ( + "bytes" "crypto/tls" "errors" "fmt" "io" + "math" "net/http" + "strconv" "strings" "time" @@ -36,9 +39,15 @@ const ( AnyDeployment int64 = -2 ) -var errWaitTimeout = errors.New("giving up") var errAuth = errors.New("auth failed") +var ( + // ErrWaitTimeout is the error returned when waiting for something times out. + ErrWaitTimeout = errors.New("wait deadline reached") + // ErrDeployment is the error returned for terminal deployment failures. + ErrDeployment = errors.New("deployment failed") +) + // Authenticator authenticates the given HTTP request. type Authenticator interface { Authenticate(request *http.Request) error @@ -114,15 +123,18 @@ type Target interface { // PrintLog writes the logs of this deployment using given options to control output. PrintLog(options LogOptions) error - // CheckVersion verifies whether clientVersion is compatible with this target. - CheckVersion(clientVersion version.Version) error + // CompatibleWith returns nil if target is compatible with the given version. + CompatibleWith(version version.Version) error } // TLSOptions holds the client certificate to use for cloud API or service requests. type TLSOptions struct { - CACertificate []byte - KeyPair []tls.Certificate - TrustAll bool + KeyPair []tls.Certificate + TrustAll bool + + CACertificatePEM []byte + CertificatePEM []byte + PrivateKeyPEM []byte CACertificateFile string CertificateFile string @@ -143,7 +155,7 @@ type LogOptions struct { func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { if !s.customClient { // Do not override TLS config if a custom client has been configured - httputil.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) + httputil.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificatePEM, s.TLSOptions.TrustAll) } if s.auth != nil { if err := s.auth.Authenticate(request); err != nil { @@ -243,6 +255,64 @@ func isOK(status int) (bool, error) { } } +func deployServiceWait(target Target, fn responseFunc, reqFn requestFunc, timeout, retryInterval time.Duration) (int, error) { + deployService, err := target.DeployService() + if err != nil { + return 0, err + } + return wait(deployService, fn, reqFn, timeout, retryInterval) +} + +func pollLogs(target Target, logsURL string, options LogOptions, retryInterval time.Duration) error { + req, err := http.NewRequest("GET", logsURL, nil) + if err != nil { + return err + } + lastFrom := options.From + requestFunc := func() *http.Request { + fromMillis := lastFrom.Unix() * 1000 + q := req.URL.Query() + q.Set("from", strconv.FormatInt(fromMillis, 10)) + if !options.To.IsZero() { + toMillis := options.To.Unix() * 1000 + q.Set("to", strconv.FormatInt(toMillis, 10)) + } + req.URL.RawQuery = q.Encode() + return req + } + logFunc := func(status int, response []byte) (bool, error) { + if ok, err := isOK(status); !ok { + return ok, err + } + logEntries, err := ReadLogEntries(bytes.NewReader(response)) + if err != nil { + return false, err + } + for _, le := range logEntries { + if !le.Time.After(lastFrom) { + continue + } + if LogLevel(le.Level) > options.Level { + continue + } + fmt.Fprintln(options.Writer, le.Format(options.Dequote)) + } + if len(logEntries) > 0 { + lastFrom = logEntries[len(logEntries)-1].Time + } + return false, nil + } + var timeout time.Duration + if options.Follow { + timeout = math.MaxInt64 // No timeout + } + // Ignore wait error because logFunc has no concept of completion, we just want to print log entries until timeout is reached + if _, err := deployServiceWait(target, logFunc, requestFunc, timeout, retryInterval); err != nil && !errors.Is(err, ErrWaitTimeout) { + return fmt.Errorf("failed to read logs: %s", err) + } + return nil +} + // responseFunc returns whether a HTTP request is considered successful, based on its status and response data. // Returning false indicates that the operation should be retried. An error is returned if the response is considered // terminal and that the request should not be retried. @@ -290,7 +360,7 @@ func wait(service *Service, okFn responseFunc, reqFn requestFunc, timeout, retry time.Sleep(retryInterval) } if err == nil { - return status, errWaitTimeout + return status, ErrWaitTimeout } return status, err } diff --git a/client/go/internal/vespa/target_cloud.go b/client/go/internal/vespa/target_cloud.go index c063b99edef..05d6bdd224e 100644 --- a/client/go/internal/vespa/target_cloud.go +++ b/client/go/internal/vespa/target_cloud.go @@ -2,11 +2,8 @@ package vespa import ( - "bytes" "encoding/json" - "errors" "fmt" - "math" "net/http" "sort" "strconv" @@ -148,7 +145,7 @@ func (t *cloudTarget) ContainerServices(timeout time.Duration) ([]*Service, erro return services, nil } -func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { +func (t *cloudTarget) CompatibleWith(clientVersion version.Version) error { if clientVersion.IsZero() { // development version is always fine return nil } @@ -190,61 +187,7 @@ func (t *cloudTarget) logsURL() string { } func (t *cloudTarget) PrintLog(options LogOptions) error { - req, err := http.NewRequest("GET", t.logsURL(), nil) - if err != nil { - return err - } - lastFrom := options.From - requestFunc := func() *http.Request { - fromMillis := lastFrom.Unix() * 1000 - q := req.URL.Query() - q.Set("from", strconv.FormatInt(fromMillis, 10)) - if !options.To.IsZero() { - toMillis := options.To.Unix() * 1000 - q.Set("to", strconv.FormatInt(toMillis, 10)) - } - req.URL.RawQuery = q.Encode() - return req - } - logFunc := func(status int, response []byte) (bool, error) { - if ok, err := isOK(status); !ok { - return ok, err - } - logEntries, err := ReadLogEntries(bytes.NewReader(response)) - if err != nil { - return false, err - } - for _, le := range logEntries { - if !le.Time.After(lastFrom) { - continue - } - if LogLevel(le.Level) > options.Level { - continue - } - fmt.Fprintln(options.Writer, le.Format(options.Dequote)) - } - if len(logEntries) > 0 { - lastFrom = logEntries[len(logEntries)-1].Time - } - return false, nil - } - var timeout time.Duration - if options.Follow { - timeout = math.MaxInt64 // No timeout - } - // Ignore wait error because logFunc has no concept of completion, we just want to print log entries until timeout is reached - if _, err := t.deployServiceWait(logFunc, requestFunc, timeout); err != nil && !errors.Is(err, errWaitTimeout) { - return fmt.Errorf("failed to read logs: %s", err) - } - return nil -} - -func (t *cloudTarget) deployServiceWait(fn responseFunc, reqFn requestFunc, timeout time.Duration) (int, error) { - deployService, err := t.DeployService() - if err != nil { - return 0, err - } - return wait(deployService, fn, reqFn, timeout, t.retryInterval) + return pollLogs(t, t.logsURL(), options, t.retryInterval) } func (t *cloudTarget) discoverLatestRun(timeout time.Duration) (int64, error) { @@ -269,7 +212,7 @@ func (t *cloudTarget) discoverLatestRun(timeout time.Duration) (int64, error) { } return false, nil } - _, err = t.deployServiceWait(jobsSuccessFunc, requestFunc, timeout) + _, err = deployServiceWait(t, jobsSuccessFunc, requestFunc, timeout, t.retryInterval) return lastRunID, err } @@ -309,17 +252,17 @@ func (t *cloudTarget) AwaitDeployment(runID int64, timeout time.Duration) (int64 return false, nil } if resp.Status != "success" { - return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status) + return false, fmt.Errorf("%w: run %d ended with unsuccessful status: %s", ErrDeployment, runID, resp.Status) } success = true return success, nil } - _, err = t.deployServiceWait(jobSuccessFunc, requestFunc, timeout) + _, err = deployServiceWait(t, jobSuccessFunc, requestFunc, timeout, t.retryInterval) if err != nil { - return 0, fmt.Errorf("deployment run %d incomplete%s: %w", runID, waitDescription(timeout), err) + return 0, fmt.Errorf("deployment run %d not yet complete%s: %w", runID, waitDescription(timeout), err) } if !success { - return 0, fmt.Errorf("deployment run %d incomplete%s", runID, waitDescription(timeout)) + return 0, fmt.Errorf("deployment run %d not yet complete%s", runID, waitDescription(timeout)) } return runID, nil } @@ -378,7 +321,7 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) (map[string]strin } return true, nil } - if _, err := t.deployServiceWait(endpointFunc, func() *http.Request { return req }, timeout); err != nil { + if _, err := deployServiceWait(t, endpointFunc, func() *http.Request { return req }, timeout, t.retryInterval); err != nil { return nil, fmt.Errorf("no endpoints found in zone %s%s: %w", t.deploymentOptions.Deployment.Zone, waitDescription(timeout), err) } if len(urlsByCluster) == 0 { diff --git a/client/go/internal/vespa/target_custom.go b/client/go/internal/vespa/target_custom.go index 9d62f7dc297..1f72308178a 100644 --- a/client/go/internal/vespa/target_custom.go +++ b/client/go/internal/vespa/target_custom.go @@ -64,10 +64,48 @@ func (t *customTarget) IsCloud() bool { return false } func (t *customTarget) Deployment() Deployment { return DefaultDeployment } func (t *customTarget) PrintLog(options LogOptions) error { - return fmt.Errorf("log access is only supported on cloud: run vespa-logfmt on the admin node instead, or export from a container image (here named 'vespa') using docker exec vespa vespa-logfmt") + deployService, err := t.DeployService() + if err != nil { + return err + } + logsURL := deployService.BaseURL + "/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/logs" + return pollLogs(t, logsURL, options, t.retryInterval) } -func (t *customTarget) CheckVersion(version version.Version) error { return nil } +func (t *customTarget) CompatibleWith(minVersion version.Version) error { + if minVersion.IsZero() { // development version is always fine + return nil + } + deployService, err := t.DeployService() + if err != nil { + return err + } + versionURL := deployService.BaseURL + "/state/v1/version" + req, err := http.NewRequest("GET", versionURL, nil) + if err != nil { + return err + } + var versionResponse struct { + Version string `json:"version"` + } + response, err := deployService.Do(req, 10*time.Second) + if err != nil { + return err + } + defer response.Body.Close() + dec := json.NewDecoder(response.Body) + if err := dec.Decode(&versionResponse); err != nil { + return err + } + targetVersion, err := version.Parse(versionResponse.Version) + if err != nil { + return err + } + if targetVersion.Less(minVersion) { + return fmt.Errorf("platform version is older than required version: %s < %s", targetVersion, minVersion) + } + return nil +} func (t *customTarget) newService(url, name string, deployAPI bool) *Service { return &Service{ diff --git a/client/go/internal/vespa/target_test.go b/client/go/internal/vespa/target_test.go index 4c2fda8368e..2b4cf485b83 100644 --- a/client/go/internal/vespa/target_test.go +++ b/client/go/internal/vespa/target_test.go @@ -20,7 +20,7 @@ func TestLocalTarget(t *testing.T) { client := &mock.HTTPClient{} lt := LocalTarget(client, TLSOptions{}, 0) assertServiceURL(t, "http://127.0.0.1:19071", lt, "deploy") - for i := 0; i < 2; i++ { + for range 2 { response := ` { "services": [ @@ -83,7 +83,7 @@ func TestCustomTargetWait(t *testing.T) { client.NextStatus(500) assertService(t, true, target, "", 0) // Fails multiple times - for i := 0; i < 3; i++ { + for range 3 { client.NextStatus(500) client.NextResponseError(io.EOF) } @@ -117,6 +117,22 @@ func TestCustomTargetAwaitDeployment(t *testing.T) { assert.Equal(t, int64(42), convergedID) } +func TestCustomTargetCompatibleWith(t *testing.T) { + client := &mock.HTTPClient{} + target := CustomTarget(client, "http://192.0.2.42", TLSOptions{}, 0) + for range 3 { + client.NextResponse(mock.HTTPResponse{ + URI: "/state/v1/version", + Status: 200, + Body: []byte(`{"version": "1.2.3"}`), + }) + } + assert.Nil(t, target.CompatibleWith(version.MustParse("1.2.2"))) + assert.Nil(t, target.CompatibleWith(version.MustParse("1.2.3"))) + assert.NotNil(t, target.CompatibleWith(version.MustParse("1.2.4"))) + assert.True(t, client.Consumed()) +} + func TestCloudTargetWait(t *testing.T) { var logWriter bytes.Buffer target, client := createCloudTarget(t, &logWriter) @@ -231,14 +247,14 @@ func TestLog(t *testing.T) { assert.Equal(t, expected, buf.String()) } -func TestCheckVersion(t *testing.T) { +func TestCloudCompatibleWith(t *testing.T) { target, client := createCloudTarget(t, io.Discard) - for i := 0; i < 3; i++ { + for range 3 { client.NextResponse(mock.HTTPResponse{URI: "/cli/v1/", Status: 200, Body: []byte(`{"minVersion":"8.0.0"}`)}) } - assert.Nil(t, target.CheckVersion(version.MustParse("8.0.0"))) - assert.Nil(t, target.CheckVersion(version.MustParse("8.1.0"))) - assert.NotNil(t, target.CheckVersion(version.MustParse("7.0.0"))) + assert.Nil(t, target.CompatibleWith(version.MustParse("8.0.0"))) + assert.Nil(t, target.CompatibleWith(version.MustParse("8.1.0"))) + assert.NotNil(t, target.CompatibleWith(version.MustParse("7.0.0"))) } func createCloudTarget(t *testing.T, logWriter io.Writer) (Target, *mock.HTTPClient) { diff --git a/client/js/app/package.json b/client/js/app/package.json index 0725a768ab5..04aec637ea1 100644 --- a/client/js/app/package.json +++ b/client/js/app/package.json @@ -31,7 +31,7 @@ "eslint-plugin-react": "^7", "eslint-plugin-react-hooks": "^4", "eslint-plugin-react-perf": "^3", - "eslint-plugin-unused-imports": "^3", + "eslint-plugin-unused-imports": "^4.0.0", "husky": "^9.0.0", "jest": "^29", "lodash": "^4", diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index d1f7c7c9516..2172bac94b8 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -3,14 +3,14 @@ "@ampproject/remapping@^2.2.0": - version "2.2.1" - resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630" - integrity sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg== + version "2.3.0" + resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.3.0.tgz#ed441b6fa600072520ce18b43d2c8cc8caecc7f4" + integrity sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw== dependencies: - "@jridgewell/gen-mapping" "^0.3.0" - "@jridgewell/trace-mapping" "^0.3.9" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.24" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.5": +"@babel/code-frame@^7.0.0": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.5.tgz#9009b69a8c602293476ad598ff53e4562e15c244" integrity sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA== @@ -26,11 +26,32 @@ "@babel/highlight" "^7.22.13" chalk "^2.4.2" +"@babel/code-frame@^7.22.13": + version "7.24.2" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.24.2.tgz#718b4b19841809a58b29b68cde80bc5e1aa6d9ae" + integrity sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ== + dependencies: + "@babel/highlight" "^7.24.2" + picocolors "^1.0.0" + +"@babel/code-frame@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.24.7.tgz#882fd9e09e8ee324e496bd040401c6f046ef4465" + integrity sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA== + dependencies: + "@babel/highlight" "^7.24.7" + picocolors "^1.0.0" + "@babel/compat-data@^7.22.9": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.5.tgz#ffb878728bb6bdcb6f4510aa51b1be9afb8cfd98" integrity sha512-uU27kfDRlhfKl+w1U6vp16IuvSLtjAxdArVXPa9BvLkrr7CYIsxH5adpHObeAGY/41+syctUWOZ140a2Rvkgjw== +"@babel/compat-data@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.24.7.tgz#d23bbea508c3883ba8251fb4164982c36ea577ed" + integrity sha512-qJzAIcv03PyaWqxRgO4mSU3lihncDT296vnyuE2O8uA4w3UHWI4S3hgeZd1L8W1Bft40w9JxJ2b412iDUFFRhw== + "@babel/core@^7.1.0", "@babel/core@^7.12.17": version "7.22.9" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.9.tgz#bd96492c68822198f33e8a256061da3cf391f58f" @@ -73,21 +94,21 @@ json5 "^2.2.3" semver "^6.3.1" -"@babel/core@^7.23.5": - version "7.23.5" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.5.tgz#6e23f2acbcb77ad283c5ed141f824fd9f70101c7" - integrity sha512-Cwc2XjUrG4ilcfOw4wBAK+enbdgwAcAJCfGUItPBKR7Mjw4aEfAFYrLxeRp4jWgtNIKn3n2AlBOfwwafl+42/g== +"@babel/core@^7.24.5": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.24.7.tgz#b676450141e0b52a3d43bc91da86aa608f950ac4" + integrity sha512-nykK+LEK86ahTkX/3TgauT0ikKoNCfKHEaZYTUVupJdTLzGNvrblu4u6fa7DhZONAltdf8e662t/abY8idrd/g== dependencies: "@ampproject/remapping" "^2.2.0" - "@babel/code-frame" "^7.23.5" - "@babel/generator" "^7.23.5" - "@babel/helper-compilation-targets" "^7.22.15" - "@babel/helper-module-transforms" "^7.23.3" - "@babel/helpers" "^7.23.5" - "@babel/parser" "^7.23.5" - "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.5" - "@babel/types" "^7.23.5" + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.24.7" + "@babel/helper-compilation-targets" "^7.24.7" + "@babel/helper-module-transforms" "^7.24.7" + "@babel/helpers" "^7.24.7" + "@babel/parser" "^7.24.7" + "@babel/template" "^7.24.7" + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" convert-source-map "^2.0.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -124,14 +145,14 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/generator@^7.23.5": - version "7.23.5" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.5.tgz#17d0a1ea6b62f351d281350a5f80b87a810c4755" - integrity sha512-BPssCHrBD+0YrxviOa3QzpqwhNIXKEtOa2jQrm4FlmkC2apYgRnQcmPWiGZDlGxiNtltnUFolMe8497Esry+jA== +"@babel/generator@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.24.7.tgz#1654d01de20ad66b4b4d99c135471bc654c55e6d" + integrity sha512-oipXieGC3i45Y1A41t4tAqpnEZWgB/lC6Ehh6+rOviR5XWpTtMmLN+fGjz9vOiNRt0p6RtO6DtD0pdU3vpqdSA== dependencies: - "@babel/types" "^7.23.5" - "@jridgewell/gen-mapping" "^0.3.2" - "@jridgewell/trace-mapping" "^0.3.17" + "@babel/types" "^7.24.7" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" jsesc "^2.5.1" "@babel/helper-compilation-targets@^7.22.15": @@ -156,25 +177,43 @@ lru-cache "^5.1.1" semver "^6.3.1" -"@babel/helper-environment-visitor@^7.22.20", "@babel/helper-environment-visitor@^7.22.5": +"@babel/helper-compilation-targets@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.24.7.tgz#4eb6c4a80d6ffeac25ab8cd9a21b5dfa48d503a9" + integrity sha512-ctSdRHBi20qWOfy27RUb4Fhp07KSJ3sXcuSvTrXrc4aG8NSYDo1ici3Vhg9bg69y5bj0Mr1lh0aeEgTvc12rMg== + dependencies: + "@babel/compat-data" "^7.24.7" + "@babel/helper-validator-option" "^7.24.7" + browserslist "^4.22.2" + lru-cache "^5.1.1" + semver "^6.3.1" + +"@babel/helper-environment-visitor@^7.22.20", "@babel/helper-environment-visitor@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.24.7.tgz#4b31ba9551d1f90781ba83491dd59cf9b269f7d9" + integrity sha512-DoiN84+4Gnd0ncbBOM9AZENV4a5ZiL39HYMyZJGZ/AZEykHYdJw0wW3kdcsh9/Kn+BRXHLkkklZ51ecPKmI1CQ== + dependencies: + "@babel/types" "^7.24.7" + +"@babel/helper-environment-visitor@^7.22.5": version "7.22.20" resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== -"@babel/helper-function-name@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" - integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== +"@babel/helper-function-name@^7.23.0", "@babel/helper-function-name@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.24.7.tgz#75f1e1725742f39ac6584ee0b16d94513da38dd2" + integrity sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA== dependencies: - "@babel/template" "^7.22.15" - "@babel/types" "^7.23.0" + "@babel/template" "^7.24.7" + "@babel/types" "^7.24.7" -"@babel/helper-hoist-variables@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" - integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== +"@babel/helper-hoist-variables@^7.22.5", "@babel/helper-hoist-variables@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.24.7.tgz#b4ede1cde2fd89436397f30dc9376ee06b0f25ee" + integrity sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ== dependencies: - "@babel/types" "^7.22.5" + "@babel/types" "^7.24.7" "@babel/helper-module-imports@^7.16.7", "@babel/helper-module-imports@^7.22.15", "@babel/helper-module-imports@^7.22.5": version "7.22.15" @@ -183,6 +222,14 @@ dependencies: "@babel/types" "^7.22.15" +"@babel/helper-module-imports@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz#f2f980392de5b84c3328fc71d38bd81bbb83042b" + integrity sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + "@babel/helper-module-transforms@^7.22.17", "@babel/helper-module-transforms@^7.22.9": version "7.22.17" resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.17.tgz#7edf129097a51ccc12443adbc6320e90eab76693" @@ -205,22 +252,27 @@ "@babel/helper-split-export-declaration" "^7.22.6" "@babel/helper-validator-identifier" "^7.22.5" -"@babel/helper-module-transforms@^7.23.3": - version "7.23.3" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz#d7d12c3c5d30af5b3c0fcab2a6d5217773e2d0f1" - integrity sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ== +"@babel/helper-module-transforms@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.24.7.tgz#31b6c9a2930679498db65b685b1698bfd6c7daf8" + integrity sha512-1fuJEwIrp+97rM4RWdO+qrRsZlAeL1lQJoPqtCYWv0NL115XM93hIH4CSRln2w52SqvmY5hqdtauB6QFCDiZNQ== dependencies: - "@babel/helper-environment-visitor" "^7.22.20" - "@babel/helper-module-imports" "^7.22.15" - "@babel/helper-simple-access" "^7.22.5" - "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/helper-validator-identifier" "^7.22.20" + "@babel/helper-environment-visitor" "^7.24.7" + "@babel/helper-module-imports" "^7.24.7" + "@babel/helper-simple-access" "^7.24.7" + "@babel/helper-split-export-declaration" "^7.24.7" + "@babel/helper-validator-identifier" "^7.24.7" "@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.14.5", "@babel/helper-plugin-utils@^7.22.5", "@babel/helper-plugin-utils@^7.8.0": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz#dd7ee3735e8a313b9f7b05a773d892e88e6d7295" integrity sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg== +"@babel/helper-plugin-utils@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.7.tgz#98c84fe6fe3d0d3ae7bfc3a5e166a46844feb2a0" + integrity sha512-Rq76wjt7yz9AAc1KnlRKNAi/dMSVWgDRx43FHoJEbcYU6xOWaE2dVPwcdTukJrjxS65GITyfbvEYHvkirZ6uEg== + "@babel/helper-simple-access@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz#4938357dc7d782b80ed6dbb03a0fba3d22b1d5de" @@ -228,6 +280,14 @@ dependencies: "@babel/types" "^7.22.5" +"@babel/helper-simple-access@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.24.7.tgz#bcade8da3aec8ed16b9c4953b74e506b51b5edb3" + integrity sha512-zBAIvbCMh5Ts+b86r/CjU+4XGYIs+R1j951gxI3KmmxBMhCg4oQMsv6ZXQ64XOm/cvzfU1FmoCyt6+owc5QMYg== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + "@babel/helper-split-export-declaration@^7.22.6": version "7.22.6" resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" @@ -235,16 +295,38 @@ dependencies: "@babel/types" "^7.22.5" -"@babel/helper-string-parser@^7.22.5", "@babel/helper-string-parser@^7.23.4": +"@babel/helper-split-export-declaration@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.24.7.tgz#83949436890e07fa3d6873c61a96e3bbf692d856" + integrity sha512-oy5V7pD+UvfkEATUKvIjvIAH/xCzfsFVw7ygW2SI6NClZzquT+mwdTfgfdbUiceh6iQO0CHtCPsyze/MZ2YbAA== + dependencies: + "@babel/types" "^7.24.7" + +"@babel/helper-string-parser@^7.22.5": version "7.23.4" resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz#9478c707febcbbe1ddb38a3d91a2e054ae622d83" integrity sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ== -"@babel/helper-validator-identifier@^7.22.15", "@babel/helper-validator-identifier@^7.22.19", "@babel/helper-validator-identifier@^7.22.20", "@babel/helper-validator-identifier@^7.22.5": +"@babel/helper-string-parser@^7.24.1", "@babel/helper-string-parser@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.24.7.tgz#4d2d0f14820ede3b9807ea5fc36dfc8cd7da07f2" + integrity sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg== + +"@babel/helper-validator-identifier@^7.22.15", "@babel/helper-validator-identifier@^7.22.19", "@babel/helper-validator-identifier@^7.22.5": version "7.22.20" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== +"@babel/helper-validator-identifier@^7.22.20": + version "7.24.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.5.tgz#918b1a7fa23056603506370089bd990d8720db62" + integrity sha512-3q93SSKX2TWCG30M2G2kwaKeTYgEUp5Snjuj8qm729SObL6nbtUldAi37qbxkD5gg3xnBio+f9nqpSepGZMvxA== + +"@babel/helper-validator-identifier@^7.24.5", "@babel/helper-validator-identifier@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz#75b889cfaf9e35c2aaf42cf0d72c8e91719251db" + integrity sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w== + "@babel/helper-validator-option@^7.22.15": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.23.5.tgz#907a3fbd4523426285365d1206c423c4c5520307" @@ -255,6 +337,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz#694c30dfa1d09a6534cdfcafbe56789d36aba040" integrity sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA== +"@babel/helper-validator-option@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.24.7.tgz#24c3bb77c7a425d1742eec8fb433b5a1b38e62f6" + integrity sha512-yy1/KvjhV/ZCL+SM7hBrvnZJ3ZuT9OuZgIJAGpPEToANvc3iM6iDvBnRjtElWibHU6n8/LPR/EjX9EtIEYO3pw== + "@babel/helpers@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.22.15.tgz#f09c3df31e86e3ea0b7ff7556d85cdebd47ea6f1" @@ -273,16 +360,15 @@ "@babel/traverse" "^7.22.11" "@babel/types" "^7.22.11" -"@babel/helpers@^7.23.5": - version "7.23.5" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.23.5.tgz#52f522840df8f1a848d06ea6a79b79eefa72401e" - integrity sha512-oO7us8FzTEsG3U6ag9MfdF1iA/7Z6dz+MtFhifZk8C8o453rGJFFWUP1t+ULM9TUIAzC9uxXEiXjOiVMyd7QPg== +"@babel/helpers@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.24.7.tgz#aa2ccda29f62185acb5d42fb4a3a1b1082107416" + integrity sha512-NlmJJtvcw72yRJRcnCmGvSi+3jDEg8qFu3z0AFoymmzLx5ERVWyzd9kVXr7Th9/8yIJi2Zc6av4Tqz3wFs8QWg== dependencies: - "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.5" - "@babel/types" "^7.23.5" + "@babel/template" "^7.24.7" + "@babel/types" "^7.24.7" -"@babel/highlight@^7.22.13", "@babel/highlight@^7.23.4": +"@babel/highlight@^7.22.13": version "7.23.4" resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.23.4.tgz#edaadf4d8232e1a961432db785091207ead0621b" integrity sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A== @@ -291,16 +377,41 @@ chalk "^2.4.2" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.23.5": - version "7.23.5" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.5.tgz#37dee97c4752af148e1d38c34b856b2507660563" - integrity sha512-hOOqoiNXrmGdFbhgCzu6GiURxUgM27Xwd/aPuu8RfHEZPBzL1Z54okAHAQjXfcQNwvrlkAmAp4SlRTZ45vlthQ== +"@babel/highlight@^7.23.4": + version "7.24.5" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.24.5.tgz#bc0613f98e1dd0720e99b2a9ee3760194a704b6e" + integrity sha512-8lLmua6AVh/8SLJRRVD6V8p73Hir9w5mJrhE+IPpILG31KKlI9iz5zmBYKcWPS59qSfgP9RaSBQSHHE81WKuEw== + dependencies: + "@babel/helper-validator-identifier" "^7.24.5" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/highlight@^7.24.2", "@babel/highlight@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.24.7.tgz#a05ab1df134b286558aae0ed41e6c5f731bf409d" + integrity sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw== + dependencies: + "@babel/helper-validator-identifier" "^7.24.7" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/parser@^7.1.0", "@babel/parser@^7.20.7", "@babel/parser@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.24.7.tgz#9a5226f92f0c5c8ead550b750f5608e766c8ce85" + integrity sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw== "@babel/parser@^7.14.7", "@babel/parser@^7.22.16": version "7.22.16" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.16.tgz#180aead7f247305cce6551bea2720934e2fa2c95" integrity sha512-+gPfKv8UWeKKeJTUxe59+OobVcrYHETCsORl61EmSkmgymguYk/X5bp7GuUIXaFsc6y++v8ZxPsLSSuujqDphA== +"@babel/parser@^7.22.15": + version "7.24.5" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.24.5.tgz#4a4d5ab4315579e5398a82dcf636ca80c3392790" + integrity sha512-EOv5IK8arwh3LI47dz1b0tKUb/1uhHAnHJOrjgtQMIpu1uXd9mlFrJg9IUgGUgZ41Ch0K8REPTYpO7B76b4vJg== + "@babel/parser@^7.22.7": version "7.22.13" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.13.tgz#23fb17892b2be7afef94f573031c2f4b42839a2b" @@ -418,19 +529,19 @@ "@babel/helper-plugin-utils" "^7.22.5" "@babel/helper-simple-access" "^7.22.5" -"@babel/plugin-transform-react-jsx-self@^7.23.3": - version "7.23.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.23.3.tgz#ed3e7dadde046cce761a8e3cf003a13d1a7972d9" - integrity sha512-qXRvbeKDSfwnlJnanVRp0SfuWE5DQhwQr5xtLBzp56Wabyo+4CMosF6Kfp+eOD/4FYpql64XVJ2W0pVLlJZxOQ== +"@babel/plugin-transform-react-jsx-self@^7.24.5": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.24.7.tgz#66bff0248ea0b549972e733516ffad577477bdab" + integrity sha512-fOPQYbGSgH0HUp4UJO4sMBFjY6DuWq+2i8rixyUMb3CdGixs/gccURvYOAhajBdKDoGajFr3mUq5rH3phtkGzw== dependencies: - "@babel/helper-plugin-utils" "^7.22.5" + "@babel/helper-plugin-utils" "^7.24.7" -"@babel/plugin-transform-react-jsx-source@^7.23.3": - version "7.23.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.23.3.tgz#03527006bdc8775247a78643c51d4e715fe39a3e" - integrity sha512-91RS0MDnAWDNvGC6Wio5XYkyWI39FMFO+JK9+4AlgaTH+yWwVTsw7/sn6LK0lH7c5F+TFkpv/3LfCJ1Ydwof/g== +"@babel/plugin-transform-react-jsx-source@^7.24.1": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.24.7.tgz#1198aab2548ad19582013815c938d3ebd8291ee3" + integrity sha512-J2z+MWzZHVOemyLweMqngXrgGC42jQ//R0KdxqkIz/OrbVIIlhFI3WigZ5fO+nwFvBlncr4MGapd8vTyc7RPNQ== dependencies: - "@babel/helper-plugin-utils" "^7.22.5" + "@babel/helper-plugin-utils" "^7.24.7" "@babel/runtime@^7.10.2", "@babel/runtime@^7.13.10", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.7": version "7.22.15" @@ -446,7 +557,16 @@ dependencies: regenerator-runtime "^0.14.0" -"@babel/template@^7.22.15", "@babel/template@^7.22.5", "@babel/template@^7.3.3": +"@babel/template@^7.22.15", "@babel/template@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.24.7.tgz#02efcee317d0609d2c07117cb70ef8fb17ab7315" + integrity sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/parser" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/template@^7.22.5", "@babel/template@^7.3.3": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== @@ -471,29 +591,29 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/traverse@^7.23.5": - version "7.23.5" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.5.tgz#f546bf9aba9ef2b042c0e00d245990c15508e7ec" - integrity sha512-czx7Xy5a6sapWWRx61m1Ke1Ra4vczu1mCTtJam5zRTBOonfdJ+S/B6HYmGYu3fJtr8GGET3si6IhgWVBhJ/m8w== - dependencies: - "@babel/code-frame" "^7.23.5" - "@babel/generator" "^7.23.5" - "@babel/helper-environment-visitor" "^7.22.20" - "@babel/helper-function-name" "^7.23.0" - "@babel/helper-hoist-variables" "^7.22.5" - "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.23.5" - "@babel/types" "^7.23.5" - debug "^4.1.0" +"@babel/traverse@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.24.7.tgz#de2b900163fa741721ba382163fe46a936c40cf5" + integrity sha512-yb65Ed5S/QAcewNPh0nZczy9JdYXkkAbIsEo+P7BE7yO3txAY30Y/oPa3QkQ5It3xVG2kpKMg9MsdxZaO31uKA== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.24.7" + "@babel/helper-environment-visitor" "^7.24.7" + "@babel/helper-function-name" "^7.24.7" + "@babel/helper-hoist-variables" "^7.24.7" + "@babel/helper-split-export-declaration" "^7.24.7" + "@babel/parser" "^7.24.7" + "@babel/types" "^7.24.7" + debug "^4.3.1" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.23.0", "@babel/types@^7.23.5": - version "7.23.5" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.5.tgz#48d730a00c95109fa4393352705954d74fb5b602" - integrity sha512-ON5kSOJwVO6xXVRTvOI0eOnWe7VdUcIpsovGo9U/Br4Ie4UVFQTboO2cYnDhAGU6Fp+UxSiT+pMft0SMHfuq6w== +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.24.7.tgz#6027fe12bc1aa724cd32ab113fb7f1988f1f66f2" + integrity sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q== dependencies: - "@babel/helper-string-parser" "^7.23.4" - "@babel/helper-validator-identifier" "^7.22.20" + "@babel/helper-string-parser" "^7.24.7" + "@babel/helper-validator-identifier" "^7.24.7" to-fast-properties "^2.0.0" "@babel/types@^7.22.10", "@babel/types@^7.22.11", "@babel/types@^7.3.3": @@ -506,12 +626,12 @@ to-fast-properties "^2.0.0" "@babel/types@^7.22.15": - version "7.23.9" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.9.tgz#1dd7b59a9a2b5c87f8b41e52770b5ecbf492e002" - integrity sha512-dQjSq/7HaSjRM43FFGnv5keM2HsxpmyV1PfaSVm0nzzjwwTmjOe6J4bC8e3+pTEIgHaHj+1ZlLThRJ2auc/w1Q== + version "7.24.5" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.24.5.tgz#7661930afc638a5383eb0c4aee59b74f38db84d7" + integrity sha512-6mQNsaLeXTw0nxYUYu+NSa4Hx4BlF1x1x8/PMFbiR+GBSr+2DkECc69b8hgy2frEodNcvPffeH8YfWd3LI6jhQ== dependencies: - "@babel/helper-string-parser" "^7.23.4" - "@babel/helper-validator-identifier" "^7.22.20" + "@babel/helper-string-parser" "^7.24.1" + "@babel/helper-validator-identifier" "^7.24.5" to-fast-properties "^2.0.0" "@babel/types@^7.22.17": @@ -523,15 +643,6 @@ "@babel/helper-validator-identifier" "^7.22.19" to-fast-properties "^2.0.0" -"@babel/types@^7.22.5": - version "7.23.6" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.6.tgz#be33fdb151e1f5a56877d704492c240fc71c7ccd" - integrity sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg== - dependencies: - "@babel/helper-string-parser" "^7.23.4" - "@babel/helper-validator-identifier" "^7.22.20" - to-fast-properties "^2.0.0" - "@babel/types@^7.23.3": version "7.23.3" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.3.tgz#d5ea892c07f2ec371ac704420f4dcdb07b5f9598" @@ -642,120 +753,120 @@ resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz#d0fce5d07b0620caa282b5131c297bb60f9d87e6" integrity sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww== -"@esbuild/aix-ppc64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.20.2.tgz#a70f4ac11c6a1dfc18b8bbb13284155d933b9537" - integrity sha512-D+EBOJHXdNZcLJRBkhENNG8Wji2kgc9AZ9KiPr1JuZjsNtyHzrsfLRrY0tk2H2aoFu6RANO1y1iPPUCDYWkb5g== - -"@esbuild/android-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz#db1c9202a5bc92ea04c7b6840f1bbe09ebf9e6b9" - integrity sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg== - -"@esbuild/android-arm@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.20.2.tgz#3b488c49aee9d491c2c8f98a909b785870d6e995" - integrity sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w== - -"@esbuild/android-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.20.2.tgz#3b1628029e5576249d2b2d766696e50768449f98" - integrity sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg== - -"@esbuild/darwin-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz#6e8517a045ddd86ae30c6608c8475ebc0c4000bb" - integrity sha512-4J6IRT+10J3aJH3l1yzEg9y3wkTDgDk7TSDFX+wKFiWjqWp/iCfLIYzGyasx9l0SAFPT1HwSCR+0w/h1ES/MjA== - -"@esbuild/darwin-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz#90ed098e1f9dd8a9381695b207e1cff45540a0d0" - integrity sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA== - -"@esbuild/freebsd-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz#d71502d1ee89a1130327e890364666c760a2a911" - integrity sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw== - -"@esbuild/freebsd-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz#aa5ea58d9c1dd9af688b8b6f63ef0d3d60cea53c" - integrity sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw== - -"@esbuild/linux-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz#055b63725df678379b0f6db9d0fa85463755b2e5" - integrity sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A== - -"@esbuild/linux-arm@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz#76b3b98cb1f87936fbc37f073efabad49dcd889c" - integrity sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg== - -"@esbuild/linux-ia32@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz#c0e5e787c285264e5dfc7a79f04b8b4eefdad7fa" - integrity sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig== - -"@esbuild/linux-loong64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz#a6184e62bd7cdc63e0c0448b83801001653219c5" - integrity sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ== - -"@esbuild/linux-mips64el@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz#d08e39ce86f45ef8fc88549d29c62b8acf5649aa" - integrity sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA== - -"@esbuild/linux-ppc64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz#8d252f0b7756ffd6d1cbde5ea67ff8fd20437f20" - integrity sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg== - -"@esbuild/linux-riscv64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz#19f6dcdb14409dae607f66ca1181dd4e9db81300" - integrity sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg== - -"@esbuild/linux-s390x@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz#3c830c90f1a5d7dd1473d5595ea4ebb920988685" - integrity sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ== - -"@esbuild/linux-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz#86eca35203afc0d9de0694c64ec0ab0a378f6fff" - integrity sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw== - -"@esbuild/netbsd-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz#e771c8eb0e0f6e1877ffd4220036b98aed5915e6" - integrity sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ== - -"@esbuild/openbsd-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz#9a795ae4b4e37e674f0f4d716f3e226dd7c39baf" - integrity sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ== - -"@esbuild/sunos-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz#7df23b61a497b8ac189def6e25a95673caedb03f" - integrity sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w== - -"@esbuild/win32-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz#f1ae5abf9ca052ae11c1bc806fb4c0f519bacf90" - integrity sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ== - -"@esbuild/win32-ia32@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz#241fe62c34d8e8461cd708277813e1d0ba55ce23" - integrity sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ== - -"@esbuild/win32-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz#9c907b21e30a52db959ba4f80bb01a0cc403d5cc" - integrity sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ== +"@esbuild/aix-ppc64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz#c7184a326533fcdf1b8ee0733e21c713b975575f" + integrity sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ== + +"@esbuild/android-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz#09d9b4357780da9ea3a7dfb833a1f1ff439b4052" + integrity sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A== + +"@esbuild/android-arm@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.21.5.tgz#9b04384fb771926dfa6d7ad04324ecb2ab9b2e28" + integrity sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg== + +"@esbuild/android-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.21.5.tgz#29918ec2db754cedcb6c1b04de8cd6547af6461e" + integrity sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA== + +"@esbuild/darwin-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz#e495b539660e51690f3928af50a76fb0a6ccff2a" + integrity sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ== + +"@esbuild/darwin-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz#c13838fa57372839abdddc91d71542ceea2e1e22" + integrity sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw== + +"@esbuild/freebsd-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz#646b989aa20bf89fd071dd5dbfad69a3542e550e" + integrity sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g== + +"@esbuild/freebsd-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz#aa615cfc80af954d3458906e38ca22c18cf5c261" + integrity sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ== + +"@esbuild/linux-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz#70ac6fa14f5cb7e1f7f887bcffb680ad09922b5b" + integrity sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q== + +"@esbuild/linux-arm@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz#fc6fd11a8aca56c1f6f3894f2bea0479f8f626b9" + integrity sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA== + +"@esbuild/linux-ia32@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz#3271f53b3f93e3d093d518d1649d6d68d346ede2" + integrity sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg== + +"@esbuild/linux-loong64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz#ed62e04238c57026aea831c5a130b73c0f9f26df" + integrity sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg== + +"@esbuild/linux-mips64el@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz#e79b8eb48bf3b106fadec1ac8240fb97b4e64cbe" + integrity sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg== + +"@esbuild/linux-ppc64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz#5f2203860a143b9919d383ef7573521fb154c3e4" + integrity sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w== + +"@esbuild/linux-riscv64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz#07bcafd99322d5af62f618cb9e6a9b7f4bb825dc" + integrity sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA== + +"@esbuild/linux-s390x@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz#b7ccf686751d6a3e44b8627ababc8be3ef62d8de" + integrity sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A== + +"@esbuild/linux-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz#6d8f0c768e070e64309af8004bb94e68ab2bb3b0" + integrity sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ== + +"@esbuild/netbsd-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz#bbe430f60d378ecb88decb219c602667387a6047" + integrity sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg== + +"@esbuild/openbsd-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz#99d1cf2937279560d2104821f5ccce220cb2af70" + integrity sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow== + +"@esbuild/sunos-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz#08741512c10d529566baba837b4fe052c8f3487b" + integrity sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg== + +"@esbuild/win32-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz#675b7385398411240735016144ab2e99a60fc75d" + integrity sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A== + +"@esbuild/win32-ia32@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz#1bfc3ce98aa6ca9a0969e4d2af72144c59c1193b" + integrity sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA== + +"@esbuild/win32-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz#acad351d582d157bb145535db2a6ff53dd514b5c" + integrity sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw== "@eslint-community/eslint-utils@^4.2.0": version "4.4.0" @@ -765,9 +876,18 @@ eslint-visitor-keys "^3.3.0" "@eslint-community/regexpp@^4.6.1": - version "4.10.0" - resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.0.tgz#548f6de556857c8bb73bbee70c35dc82a2e74d63" - integrity sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA== + version "4.10.1" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.1.tgz#361461e5cb3845d874e61731c11cfedd664d83a0" + integrity sha512-Zm2NGpWELsQAD1xsJzGQpYfvICSsFkEpU0jxBjfdC6uNEWXcHnfs9hScFWtXVDVl+rBQJGrl4g1vcKIejpH9dA== + +"@eslint/config-array@^0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@eslint/config-array/-/config-array-0.16.0.tgz#bb3364fc39ee84ec3a62abdc4b8d988d99dfd706" + integrity sha512-/jmuSd74i4Czf1XXn7wGRWZCuyaUZ330NH1Bek0Pplatt4Sy1S5haN21SCLLdbeKslQ+S0wEJ+++v5YibSi+Lg== + dependencies: + "@eslint/object-schema" "^2.1.4" + debug "^4.3.1" + minimatch "^3.0.5" "@eslint/eslintrc@^3.1.0": version "3.1.0" @@ -784,10 +904,15 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@9.3.0": - version "9.3.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.3.0.tgz#2e8f65c9c55227abc4845b1513c69c32c679d8fe" - integrity sha512-niBqk8iwv96+yuTwjM6bWg8ovzAPF9qkICsGtcoa5/dmqcEMfdwNAX7+/OHcJHc7wj7XqPxH98oAHytFYlw6Sw== +"@eslint/js@9.5.0": + version "9.5.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.5.0.tgz#0e9c24a670b8a5c86bff97b40be13d8d8f238045" + integrity sha512-A7+AOT2ICkodvtsWnxZP4Xxk3NbZ3VMHd8oihydLRGrJgqqdEz1qSeEgXYyT/Cu8h1TWWsQRejIx48mtjZ5y1w== + +"@eslint/object-schema@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@eslint/object-schema/-/object-schema-2.1.4.tgz#9e69f8bb4031e11df79e03db09f9dbbae1740843" + integrity sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ== "@floating-ui/core@^1.4.2": version "1.5.0" @@ -852,31 +977,17 @@ "@fortawesome/fontawesome-common-types" "6.5.2" "@fortawesome/react-fontawesome@^0": - version "0.2.1" - resolved "https://registry.yarnpkg.com/@fortawesome/react-fontawesome/-/react-fontawesome-0.2.1.tgz#0bd69e89bd7548b4903ffa989ebf50cc82ba091c" - integrity sha512-ldr5QO2MneAX5W5WBCYB2pZp/PiHDD1hy9YEBLcXUyJb0qnO86oP8RU+CgmYVSH/R4Dbe2ernhcWOrcgaKD9NQ== + version "0.2.2" + resolved "https://registry.yarnpkg.com/@fortawesome/react-fontawesome/-/react-fontawesome-0.2.2.tgz#68b058f9132b46c8599875f6a636dad231af78d4" + integrity sha512-EnkrprPNqI6SXJl//m29hpaNzOp1bruISWaOiRtkMi/xSvHJlzc2j2JAYS7egxt/EbjSNV/k6Xy0AQI6vB2+1g== dependencies: prop-types "^15.8.1" -"@humanwhocodes/config-array@^0.13.0": - version "0.13.0" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.13.0.tgz#fb907624df3256d04b9aa2df50d7aa97ec648748" - integrity sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw== - dependencies: - "@humanwhocodes/object-schema" "^2.0.3" - debug "^4.3.1" - minimatch "^3.0.5" - "@humanwhocodes/module-importer@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== -"@humanwhocodes/object-schema@^2.0.3": - version "2.0.3" - resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz#4a2868d75d6d6963e423bcf90b7fd1be343409d3" - integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== - "@humanwhocodes/retry@^0.3.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.3.0.tgz#6d86b8cb322660f03d3f0aa94b99bdd8e172d570" @@ -1122,7 +1233,7 @@ "@types/yargs" "^17.0.8" chalk "^4.0.0" -"@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2": +"@jridgewell/gen-mapping@^0.3.2": version "0.3.3" resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098" integrity sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ== @@ -1131,15 +1242,24 @@ "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping" "^0.3.9" +"@jridgewell/gen-mapping@^0.3.5": + version "0.3.5" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" + integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== + dependencies: + "@jridgewell/set-array" "^1.2.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.24" + "@jridgewell/resolve-uri@^3.1.0": - version "3.1.1" - resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" - integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== -"@jridgewell/set-array@^1.0.1": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" - integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== +"@jridgewell/set-array@^1.0.1", "@jridgewell/set-array@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" + integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== "@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": version "1.4.15" @@ -1154,7 +1274,7 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9": +"@jridgewell/trace-mapping@^0.3.17": version "0.3.20" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz#72e45707cf240fa6b081d0366f8265b0cd10197f" integrity sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q== @@ -1162,6 +1282,14 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25", "@jridgewell/trace-mapping@^0.3.9": + version "0.3.25" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@mantine/core@^5": version "5.10.5" resolved "https://registry.yarnpkg.com/@mantine/core/-/core-5.10.5.tgz#071e14dcf8b94a36d0243f1f4b30305ac0074afd" @@ -1320,85 +1448,85 @@ resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.16.1.tgz#73db3c48b975eeb06d0006481bde4f5f2d17d1cd" integrity sha512-es2g3dq6Nb07iFxGk5GuHN20RwBZOsuDQN7izWIisUcv9r+d2C5jQxqmgkdebXgReWfiyUabcki6Fg77mSNrig== -"@rollup/rollup-android-arm-eabi@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.17.2.tgz#1a32112822660ee104c5dd3a7c595e26100d4c2d" - integrity sha512-NM0jFxY8bB8QLkoKxIQeObCaDlJKewVlIEkuyYKm5An1tdVZ966w2+MPQ2l8LBZLjR+SgyV+nRkTIunzOYBMLQ== - -"@rollup/rollup-android-arm64@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.17.2.tgz#5aeef206d65ff4db423f3a93f71af91b28662c5b" - integrity sha512-yeX/Usk7daNIVwkq2uGoq2BYJKZY1JfyLTaHO/jaiSwi/lsf8fTFoQW/n6IdAsx5tx+iotu2zCJwz8MxI6D/Bw== - -"@rollup/rollup-darwin-arm64@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.17.2.tgz#6b66aaf003c70454c292cd5f0236ebdc6ffbdf1a" - integrity sha512-kcMLpE6uCwls023+kknm71ug7MZOrtXo+y5p/tsg6jltpDtgQY1Eq5sGfHcQfb+lfuKwhBmEURDga9N0ol4YPw== - -"@rollup/rollup-darwin-x64@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.17.2.tgz#f64fc51ed12b19f883131ccbcea59fc68cbd6c0b" - integrity sha512-AtKwD0VEx0zWkL0ZjixEkp5tbNLzX+FCqGG1SvOu993HnSz4qDI6S4kGzubrEJAljpVkhRSlg5bzpV//E6ysTQ== - -"@rollup/rollup-linux-arm-gnueabihf@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.17.2.tgz#1a7641111be67c10111f7122d1e375d1226cbf14" - integrity sha512-3reX2fUHqN7sffBNqmEyMQVj/CKhIHZd4y631duy0hZqI8Qoqf6lTtmAKvJFYa6bhU95B1D0WgzHkmTg33In0A== - -"@rollup/rollup-linux-arm-musleabihf@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.17.2.tgz#c93fd632923e0fee25aacd2ae414288d0b7455bb" - integrity sha512-uSqpsp91mheRgw96xtyAGP9FW5ChctTFEoXP0r5FAzj/3ZRv3Uxjtc7taRQSaQM/q85KEKjKsZuiZM3GyUivRg== - -"@rollup/rollup-linux-arm64-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.17.2.tgz#fa531425dd21d058a630947527b4612d9d0b4a4a" - integrity sha512-EMMPHkiCRtE8Wdk3Qhtciq6BndLtstqZIroHiiGzB3C5LDJmIZcSzVtLRbwuXuUft1Cnv+9fxuDtDxz3k3EW2A== - -"@rollup/rollup-linux-arm64-musl@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.17.2.tgz#8acc16f095ceea5854caf7b07e73f7d1802ac5af" - integrity sha512-NMPylUUZ1i0z/xJUIx6VUhISZDRT+uTWpBcjdv0/zkp7b/bQDF+NfnfdzuTiB1G6HTodgoFa93hp0O1xl+/UbA== - -"@rollup/rollup-linux-powerpc64le-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.17.2.tgz#94e69a8499b5cf368911b83a44bb230782aeb571" - integrity sha512-T19My13y8uYXPw/L/k0JYaX1fJKFT/PWdXiHr8mTbXWxjVF1t+8Xl31DgBBvEKclw+1b00Chg0hxE2O7bTG7GQ== - -"@rollup/rollup-linux-riscv64-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.17.2.tgz#7ef1c781c7e59e85a6ce261cc95d7f1e0b56db0f" - integrity sha512-BOaNfthf3X3fOWAB+IJ9kxTgPmMqPPH5f5k2DcCsRrBIbWnaJCgX2ll77dV1TdSy9SaXTR5iDXRL8n7AnoP5cg== - -"@rollup/rollup-linux-s390x-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.17.2.tgz#f15775841c3232fca9b78cd25a7a0512c694b354" - integrity sha512-W0UP/x7bnn3xN2eYMql2T/+wpASLE5SjObXILTMPUBDB/Fg/FxC+gX4nvCfPBCbNhz51C+HcqQp2qQ4u25ok6g== - -"@rollup/rollup-linux-x64-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.17.2.tgz#b521d271798d037ad70c9f85dd97d25f8a52e811" - integrity sha512-Hy7pLwByUOuyaFC6mAr7m+oMC+V7qyifzs/nW2OJfC8H4hbCzOX07Ov0VFk/zP3kBsELWNFi7rJtgbKYsav9QQ== - -"@rollup/rollup-linux-x64-musl@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.17.2.tgz#9254019cc4baac35800991315d133cc9fd1bf385" - integrity sha512-h1+yTWeYbRdAyJ/jMiVw0l6fOOm/0D1vNLui9iPuqgRGnXA0u21gAqOyB5iHjlM9MMfNOm9RHCQ7zLIzT0x11Q== - -"@rollup/rollup-win32-arm64-msvc@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.17.2.tgz#27f65a89f6f52ee9426ec11e3571038e4671790f" - integrity sha512-tmdtXMfKAjy5+IQsVtDiCfqbynAQE/TQRpWdVataHmhMb9DCoJxp9vLcCBjEQWMiUYxO1QprH/HbY9ragCEFLA== - -"@rollup/rollup-win32-ia32-msvc@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.17.2.tgz#a2fbf8246ed0bb014f078ca34ae6b377a90cb411" - integrity sha512-7II/QCSTAHuE5vdZaQEwJq2ZACkBpQDOmQsE6D6XUbnBHW8IAhm4eTufL6msLJorzrHDFv3CF8oCA/hSIRuZeQ== - -"@rollup/rollup-win32-x64-msvc@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.17.2.tgz#5a2d08b81e8064b34242d5cc9973ef8dd1e60503" - integrity sha512-TGGO7v7qOq4CYmSBVEYpI1Y5xDuCEnbVC5Vth8mOsW0gDSzxNrVERPc790IGHsrT2dQSimgMr9Ub3Y1Jci5/8w== +"@rollup/rollup-android-arm-eabi@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.0.tgz#bbd0e616b2078cd2d68afc9824d1fadb2f2ffd27" + integrity sha512-Tya6xypR10giZV1XzxmH5wr25VcZSncG0pZIjfePT0OVBvqNEurzValetGNarVrGiq66EBVAFn15iYX4w6FKgQ== + +"@rollup/rollup-android-arm64@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.18.0.tgz#97255ef6384c5f73f4800c0de91f5f6518e21203" + integrity sha512-avCea0RAP03lTsDhEyfy+hpfr85KfyTctMADqHVhLAF3MlIkq83CP8UfAHUssgXTYd+6er6PaAhx/QGv4L1EiA== + +"@rollup/rollup-darwin-arm64@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.18.0.tgz#b6dd74e117510dfe94541646067b0545b42ff096" + integrity sha512-IWfdwU7KDSm07Ty0PuA/W2JYoZ4iTj3TUQjkVsO/6U+4I1jN5lcR71ZEvRh52sDOERdnNhhHU57UITXz5jC1/w== + +"@rollup/rollup-darwin-x64@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.18.0.tgz#e07d76de1cec987673e7f3d48ccb8e106d42c05c" + integrity sha512-n2LMsUz7Ynu7DoQrSQkBf8iNrjOGyPLrdSg802vk6XT3FtsgX6JbE8IHRvposskFm9SNxzkLYGSq9QdpLYpRNA== + +"@rollup/rollup-linux-arm-gnueabihf@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.18.0.tgz#9f1a6d218b560c9d75185af4b8bb42f9f24736b8" + integrity sha512-C/zbRYRXFjWvz9Z4haRxcTdnkPt1BtCkz+7RtBSuNmKzMzp3ZxdM28Mpccn6pt28/UWUCTXa+b0Mx1k3g6NOMA== + +"@rollup/rollup-linux-arm-musleabihf@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.18.0.tgz#53618b92e6ffb642c7b620e6e528446511330549" + integrity sha512-l3m9ewPgjQSXrUMHg93vt0hYCGnrMOcUpTz6FLtbwljo2HluS4zTXFy2571YQbisTnfTKPZ01u/ukJdQTLGh9A== + +"@rollup/rollup-linux-arm64-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.18.0.tgz#99a7ba5e719d4f053761a698f7b52291cefba577" + integrity sha512-rJ5D47d8WD7J+7STKdCUAgmQk49xuFrRi9pZkWoRD1UeSMakbcepWXPF8ycChBoAqs1pb2wzvbY6Q33WmN2ftw== + +"@rollup/rollup-linux-arm64-musl@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.18.0.tgz#f53db99a45d9bc00ce94db8a35efa7c3c144a58c" + integrity sha512-be6Yx37b24ZwxQ+wOQXXLZqpq4jTckJhtGlWGZs68TgdKXJgw54lUUoFYrg6Zs/kjzAQwEwYbp8JxZVzZLRepQ== + +"@rollup/rollup-linux-powerpc64le-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.18.0.tgz#cbb0837408fe081ce3435cf3730e090febafc9bf" + integrity sha512-hNVMQK+qrA9Todu9+wqrXOHxFiD5YmdEi3paj6vP02Kx1hjd2LLYR2eaN7DsEshg09+9uzWi2W18MJDlG0cxJA== + +"@rollup/rollup-linux-riscv64-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.18.0.tgz#8ed09c1d1262ada4c38d791a28ae0fea28b80cc9" + integrity sha512-ROCM7i+m1NfdrsmvwSzoxp9HFtmKGHEqu5NNDiZWQtXLA8S5HBCkVvKAxJ8U+CVctHwV2Gb5VUaK7UAkzhDjlg== + +"@rollup/rollup-linux-s390x-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.18.0.tgz#938138d3c8e0c96f022252a28441dcfb17afd7ec" + integrity sha512-0UyyRHyDN42QL+NbqevXIIUnKA47A+45WyasO+y2bGJ1mhQrfrtXUpTxCOrfxCR4esV3/RLYyucGVPiUsO8xjg== + +"@rollup/rollup-linux-x64-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.18.0.tgz#1a7481137a54740bee1ded4ae5752450f155d942" + integrity sha512-xuglR2rBVHA5UsI8h8UbX4VJ470PtGCf5Vpswh7p2ukaqBGFTnsfzxUBetoWBWymHMxbIG0Cmx7Y9qDZzr648w== + +"@rollup/rollup-linux-x64-musl@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.18.0.tgz#f1186afc601ac4f4fc25fac4ca15ecbee3a1874d" + integrity sha512-LKaqQL9osY/ir2geuLVvRRs+utWUNilzdE90TpyoX0eNqPzWjRm14oMEE+YLve4k/NAqCdPkGYDaDF5Sw+xBfg== + +"@rollup/rollup-win32-arm64-msvc@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.18.0.tgz#ed6603e93636a96203c6915be4117245c1bd2daf" + integrity sha512-7J6TkZQFGo9qBKH0pk2cEVSRhJbL6MtfWxth7Y5YmZs57Pi+4x6c2dStAUvaQkHQLnEQv1jzBUW43GvZW8OFqA== + +"@rollup/rollup-win32-ia32-msvc@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.18.0.tgz#14e0b404b1c25ebe6157a15edb9c46959ba74c54" + integrity sha512-Txjh+IxBPbkUB9+SXZMpv+b/vnTEtFyfWZgJ6iyCmt2tdx0OF5WhFowLmnh8ENGNpfUlUZkdI//4IEmhwPieNg== + +"@rollup/rollup-win32-x64-msvc@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.0.tgz#5d694d345ce36b6ecf657349e03eb87297e68da4" + integrity sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -1442,9 +1570,9 @@ "@types/babel__traverse" "*" "@types/babel__generator@*": - version "7.6.7" - resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.7.tgz#a7aebf15c7bc0eb9abd638bdb5c0b8700399c9d0" - integrity sha512-6Sfsq+EaaLrw4RmdFWE9Onp63TOUue71AWb4Gpa6JxzgTYtimbM086WnYTy2U67AofR++QKCo08ZP6pwx8YFHQ== + version "7.6.8" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.8.tgz#f836c61f48b1346e7d2b0d93c6dacc5b9535d3ab" + integrity sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw== dependencies: "@babel/types" "^7.0.0" @@ -1457,9 +1585,9 @@ "@babel/types" "^7.0.0" "@types/babel__traverse@*": - version "7.20.4" - resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.4.tgz#ec2c06fed6549df8bc0eb4615b683749a4a92e1b" - integrity sha512-mSM/iKUk5fDDrEV/e83qY+Cr3I1+Q3qqTuEn++HAWYjEa1+NxZr6CNrcJGf2ZTnq4HoFGC3zaTPZTobCzCFukA== + version "7.20.6" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.6.tgz#8dc9f0ae0f202c08d8d4dab648912c8d6038e3f7" + integrity sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg== dependencies: "@babel/types" "^7.20.7" @@ -1541,15 +1669,15 @@ "@types/yargs-parser" "*" "@vitejs/plugin-react@^4": - version "4.2.1" - resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.2.1.tgz#744d8e4fcb120fc3dbaa471dadd3483f5a304bb9" - integrity sha512-oojO9IDc4nCUUi8qIR11KoQm0XFFLIwsRBwHRR4d/88IWghn1y6ckz/bJ8GHDCsYEJee8mDzqtJxh15/cisJNQ== + version "4.3.1" + resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.3.1.tgz#d0be6594051ded8957df555ff07a991fb618b48e" + integrity sha512-m/V2syj5CuVnaxcUJOQRel/Wr31FFXRFlnOoq1TVtkCxsY5veGMTEmpWHndrhB2U8ScHtCQB1e+4hWYExQc6Lg== dependencies: - "@babel/core" "^7.23.5" - "@babel/plugin-transform-react-jsx-self" "^7.23.3" - "@babel/plugin-transform-react-jsx-source" "^7.23.3" + "@babel/core" "^7.24.5" + "@babel/plugin-transform-react-jsx-self" "^7.24.5" + "@babel/plugin-transform-react-jsx-source" "^7.24.1" "@types/babel__core" "^7.20.5" - react-refresh "^0.14.0" + react-refresh "^0.14.2" acorn-jsx@^5.3.2: version "5.3.2" @@ -1557,9 +1685,9 @@ acorn-jsx@^5.3.2: integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== acorn@^8.11.3: - version "8.11.3" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a" - integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg== + version "8.12.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.0.tgz#1627bfa2e058148036133b8d9b51a700663c294c" + integrity sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw== ajv@^6.12.4: version "6.12.6" @@ -1660,7 +1788,19 @@ array-buffer-byte-length@^1.0.1: call-bind "^1.0.5" is-array-buffer "^3.0.4" -array-includes@^3.1.6, array-includes@^3.1.7: +array-includes@^3.1.6, array-includes@^3.1.8: + version "3.1.8" + resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.8.tgz#5e370cbe172fdd5dd6530c1d4aadda25281ba97d" + integrity sha512-itaWrbYbqpGXkGhZPGUulwnhVf5Hpy1xiCFsGqyIGglbBxmG5vSjxQen3/WGOjPpNEv1RtBLKxbmVXm8HpJStQ== + dependencies: + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.2" + es-object-atoms "^1.0.0" + get-intrinsic "^1.2.4" + is-string "^1.0.7" + +array-includes@^3.1.7: version "3.1.7" resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.7.tgz#8cd2e01b26f7a3086cbc87271593fe921c62abda" integrity sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ== @@ -1676,15 +1816,16 @@ array-unique@^0.3.2: resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" integrity sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ== -array.prototype.findlast@^1.2.4: - version "1.2.4" - resolved "https://registry.yarnpkg.com/array.prototype.findlast/-/array.prototype.findlast-1.2.4.tgz#eeb9e45fc894055c82e5675c463e8077b827ad36" - integrity sha512-BMtLxpV+8BD+6ZPFIWmnUBpQoy+A+ujcg4rhp2iwCRJYA7PEh2MS4NL3lz8EiDlLrJPp2hg9qWihr5pd//jcGw== +array.prototype.findlast@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/array.prototype.findlast/-/array.prototype.findlast-1.2.5.tgz#3e4fbcb30a15a7f5bf64cf2faae22d139c2e4904" + integrity sha512-CVvd6FHg1Z3POpBLxO6E6zr+rSKEQ9L6rZHAaY7lLfhKsWYUBBOuMs0e9o24oopj6H+geRCX0YJ+TJLBK2eHyQ== dependencies: - call-bind "^1.0.5" + call-bind "^1.0.7" define-properties "^1.2.1" - es-abstract "^1.22.3" + es-abstract "^1.23.2" es-errors "^1.3.0" + es-object-atoms "^1.0.0" es-shim-unscopables "^1.0.2" array.prototype.findlastindex@^1.2.3: @@ -1728,15 +1869,15 @@ array.prototype.toreversed@^1.1.2: es-abstract "^1.22.1" es-shim-unscopables "^1.0.0" -array.prototype.tosorted@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/array.prototype.tosorted/-/array.prototype.tosorted-1.1.3.tgz#c8c89348337e51b8a3c48a9227f9ce93ceedcba8" - integrity sha512-/DdH4TiTmOKzyQbp/eadcCVexiCb36xJg7HshYOYJnNZFDj33GEv0P7GxsynpShhq4OLYJzbGcBDkLsDt7MnNg== +array.prototype.tosorted@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/array.prototype.tosorted/-/array.prototype.tosorted-1.1.4.tgz#fe954678ff53034e717ea3352a03f0b0b86f7ffc" + integrity sha512-p6Fx8B7b7ZhL/gmUsAy0D15WhvDccw3mnGNbZpi3pmeJdxtWsj2jEaI4Y6oo3XiHfzuSgPwKc04MYt6KgvC/wA== dependencies: - call-bind "^1.0.5" + call-bind "^1.0.7" define-properties "^1.2.1" - es-abstract "^1.22.3" - es-errors "^1.1.0" + es-abstract "^1.23.3" + es-errors "^1.3.0" es-shim-unscopables "^1.0.2" arraybuffer.prototype.slice@^1.0.3: @@ -1930,6 +2071,16 @@ browserslist@^4.21.9: node-releases "^2.0.14" update-browserslist-db "^1.0.13" +browserslist@^4.22.2: + version "4.23.1" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.1.tgz#ce4af0534b3d37db5c1a4ca98b9080f985041e96" + integrity sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw== + dependencies: + caniuse-lite "^1.0.30001629" + electron-to-chromium "^1.4.796" + node-releases "^2.0.14" + update-browserslist-db "^1.0.16" + bser@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05" @@ -1988,6 +2139,11 @@ caniuse-lite@^1.0.30001565: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001566.tgz#61a8e17caf3752e3e426d4239c549ebbb37fef0d" integrity sha512-ggIhCsTxmITBAMmK8yZjEhCO5/47jKXPu6Dha/wuCS4JePVL+3uiDEBuhu2aIoT+bqTOR8L76Ip1ARL9xYsEJA== +caniuse-lite@^1.0.30001629: + version "1.0.30001632" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001632.tgz#964207b7cba5851701afb4c8afaf1448db3884b6" + integrity sha512-udx3o7yHJfUxMLkGohMlVHCvFvWmirKh9JAH/d7WOLPetlH+LTL5cocMZ0t7oZx/mdlOWXti97xLZWc8uURRHg== + capture-exit@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/capture-exit/-/capture-exit-2.0.0.tgz#fb953bfaebeb781f62898239dabb426d08a509a4" @@ -2186,7 +2342,7 @@ data-view-buffer@^1.0.1: es-errors "^1.3.0" is-data-view "^1.0.1" -data-view-byte-length@^1.0.0: +data-view-byte-length@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/data-view-byte-length/-/data-view-byte-length-1.0.1.tgz#90721ca95ff280677eb793749fce1011347669e2" integrity sha512-4J7wRJD3ABAzr8wP+OcIcqq2dlUKp4DVflx++hs5h5ZKydWMI6/D/fAot+yh6g2tHh8fLFTvNOaVN357NvSrOQ== @@ -2218,7 +2374,14 @@ debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2: +debug@^4.1.0, debug@^4.3.1, debug@^4.3.2: + version "4.3.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" + integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== + dependencies: + ms "2.1.2" + +debug@^4.1.1: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== @@ -2254,7 +2417,7 @@ define-data-property@^1.0.1, define-data-property@^1.1.4: es-errors "^1.3.0" gopd "^1.0.1" -define-properties@^1.1.3, define-properties@^1.2.0, define-properties@^1.2.1: +define-properties@^1.2.0, define-properties@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.1.tgz#10781cc616eb951a80a034bafcaa7377f6af2b6c" integrity sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg== @@ -2315,6 +2478,11 @@ electron-to-chromium@^1.4.601: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.609.tgz#5790a70aaa96de232501b56e14b64d17aff93988" integrity sha512-ihiCP7PJmjoGNuLpl7TjNA8pCQWu09vGyjlPYw1Rqww4gvNuCcmvl+44G+2QyJ6S2K4o+wbTS++Xz0YN8Q9ERw== +electron-to-chromium@^1.4.796: + version "1.4.796" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.796.tgz#48dd6ff634b7f7df6313bd27aaa713f3af4a2b29" + integrity sha512-NglN/xprcM+SHD2XCli4oC6bWe6kHoytcyLKCWXmRL854F0qhPhaYgUswUsglnPxYaNQIg2uMY4BvaomIf3kLA== + emittery@^0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.13.1.tgz#c04b8c3457490e0847ae51fced3af52d338e3dad" @@ -2339,67 +2507,21 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-abstract@^1.22.1, es-abstract@^1.22.3: - version "1.22.5" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.22.5.tgz#1417df4e97cc55f09bf7e58d1e614bc61cb8df46" - integrity sha512-oW69R+4q2wG+Hc3KZePPZxOiisRIqfKBVo/HLx94QcJeWGU/8sZhCvc829rd1kS366vlJbzBfXf9yWwf0+Ko7w== - dependencies: - array-buffer-byte-length "^1.0.1" - arraybuffer.prototype.slice "^1.0.3" - available-typed-arrays "^1.0.7" - call-bind "^1.0.7" - es-define-property "^1.0.0" - es-errors "^1.3.0" - es-set-tostringtag "^2.0.3" - es-to-primitive "^1.2.1" - function.prototype.name "^1.1.6" - get-intrinsic "^1.2.4" - get-symbol-description "^1.0.2" - globalthis "^1.0.3" - gopd "^1.0.1" - has-property-descriptors "^1.0.2" - has-proto "^1.0.3" - has-symbols "^1.0.3" - hasown "^2.0.1" - internal-slot "^1.0.7" - is-array-buffer "^3.0.4" - is-callable "^1.2.7" - is-negative-zero "^2.0.3" - is-regex "^1.1.4" - is-shared-array-buffer "^1.0.3" - is-string "^1.0.7" - is-typed-array "^1.1.13" - is-weakref "^1.0.2" - object-inspect "^1.13.1" - object-keys "^1.1.1" - object.assign "^4.1.5" - regexp.prototype.flags "^1.5.2" - safe-array-concat "^1.1.0" - safe-regex-test "^1.0.3" - string.prototype.trim "^1.2.8" - string.prototype.trimend "^1.0.7" - string.prototype.trimstart "^1.0.7" - typed-array-buffer "^1.0.2" - typed-array-byte-length "^1.0.1" - typed-array-byte-offset "^1.0.2" - typed-array-length "^1.0.5" - unbox-primitive "^1.0.2" - which-typed-array "^1.1.14" - -es-abstract@^1.23.0: - version "1.23.0" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.23.0.tgz#a575e7bc0a570180c8ecd64a4de43f4e7ba0c767" - integrity sha512-vmuE7Uoevk2xkwu5Gwa7RfJk/ebVV6xRv7KuZNbUglmJHhWPMbLL20ztreVpBbdxBZijETx3Aml3NssX4SFMvQ== +es-abstract@^1.22.1, es-abstract@^1.22.3, es-abstract@^1.23.0, es-abstract@^1.23.1, es-abstract@^1.23.2, es-abstract@^1.23.3: + version "1.23.3" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.23.3.tgz#8f0c5a35cd215312573c5a27c87dfd6c881a0aa0" + integrity sha512-e+HfNH61Bj1X9/jLc5v1owaLYuHdeHHSQlkhCBiTK8rBvKaULl/beGMxwrMXjpYrv4pz22BlY570vVePA2ho4A== dependencies: array-buffer-byte-length "^1.0.1" arraybuffer.prototype.slice "^1.0.3" available-typed-arrays "^1.0.7" call-bind "^1.0.7" data-view-buffer "^1.0.1" - data-view-byte-length "^1.0.0" + data-view-byte-length "^1.0.1" data-view-byte-offset "^1.0.0" es-define-property "^1.0.0" es-errors "^1.3.0" + es-object-atoms "^1.0.0" es-set-tostringtag "^2.0.3" es-to-primitive "^1.2.1" function.prototype.name "^1.1.6" @@ -2410,7 +2532,7 @@ es-abstract@^1.23.0: has-property-descriptors "^1.0.2" has-proto "^1.0.3" has-symbols "^1.0.3" - hasown "^2.0.1" + hasown "^2.0.2" internal-slot "^1.0.7" is-array-buffer "^3.0.4" is-callable "^1.2.7" @@ -2425,17 +2547,17 @@ es-abstract@^1.23.0: object-keys "^1.1.1" object.assign "^4.1.5" regexp.prototype.flags "^1.5.2" - safe-array-concat "^1.1.0" + safe-array-concat "^1.1.2" safe-regex-test "^1.0.3" - string.prototype.trim "^1.2.8" - string.prototype.trimend "^1.0.7" - string.prototype.trimstart "^1.0.7" + string.prototype.trim "^1.2.9" + string.prototype.trimend "^1.0.8" + string.prototype.trimstart "^1.0.8" typed-array-buffer "^1.0.2" typed-array-byte-length "^1.0.1" typed-array-byte-offset "^1.0.2" - typed-array-length "^1.0.5" + typed-array-length "^1.0.6" unbox-primitive "^1.0.2" - which-typed-array "^1.1.14" + which-typed-array "^1.1.15" es-define-property@^1.0.0: version "1.0.0" @@ -2444,19 +2566,19 @@ es-define-property@^1.0.0: dependencies: get-intrinsic "^1.2.4" -es-errors@^1.0.0, es-errors@^1.1.0, es-errors@^1.2.1, es-errors@^1.3.0: +es-errors@^1.2.1, es-errors@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== -es-iterator-helpers@^1.0.17: - version "1.0.18" - resolved "https://registry.yarnpkg.com/es-iterator-helpers/-/es-iterator-helpers-1.0.18.tgz#4d3424f46b24df38d064af6fbbc89274e29ea69d" - integrity sha512-scxAJaewsahbqTYrGKJihhViaM6DDZDDoucfvzNbK0pOren1g/daDQ3IAhzn+1G14rBG7w+i5N+qul60++zlKA== +es-iterator-helpers@^1.0.19: + version "1.0.19" + resolved "https://registry.yarnpkg.com/es-iterator-helpers/-/es-iterator-helpers-1.0.19.tgz#117003d0e5fec237b4b5c08aded722e0c6d50ca8" + integrity sha512-zoMwbCcH5hwUkKJkT8kDIBZSz9I6mVG//+lDCinLCGov4+r7NIy0ld8o03M0cJxl2spVf6ESYVS6/gpIfq1FFw== dependencies: call-bind "^1.0.7" define-properties "^1.2.1" - es-abstract "^1.23.0" + es-abstract "^1.23.3" es-errors "^1.3.0" es-set-tostringtag "^2.0.3" function-bind "^1.1.2" @@ -2469,6 +2591,13 @@ es-iterator-helpers@^1.0.17: iterator.prototype "^1.1.2" safe-array-concat "^1.1.2" +es-object-atoms@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.0.0.tgz#ddb55cd47ac2e240701260bc2a8e31ecb643d941" + integrity sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw== + dependencies: + es-errors "^1.3.0" + es-set-tostringtag@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz#8bb60f0a440c2e4281962428438d58545af39777" @@ -2503,39 +2632,39 @@ esbuild-jest@^0: "@babel/plugin-transform-modules-commonjs" "^7.12.13" babel-jest "^26.6.3" -esbuild@^0.20.1: - version "0.20.2" - resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.20.2.tgz#9d6b2386561766ee6b5a55196c6d766d28c87ea1" - integrity sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g== +esbuild@^0.21.3: + version "0.21.5" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.21.5.tgz#9ca301b120922959b766360d8ac830da0d02997d" + integrity sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw== optionalDependencies: - "@esbuild/aix-ppc64" "0.20.2" - "@esbuild/android-arm" "0.20.2" - "@esbuild/android-arm64" "0.20.2" - "@esbuild/android-x64" "0.20.2" - "@esbuild/darwin-arm64" "0.20.2" - "@esbuild/darwin-x64" "0.20.2" - "@esbuild/freebsd-arm64" "0.20.2" - "@esbuild/freebsd-x64" "0.20.2" - "@esbuild/linux-arm" "0.20.2" - "@esbuild/linux-arm64" "0.20.2" - "@esbuild/linux-ia32" "0.20.2" - "@esbuild/linux-loong64" "0.20.2" - "@esbuild/linux-mips64el" "0.20.2" - "@esbuild/linux-ppc64" "0.20.2" - "@esbuild/linux-riscv64" "0.20.2" - "@esbuild/linux-s390x" "0.20.2" - "@esbuild/linux-x64" "0.20.2" - "@esbuild/netbsd-x64" "0.20.2" - "@esbuild/openbsd-x64" "0.20.2" - "@esbuild/sunos-x64" "0.20.2" - "@esbuild/win32-arm64" "0.20.2" - "@esbuild/win32-ia32" "0.20.2" - "@esbuild/win32-x64" "0.20.2" - -escalade@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" - integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== + "@esbuild/aix-ppc64" "0.21.5" + "@esbuild/android-arm" "0.21.5" + "@esbuild/android-arm64" "0.21.5" + "@esbuild/android-x64" "0.21.5" + "@esbuild/darwin-arm64" "0.21.5" + "@esbuild/darwin-x64" "0.21.5" + "@esbuild/freebsd-arm64" "0.21.5" + "@esbuild/freebsd-x64" "0.21.5" + "@esbuild/linux-arm" "0.21.5" + "@esbuild/linux-arm64" "0.21.5" + "@esbuild/linux-ia32" "0.21.5" + "@esbuild/linux-loong64" "0.21.5" + "@esbuild/linux-mips64el" "0.21.5" + "@esbuild/linux-ppc64" "0.21.5" + "@esbuild/linux-riscv64" "0.21.5" + "@esbuild/linux-s390x" "0.21.5" + "@esbuild/linux-x64" "0.21.5" + "@esbuild/netbsd-x64" "0.21.5" + "@esbuild/openbsd-x64" "0.21.5" + "@esbuild/sunos-x64" "0.21.5" + "@esbuild/win32-arm64" "0.21.5" + "@esbuild/win32-ia32" "0.21.5" + "@esbuild/win32-x64" "0.21.5" + +escalade@^3.1.1, escalade@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" + integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== escape-string-regexp@^1.0.5: version "1.0.5" @@ -2610,33 +2739,33 @@ eslint-plugin-react-perf@^3: integrity sha512-boVn4IDHAjgGoAuAQ5Zrewt8fNbMDdiR7B2AkuiSK8lrJ9FwlOZc085kCs7+8u6B+YZ+pOn+tYG00xktnGAfOw== eslint-plugin-react@^7: - version "7.34.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.34.1.tgz#6806b70c97796f5bbfb235a5d3379ece5f4da997" - integrity sha512-N97CxlouPT1AHt8Jn0mhhN2RrADlUAsk1/atcT2KyA/l9Q/E6ll7OIGwNumFmWfZ9skV3XXccYS19h80rHtgkw== + version "7.34.3" + resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.34.3.tgz#9965f27bd1250a787b5d4cfcc765e5a5d58dcb7b" + integrity sha512-aoW4MV891jkUulwDApQbPYTVZmeuSyFrudpbTAQuj5Fv8VL+o6df2xIGpw8B0hPjAaih1/Fb0om9grCdyFYemA== dependencies: - array-includes "^3.1.7" - array.prototype.findlast "^1.2.4" + array-includes "^3.1.8" + array.prototype.findlast "^1.2.5" array.prototype.flatmap "^1.3.2" array.prototype.toreversed "^1.1.2" - array.prototype.tosorted "^1.1.3" + array.prototype.tosorted "^1.1.4" doctrine "^2.1.0" - es-iterator-helpers "^1.0.17" + es-iterator-helpers "^1.0.19" estraverse "^5.3.0" jsx-ast-utils "^2.4.1 || ^3.0.0" minimatch "^3.1.2" - object.entries "^1.1.7" - object.fromentries "^2.0.7" - object.hasown "^1.1.3" - object.values "^1.1.7" + object.entries "^1.1.8" + object.fromentries "^2.0.8" + object.hasown "^1.1.4" + object.values "^1.2.0" prop-types "^15.8.1" resolve "^2.0.0-next.5" semver "^6.3.1" - string.prototype.matchall "^4.0.10" + string.prototype.matchall "^4.0.11" -eslint-plugin-unused-imports@^3: - version "3.2.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-3.2.0.tgz#63a98c9ad5f622cd9f830f70bc77739f25ccfe0d" - integrity sha512-6uXyn6xdINEpxE1MtDjxQsyXB37lfyO2yKGVVgtD7WEWQGORSOZjgrD6hBhvGv4/SO+TOlS+UnC6JppRqbuwGQ== +eslint-plugin-unused-imports@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-4.0.0.tgz#93f3a7ee6088221e4a1d7127866e05d5917a9f65" + integrity sha512-mzM+y2B7XYpQryVa1usT+Y/BdNAtAZiXzwpSyDCboFoJN/LZRN67TNvQxKtuTK/Aplya3sLNQforiubzPPaIcQ== dependencies: eslint-rule-composer "^0.3.0" @@ -2664,15 +2793,15 @@ eslint-visitor-keys@^4.0.0: integrity sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw== eslint@^9.0.0: - version "9.3.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.3.0.tgz#36a96db84592618d6ed9074d677e92f4e58c08b9" - integrity sha512-5Iv4CsZW030lpUqHBapdPo3MJetAPtejVW8B84GIcIIv8+ohFaddXsrn1Gn8uD9ijDb+kcYKFUVmC8qG8B2ORQ== + version "9.5.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.5.0.tgz#11856034b94a9e1a02cfcc7e96a9f0956963cd2f" + integrity sha512-+NAOZFrW/jFTS3dASCGBxX1pkFD0/fsO+hfAkJ4TyYKwgsXZbqzrw+seCYFCcPCYXvnD67tAnglU7GQTz6kcVw== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" + "@eslint/config-array" "^0.16.0" "@eslint/eslintrc" "^3.1.0" - "@eslint/js" "9.3.0" - "@humanwhocodes/config-array" "^0.13.0" + "@eslint/js" "9.5.0" "@humanwhocodes/module-importer" "^1.0.1" "@humanwhocodes/retry" "^0.3.0" "@nodelib/fs.walk" "^1.2.8" @@ -2684,7 +2813,7 @@ eslint@^9.0.0: eslint-scope "^8.0.1" eslint-visitor-keys "^4.0.0" espree "^10.0.1" - esquery "^1.4.2" + esquery "^1.5.0" esutils "^2.0.2" fast-deep-equal "^3.1.3" file-entry-cache "^8.0.0" @@ -2717,7 +2846,7 @@ esprima@^4.0.0: resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== -esquery@^1.4.2: +esquery@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.5.0.tgz#6ce17738de8577694edd7361c57182ac8cb0db0b" integrity sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg== @@ -3060,11 +3189,12 @@ globals@^14.0.0: integrity sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ== globalthis@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/globalthis/-/globalthis-1.0.3.tgz#5852882a52b80dc301b0660273e1ed082f0b6ccf" - integrity sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA== + version "1.0.4" + resolved "https://registry.yarnpkg.com/globalthis/-/globalthis-1.0.4.tgz#7430ed3a975d97bfb59bcce41f5cabbafa651236" + integrity sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ== dependencies: - define-properties "^1.1.3" + define-properties "^1.2.1" + gopd "^1.0.1" gopd@^1.0.1: version "1.0.1" @@ -3148,7 +3278,7 @@ has-values@^1.0.0: is-number "^3.0.0" kind-of "^4.0.0" -hasown@^2.0.0, hasown@^2.0.1: +hasown@^2.0.0, hasown@^2.0.1, hasown@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== @@ -3221,7 +3351,7 @@ inherits@2: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -internal-slot@^1.0.5, internal-slot@^1.0.7: +internal-slot@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.7.tgz#c06dcca3ed874249881007b0a5523b172a190802" integrity sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g== @@ -4401,14 +4531,14 @@ object.assign@^4.1.4, object.assign@^4.1.5: has-symbols "^1.0.3" object-keys "^1.1.1" -object.entries@^1.1.7: - version "1.1.7" - resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.7.tgz#2b47760e2a2e3a752f39dd874655c61a7f03c131" - integrity sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA== +object.entries@^1.1.8: + version "1.1.8" + resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.8.tgz#bffe6f282e01f4d17807204a24f8edd823599c41" + integrity sha512-cmopxi8VwRIAw/fkijJohSfpef5PdN0pMQJN6VC/ZKvn0LIknWD8KtgY6KlQdEc4tIjcQ3HxSMmnvtzIscdaYQ== dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-object-atoms "^1.0.0" object.fromentries@^2.0.7: version "2.0.7" @@ -4419,6 +4549,16 @@ object.fromentries@^2.0.7: define-properties "^1.2.0" es-abstract "^1.22.1" +object.fromentries@^2.0.8: + version "2.0.8" + resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.8.tgz#f7195d8a9b97bd95cbc1999ea939ecd1a2b00c65" + integrity sha512-k6E21FzySsSK5a21KRADBd/NGneRegFO5pLHfdQLpRDETUNJueLXs3WCzyQ3tFRDYgbq3KHGXfTbi2bs8WQ6rQ== + dependencies: + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.2" + es-object-atoms "^1.0.0" + object.groupby@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.1.tgz#d41d9f3c8d6c778d9cbac86b4ee9f5af103152ee" @@ -4429,13 +4569,14 @@ object.groupby@^1.0.1: es-abstract "^1.22.1" get-intrinsic "^1.2.1" -object.hasown@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/object.hasown/-/object.hasown-1.1.3.tgz#6a5f2897bb4d3668b8e79364f98ccf971bda55ae" - integrity sha512-fFI4VcYpRHvSLXxP7yiZOMAd331cPfd2p7PFDVbgUsYOfCT3tICVqXWngbjr4m49OvsBwUBQ6O2uQoJvy3RexA== +object.hasown@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/object.hasown/-/object.hasown-1.1.4.tgz#e270ae377e4c120cdcb7656ce66884a6218283dc" + integrity sha512-FZ9LZt9/RHzGySlBARE3VF+gE26TxR38SdmqOqliuTnl9wrKulaQs+4dee1V+Io8VfxqzAfHu6YuRgUy8OHoTg== dependencies: - define-properties "^1.2.0" - es-abstract "^1.22.1" + define-properties "^1.2.1" + es-abstract "^1.23.2" + es-object-atoms "^1.0.0" object.pick@^1.3.0: version "1.3.0" @@ -4444,7 +4585,16 @@ object.pick@^1.3.0: dependencies: isobject "^3.0.1" -object.values@^1.1.6, object.values@^1.1.7: +object.values@^1.1.6, object.values@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.2.0.tgz#65405a9d92cee68ac2d303002e0b8470a4d9ab1b" + integrity sha512-yBYjY9QX2hnRmZHAjG/f13MzmBzxzYgQhFrke06TTyKY5zSTEqkOeukBzIdVA3j3ulu8Qa3MbVFShV7T2RmGtQ== + dependencies: + call-bind "^1.0.7" + define-properties "^1.2.1" + es-object-atoms "^1.0.0" + +object.values@^1.1.7: version "1.1.7" resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.7.tgz#617ed13272e7e1071b43973aa1655d9291b8442a" integrity sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng== @@ -4569,10 +4719,10 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== -picocolors@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" - integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== +picocolors@^1.0.0, picocolors@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" + integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== picomatch@^2.0.4, picomatch@^2.2.3, picomatch@^2.3.1: version "2.3.1" @@ -4628,9 +4778,9 @@ prettier-linter-helpers@^1.0.0: fast-diff "^1.1.2" prettier@3: - version "3.2.5" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.5.tgz#e52bc3090586e824964a8813b09aba6233b28368" - integrity sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A== + version "3.3.2" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.3.2.tgz#03ff86dc7c835f2d2559ee76876a3914cec4a90a" + integrity sha512-rAVeHYMcv8ATV5d508CFdn+8/pHPpXeIid1DdrPwXnaAdH7cqjVbpJaT5eq4yRAFU/lsbwYwSF/n5iNrdJHPQA== pretty-format@^29.7.0: version "29.7.0" @@ -4712,10 +4862,10 @@ react-is@^18.0.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== -react-refresh@^0.14.0: - version "0.14.0" - resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.14.0.tgz#4e02825378a5f227079554d4284889354e5f553e" - integrity sha512-wViHqhAd8OHeLS/IRMJjTSDHF3U9eWi62F/MledQGPdJGDhodXJ9PBLNGr6WWL7qlH12Mt3TyTpbS+hGXMjCzQ== +react-refresh@^0.14.2: + version "0.14.2" + resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.14.2.tgz#3833da01ce32da470f1f936b9d477da5c7028bf9" + integrity sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA== react-router-dom@^6: version "6.23.1" @@ -4759,15 +4909,15 @@ react@^18: loose-envify "^1.1.0" reflect.getprototypeof@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.5.tgz#e0bd28b597518f16edaf9c0e292c631eb13e0674" - integrity sha512-62wgfC8dJWrmxv44CA36pLDnP6KKl3Vhxb7PL+8+qrrFMMoJij4vgiMP8zV4O8+CBMXY1mHxI5fITGHXFHVmQQ== + version "1.0.6" + resolved "https://registry.yarnpkg.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz#3ab04c32a8390b770712b7a8633972702d278859" + integrity sha512-fmfw4XgoDke3kdI6h4xcUz1dG8uaiv5q9gcEwLS4Pnth2kxT+GZ7YehS1JTMGBQmtV7Y4GFGbs2re2NqhdozUg== dependencies: - call-bind "^1.0.5" + call-bind "^1.0.7" define-properties "^1.2.1" - es-abstract "^1.22.3" - es-errors "^1.0.0" - get-intrinsic "^1.2.3" + es-abstract "^1.23.1" + es-errors "^1.3.0" + get-intrinsic "^1.2.4" globalthis "^1.0.3" which-builtin-type "^1.1.3" @@ -4784,7 +4934,7 @@ regex-not@^1.0.0, regex-not@^1.0.2: extend-shallow "^3.0.2" safe-regex "^1.1.0" -regexp.prototype.flags@^1.5.0, regexp.prototype.flags@^1.5.2: +regexp.prototype.flags@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz#138f644a3350f981a858c44f6bb1a61ff59be334" integrity sha512-NcDiDkTLuPR+++OCKB0nWafEmhg/Da8aUPLPMQbK+bxKKCm1/S5he+AqYa4PlMCVBalb4/yxIRub6qkEx5yJbw== @@ -4879,28 +5029,28 @@ reusify@^1.0.4: integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== rollup@^4.13.0: - version "4.17.2" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.17.2.tgz#26d1785d0144122277fdb20ab3a24729ae68301f" - integrity sha512-/9ClTJPByC0U4zNLowV1tMBe8yMEAxewtR3cUNX5BoEpGH3dQEWpJLr6CLp0fPdYRF/fzVOgvDb1zXuakwF5kQ== + version "4.18.0" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.18.0.tgz#497f60f0c5308e4602cf41136339fbf87d5f5dda" + integrity sha512-QmJz14PX3rzbJCN1SG4Xe/bAAX2a6NpCP8ab2vfu2GiUr8AQcr2nCV/oEO3yneFarB67zk8ShlIyWb2LGTb3Sg== dependencies: "@types/estree" "1.0.5" optionalDependencies: - "@rollup/rollup-android-arm-eabi" "4.17.2" - "@rollup/rollup-android-arm64" "4.17.2" - "@rollup/rollup-darwin-arm64" "4.17.2" - "@rollup/rollup-darwin-x64" "4.17.2" - "@rollup/rollup-linux-arm-gnueabihf" "4.17.2" - "@rollup/rollup-linux-arm-musleabihf" "4.17.2" - "@rollup/rollup-linux-arm64-gnu" "4.17.2" - "@rollup/rollup-linux-arm64-musl" "4.17.2" - "@rollup/rollup-linux-powerpc64le-gnu" "4.17.2" - "@rollup/rollup-linux-riscv64-gnu" "4.17.2" - "@rollup/rollup-linux-s390x-gnu" "4.17.2" - "@rollup/rollup-linux-x64-gnu" "4.17.2" - "@rollup/rollup-linux-x64-musl" "4.17.2" - "@rollup/rollup-win32-arm64-msvc" "4.17.2" - "@rollup/rollup-win32-ia32-msvc" "4.17.2" - "@rollup/rollup-win32-x64-msvc" "4.17.2" + "@rollup/rollup-android-arm-eabi" "4.18.0" + "@rollup/rollup-android-arm64" "4.18.0" + "@rollup/rollup-darwin-arm64" "4.18.0" + "@rollup/rollup-darwin-x64" "4.18.0" + "@rollup/rollup-linux-arm-gnueabihf" "4.18.0" + "@rollup/rollup-linux-arm-musleabihf" "4.18.0" + "@rollup/rollup-linux-arm64-gnu" "4.18.0" + "@rollup/rollup-linux-arm64-musl" "4.18.0" + "@rollup/rollup-linux-powerpc64le-gnu" "4.18.0" + "@rollup/rollup-linux-riscv64-gnu" "4.18.0" + "@rollup/rollup-linux-s390x-gnu" "4.18.0" + "@rollup/rollup-linux-x64-gnu" "4.18.0" + "@rollup/rollup-linux-x64-musl" "4.18.0" + "@rollup/rollup-win32-arm64-msvc" "4.18.0" + "@rollup/rollup-win32-ia32-msvc" "4.18.0" + "@rollup/rollup-win32-x64-msvc" "4.18.0" fsevents "~2.3.2" rsvp@^4.8.4: @@ -4915,7 +5065,7 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -safe-array-concat@^1.1.0, safe-array-concat@^1.1.2: +safe-array-concat@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.2.tgz#81d77ee0c4e8b863635227c721278dd524c20edb" integrity sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q== @@ -4992,7 +5142,7 @@ set-function-length@^1.2.1: gopd "^1.0.1" has-property-descriptors "^1.0.2" -set-function-name@^2.0.0, set-function-name@^2.0.1: +set-function-name@^2.0.1, set-function-name@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.2.tgz#16a705c5a0dc2f5e638ca96d8a8cd4e1c2b90985" integrity sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ== @@ -5036,7 +5186,7 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -side-channel@^1.0.4: +side-channel@^1.0.4, side-channel@^1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2" integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA== @@ -5174,47 +5324,51 @@ string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string.prototype.matchall@^4.0.10: - version "4.0.10" - resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.10.tgz#a1553eb532221d4180c51581d6072cd65d1ee100" - integrity sha512-rGXbGmOEosIQi6Qva94HUjgPs9vKW+dkG7Y8Q5O2OYkWL6wFaTRZO8zM4mhP94uX55wgyrXzfS2aGtGzUL7EJQ== +string.prototype.matchall@^4.0.11: + version "4.0.11" + resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.11.tgz#1092a72c59268d2abaad76582dccc687c0297e0a" + integrity sha512-NUdh0aDavY2og7IbBPenWqR9exH+E26Sv8e0/eTe1tltDGZL+GtBkDAnnyBtmekfK6/Dq3MkcGtzXFEd1LQrtg== dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - get-intrinsic "^1.2.1" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.2" + es-errors "^1.3.0" + es-object-atoms "^1.0.0" + get-intrinsic "^1.2.4" + gopd "^1.0.1" has-symbols "^1.0.3" - internal-slot "^1.0.5" - regexp.prototype.flags "^1.5.0" - set-function-name "^2.0.0" - side-channel "^1.0.4" + internal-slot "^1.0.7" + regexp.prototype.flags "^1.5.2" + set-function-name "^2.0.2" + side-channel "^1.0.6" -string.prototype.trim@^1.2.8: - version "1.2.8" - resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz#f9ac6f8af4bd55ddfa8895e6aea92a96395393bd" - integrity sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ== +string.prototype.trim@^1.2.9: + version "1.2.9" + resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.9.tgz#b6fa326d72d2c78b6df02f7759c73f8f6274faa4" + integrity sha512-klHuCNxiMZ8MlsOihJhJEBJAiMVqU3Z2nEXWfWnIqjN0gEFS9J9+IxKozWWtQGcgoa1WUZzLjKPTr4ZHNFTFxw== dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.0" + es-object-atoms "^1.0.0" -string.prototype.trimend@^1.0.7: - version "1.0.7" - resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.7.tgz#1bb3afc5008661d73e2dc015cd4853732d6c471e" - integrity sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA== +string.prototype.trimend@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.8.tgz#3651b8513719e8a9f48de7f2f77640b26652b229" + integrity sha512-p73uL5VCHCO2BZZ6krwwQE3kCzM7NKmis8S//xEC6fQonchbum4eP6kR4DLEjQFO3Wnj3Fuo8NM0kOSjVdHjZQ== dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-object-atoms "^1.0.0" -string.prototype.trimstart@^1.0.7: - version "1.0.7" - resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.7.tgz#d4cdb44b83a4737ffbac2d406e405d43d0184298" - integrity sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg== +string.prototype.trimstart@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.8.tgz#7ee834dda8c7c17eff3118472bb35bfedaa34dde" + integrity sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg== dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-object-atoms "^1.0.0" strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" @@ -5412,10 +5566,10 @@ typed-array-byte-offset@^1.0.2: has-proto "^1.0.3" is-typed-array "^1.1.13" -typed-array-length@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.5.tgz#57d44da160296d8663fd63180a1802ebf25905d5" - integrity sha512-yMi0PlwuznKHxKmcpoOdeLwxBoVPkqZxd7q2FgMkmD3bNwvF5VW0+UlUQ1k1vmktTu4Yu13Q0RIxEP8+B+wloA== +typed-array-length@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.6.tgz#57155207c76e64a3457482dfdc1c9d1d3c4c73a3" + integrity sha512-/OxDN6OtAk5KBpGb28T+HZc2M+ADtvRxXrKKbUwtsLgdoxgX13hyy7ek6bFRl5+aBs2yZzB0c4CnQfAtVypW/g== dependencies: call-bind "^1.0.7" for-each "^0.3.3" @@ -5459,13 +5613,13 @@ unset-value@^1.0.0: has-value "^0.3.1" isobject "^3.0.0" -update-browserslist-db@^1.0.13: - version "1.0.13" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" - integrity sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg== +update-browserslist-db@^1.0.13, update-browserslist-db@^1.0.16: + version "1.0.16" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz#f6d489ed90fb2f07d67784eb3f53d7891f736356" + integrity sha512-KVbTxlBYlckhF5wgfyZXTWnMn7MMZjMu9XG8bPlliUOP9ThaF4QnhP8qrjrH7DRzHfSk0oQv1wToW+iA5GajEQ== dependencies: - escalade "^3.1.1" - picocolors "^1.0.0" + escalade "^3.1.2" + picocolors "^1.0.1" uri-js@^4.2.2: version "4.4.1" @@ -5516,11 +5670,11 @@ v8-to-istanbul@^9.0.1: convert-source-map "^1.6.0" vite@^5.0.5: - version "5.2.11" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.11.tgz#726ec05555431735853417c3c0bfb36003ca0cbd" - integrity sha512-HndV31LWW05i1BLPMUCE1B9E9GFbOu1MbenhS58FuK6owSO5qHm7GiCotrNY1YE5rMeQSFBGmT5ZaLEjFizgiQ== + version "5.3.1" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.1.tgz#bb2ca6b5fd7483249d3e86b25026e27ba8a663e6" + integrity sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ== dependencies: - esbuild "^0.20.1" + esbuild "^0.21.3" postcss "^8.4.38" rollup "^4.13.0" optionalDependencies: @@ -5572,7 +5726,7 @@ which-collection@^1.0.1: is-weakmap "^2.0.2" is-weakset "^2.0.3" -which-typed-array@^1.1.14, which-typed-array@^1.1.9: +which-typed-array@^1.1.14, which-typed-array@^1.1.15, which-typed-array@^1.1.9: version "1.1.15" resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.15.tgz#264859e9b11a649b388bfaaf4f767df1f779b38d" integrity sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA== diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 3f7214c31e2..5cffa4957c6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -139,7 +139,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta public static FleetController create(FleetControllerOptions options, MetricReporter metricReporter) throws Exception { var context = new FleetControllerContextImpl(options); var timer = new RealTimer(); - var metricUpdater = new MetricUpdater(metricReporter, options.fleetControllerIndex(), options.clusterName()); + var metricUpdater = new MetricUpdater(metricReporter, timer, options.fleetControllerIndex(), options.clusterName()); var log = new EventLog(timer, metricUpdater); var cluster = new ContentCluster(options); var stateGatherer = new NodeStateGatherer(timer, timer, log); @@ -348,7 +348,8 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta ClusterState baselineState = stateBundle.getBaselineClusterState(); newStates.add(stateBundle); metricUpdater.updateClusterStateMetrics(cluster, baselineState, - ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit(), stateBundle.getFeedBlock())); + ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit(), stateBundle.getFeedBlock()), + systemStateBroadcaster.getLastStateBroadcastTimePoint()); lastMetricUpdateCycleCount = cycleCount; systemStateBroadcaster.handleNewClusterStates(stateBundle); // Iff master, always store new version in ZooKeeper _before_ publishing to any @@ -365,12 +366,20 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta private boolean maybePublishOldMetrics() { verifyInControllerThread(); - if (isMaster() && cycleCount > 300 + lastMetricUpdateCycleCount) { - ClusterStateBundle stateBundle = stateVersionTracker.getVersionedClusterStateBundle(); - ClusterState baselineState = stateBundle.getBaselineClusterState(); - metricUpdater.updateClusterStateMetrics(cluster, baselineState, - ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit(), stateBundle.getFeedBlock())); - lastMetricUpdateCycleCount = cycleCount; + if (cycleCount > 300 + lastMetricUpdateCycleCount) { + if (isMaster()) { + updateMasterClusterSyncMetrics(); + ClusterStateBundle stateBundle = stateVersionTracker.getVersionedClusterStateBundle(); + ClusterState baselineState = stateBundle.getBaselineClusterState(); + metricUpdater.updateClusterStateMetrics(cluster, baselineState, + ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit(), stateBundle.getFeedBlock()), + systemStateBroadcaster.getLastStateBroadcastTimePoint()); + lastMetricUpdateCycleCount = cycleCount; + } else { + // If we're not the master we don't have any authoritative information about + // how out of sync the cluster nodes are, so reset the metric. + metricUpdater.updateClusterBucketsOutOfSyncRatio(0); + } return true; } else { return false; @@ -542,7 +551,6 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta didWork |= metricUpdater.forWork("processNextQueuedRemoteTask", this::processNextQueuedRemoteTask); didWork |= metricUpdater.forWork("completeSatisfiedVersionDependentTasks", this::completeSatisfiedVersionDependentTasks); didWork |= metricUpdater.forWork("maybePublishOldMetrics", this::maybePublishOldMetrics); - updateClusterSyncMetrics(); processingCycle = false; ++cycleCount; @@ -564,7 +572,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } } - private void updateClusterSyncMetrics() { + private void updateMasterClusterSyncMetrics() { var stats = stateVersionTracker.getAggregatedClusterStats().getAggregatedStats(); if (stats.hasUpdatesFromAllDistributors()) { GlobalBucketSyncStatsCalculator.clusterBucketsOutOfSyncRatio(stats.getGlobalStats()) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java index d149d4043e4..d72ede7199e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.clustercontroller.utils.util.ComponentMetricReporter; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; import java.time.Duration; +import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.SortedSet; @@ -19,17 +20,27 @@ import java.util.function.BooleanSupplier; public class MetricUpdater { private final ComponentMetricReporter metricReporter; + private final Timer timer; + // Publishing and converging on a cluster state version is never instant nor atomic, but + // it usually completes within a few seconds. If convergence does not happen for more than + // 30 seconds, it's a sign something has stalled. + private Duration stateVersionConvergenceGracePeriod = Duration.ofSeconds(30); - public MetricUpdater(MetricReporter metricReporter, int controllerIndex, String clusterName) { + public MetricUpdater(MetricReporter metricReporter, Timer timer, int controllerIndex, String clusterName) { this.metricReporter = new ComponentMetricReporter(metricReporter, "cluster-controller."); this.metricReporter.addDimension("controller-index", String.valueOf(controllerIndex)); this.metricReporter.addDimension("clusterid", clusterName); + this.timer = timer; } public MetricReporter.Context createContext(Map<String, String> dimensions) { return metricReporter.createContext(dimensions); } + public void setStateVersionConvergenceGracePeriod(Duration gracePeriod) { + stateVersionConvergenceGracePeriod = gracePeriod; + } + private static int nodesInAvailableState(Map<State, Integer> nodeCounts) { return nodeCounts.getOrDefault(State.INITIALIZING, 0) + nodeCounts.getOrDefault(State.RETIRED, 0) @@ -39,10 +50,13 @@ public class MetricUpdater { + nodeCounts.getOrDefault(State.MAINTENANCE, 0); } - public void updateClusterStateMetrics(ContentCluster cluster, ClusterState state, ResourceUsageStats resourceUsage) { + public void updateClusterStateMetrics(ContentCluster cluster, ClusterState state, + ResourceUsageStats resourceUsage, Instant lastStateBroadcastTimePoint) { Map<String, String> dimensions = new HashMap<>(); dimensions.put("cluster", cluster.getName()); dimensions.put("clusterid", cluster.getName()); + Instant now = timer.getCurrentWallClockTime(); + boolean convergenceDeadlinePassed = lastStateBroadcastTimePoint.plus(stateVersionConvergenceGracePeriod).isBefore(now); for (NodeType type : NodeType.getTypes()) { dimensions.put("node-type", type.toString().toLowerCase()); MetricReporter.Context context = createContext(dimensions); @@ -50,10 +64,18 @@ public class MetricUpdater { for (State s : State.values()) { nodeCounts.put(s, 0); } + int nodesNotConverged = 0; for (Integer i : cluster.getConfiguredNodes().keySet()) { - NodeState s = state.getNodeState(new Node(type, i)); + var node = new Node(type, i); + NodeState s = state.getNodeState(node); Integer count = nodeCounts.get(s.getState()); nodeCounts.put(s.getState(), count + 1); + var info = cluster.getNodeInfo(node); + if (info != null && convergenceDeadlinePassed && s.getState().oneOf("uir")) { + if (info.getClusterStateVersionBundleAcknowledged() != state.getVersion()) { + nodesNotConverged++; + } + } } for (State s : State.values()) { String name = s.toString().toLowerCase() + ".count"; @@ -63,6 +85,7 @@ public class MetricUpdater { final int availableNodes = nodesInAvailableState(nodeCounts); final int totalNodes = Math.max(cluster.getConfiguredNodes().size(), 1); // Assumes 1-1 between distributor and storage metricReporter.set("available-nodes.ratio", (double)availableNodes / totalNodes, context); + metricReporter.set("nodes-not-converged", nodesNotConverged, context); } dimensions.remove("node-type"); MetricReporter.Context context = createContext(dimensions); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RealTimer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RealTimer.java index b4563c09b66..482b40381df 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RealTimer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RealTimer.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import java.time.Instant; import java.util.Calendar; import java.util.Locale; import java.util.TimeZone; @@ -10,8 +11,9 @@ import java.util.TimeZone; */ public class RealTimer implements Timer { - public long getCurrentTimeInMillis() { - return System.currentTimeMillis(); + @Override + public Instant getCurrentWallClockTime() { + return Instant.now(); } public static String printDuration(long time) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index c74a846fe30..0337e187b8e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -8,6 +8,7 @@ import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; +import java.time.Instant; import java.util.logging.Level; import java.util.LinkedList; import java.util.List; @@ -29,6 +30,7 @@ public class SystemStateBroadcaster { private final static long minTimeBetweenNodeErrorLogging = 10 * 60 * 1000; private final Map<Node, Long> lastErrorReported = new TreeMap<>(); + private Instant lastStateBroadcastTimePoint = Instant.EPOCH; private int lastOfficialStateVersion = -1; private int lastStateVersionBundleAcked = 0; private int lastClusterStateVersionConverged = 0; @@ -45,6 +47,7 @@ public class SystemStateBroadcaster { public void handleNewClusterStates(ClusterStateBundle state) { clusterStateBundle = state; + lastStateBroadcastTimePoint = Instant.ofEpochMilli(timer.getCurrentTimeInMillis()); } public ClusterState getClusterState() { @@ -67,6 +70,10 @@ public class SystemStateBroadcaster { return lastClusterStateBundleConverged; } + public Instant getLastStateBroadcastTimePoint() { + return lastStateBroadcastTimePoint; + } + private void reportNodeError(boolean nodeOk, NodeInfo info, String message) { long time = timer.getCurrentTimeInMillis(); Long lastReported = lastErrorReported.get(info.getNode()); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/Timer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/Timer.java index 6c7da15b1a5..eaa60b3d675 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/Timer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/Timer.java @@ -1,12 +1,18 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import java.time.Instant; + /** * Interface used to get time. This is separated into its own class, such that unit tests can fake timing to do timing related * tests without relying on the speed of the unit test processing. */ public interface Timer { - long getCurrentTimeInMillis(); + Instant getCurrentWallClockTime(); + + default long getCurrentTimeInMillis() { + return getCurrentWallClockTime().toEpochMilli(); + } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java index 51bda17860e..89095e268cb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.clustercontroller.core.ClusterStateHistoryEntry; import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.EventLog; import com.yahoo.vespa.clustercontroller.core.FleetControllerOptions; +import com.yahoo.vespa.clustercontroller.core.GlobalBucketSyncStatsCalculator; import com.yahoo.vespa.clustercontroller.core.LeafGroups; import com.yahoo.vespa.clustercontroller.core.MasterElectionHandler; import com.yahoo.vespa.clustercontroller.core.NodeInfo; @@ -174,11 +175,8 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa VdsClusterHtmlRenderer.Table table = renderer.createNewClusterHtmlTable(cluster.getName(), cluster.getSlobrokGenerationCount()); ClusterStateBundle state = stateVersionTracker.getVersionedClusterStateBundle(); - if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null - table.appendRaw("<h3 style=\"color: red\">Cluster feeding is blocked!</h3>\n"); - table.appendRaw(String.format("<p>Summary: <strong>%s</strong></p>\n", - HtmlTable.escape(state.getFeedBlockOrNull().getDescription()))); - } + renderClusterFeedBlockIfPresent(state, table); + renderClusterOutOfSyncRatio(state, stateVersionTracker, table); List<Group> groups = LeafGroups.enumerateFrom(cluster.getDistribution().getRootGroup()); for (Group group : groups) { @@ -206,6 +204,53 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa table.addTable(sb, options.stableStateTimePeriod()); } + private static void renderClusterFeedBlockIfPresent(ClusterStateBundle state, VdsClusterHtmlRenderer.Table table) { + if (state.clusterFeedIsBlocked()) { // Implies FeedBlock != null + table.appendRaw("<h3 style=\"color: red\">Cluster feeding is blocked!</h3>\n"); + table.appendRaw(String.format("<p>Summary: <strong>%s</strong></p>\n", + HtmlTable.escape(state.getFeedBlockOrNull().getDescription()))); + } + } + + private static void renderClusterOutOfSyncRatio(ClusterStateBundle state, StateVersionTracker stateVersionTracker, + VdsClusterHtmlRenderer.Table table) { + var stats = stateVersionTracker.getAggregatedClusterStats().getAggregatedStats(); + if (!stats.hasUpdatesFromAllDistributors()) { + table.appendRaw("<p>Current cluster out of sync ratio cannot be computed, as not all " + + "distributors have reported in statistics for the most recent cluster state.</p>\n"); + return; + } + var outOfSync = GlobalBucketSyncStatsCalculator.clusterBucketsOutOfSyncRatio(stats.getGlobalStats()); + if (outOfSync.isEmpty()) { + table.appendRaw("<p>Current cluster out of sync ratio cannot be computed, as not all " + + "distributors have reported valid statistics.</p>\n"); + return; + } + boolean hasMaintenance = stateHasAtLeastOneMaintenanceNode(state); + if (!hasMaintenance && outOfSync.get() == 0.0) { + table.appendRaw("<p>Cluster is currently in sync.</p>\n"); + } else { + table.appendRaw("<p>Cluster is currently <strong>%.2f%% out of sync</strong>.</p>\n".formatted(outOfSync.get() * 100.0)); + if (hasMaintenance) { + // It is intentional that a cluster with no pending buckets but with nodes in maintenance mode rather + // emits "0% out of sync" with a caveat rather than "in sync", as we don't know the latter for sure. + table.appendRaw("<p><strong>Note:</strong> since one or more nodes are currently in " + + "Maintenance mode, the true out of sync ratio may be higher.</p>\n"); + } + } + } + + private static boolean stateHasAtLeastOneMaintenanceNode(ClusterStateBundle state) { + var baseline = state.getBaselineClusterState(); + int nodes = baseline.getNodeCount(NodeType.STORAGE); + for (int i = 0; i < nodes; ++i) { + if (baseline.getNodeState(Node.ofStorage(i)).getState().oneOf("m")) { + return true; + } + } + return false; + } + private void storeNodeInfo(ContentCluster cluster, int nodeIndex, NodeType nodeType, Map<Integer, NodeInfo> nodeInfoByIndex) { NodeInfo nodeInfo = cluster.getNodeInfo(new Node(nodeType, nodeIndex)); if (nodeInfo == null) return; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java index 95f648447f4..0053c02c269 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java @@ -308,9 +308,7 @@ public class VdsClusterHtmlRenderer { int nodeEvents = eventLog.getNodeEventsSince(nodeInfo.getNode(), currentTime - eventLog.getRecentTimePeriod()); row.addCell(new HtmlTable.Cell("" + nodeEvents)); - if (nodeEvents > 20) { - row.getLastCell().addProperties(ERROR_PROPERTY); - } else if (nodeEvents > 3) { + if (nodeEvents > 3) { row.getLastCell().addProperties(WARNING_PROPERTY); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java index 8048e77b05c..af9e2104828 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java @@ -30,8 +30,9 @@ public class ContentClusterHtmlRendererTest { ClusterStateBundle stateBundle = ClusterStateBundle.ofBaselineOnly( AnnotatedClusterState.withoutAnnotations( ClusterState.stateFromString("version:34633 bits:24 distributor:211 storage:211"))); - var metricUpdater = new MetricUpdater(new NoMetricReporter(), 0, clusterName); - EventLog eventLog = new EventLog(new FakeTimer(), metricUpdater); + var timer = new FakeTimer(); + var metricUpdater = new MetricUpdater(new NoMetricReporter(), timer, 0, clusterName); + EventLog eventLog = new EventLog(timer, metricUpdater); VdsClusterHtmlRenderer.Table table = renderer.createNewClusterHtmlTable(clusterName, slobrokGeneration); ContentCluster contentCluster = mock(ContentCluster.class); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java index 9146b2812a9..1cefac311d3 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import java.time.Instant; import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.testutils.LogFormatter; @@ -22,8 +23,9 @@ public class FakeTimer implements Timer { // Don't start at zero. Clock users may initialize a 'last run' entry with 0, and we want first time to always look like a timeout private long currentTime = (long) 30 * 365 * 24 * 60 * 60 * 1000; - public synchronized long getCurrentTimeInMillis() { - return currentTime; + @Override + public synchronized Instant getCurrentWallClockTime() { + return Instant.ofEpochMilli(currentTime); } public synchronized void advanceTime(long time) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index 3b26e3b6965..688d82e5ebb 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -120,7 +120,7 @@ public abstract class FleetControllerTest implements Waiter { RpcServer rpcServer, boolean start) { waiter = createWaiter(timer); - var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex(), options.clusterName()); + var metricUpdater = new MetricUpdater(new NoMetricReporter(), timer, options.fleetControllerIndex(), options.clusterName()); var log = new EventLog(timer, metricUpdater); var cluster = new ContentCluster(options.clusterName(), options.nodes(), options.storageDistribution()); var stateGatherer = new NodeStateGatherer(timer, timer, log); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java index 7175aefa97c..5eeb4c55387 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java @@ -2,10 +2,13 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; import org.junit.jupiter.api.Test; +import java.time.Duration; +import java.time.Instant; import java.util.Map; import static com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext.hasMetricContext; @@ -25,7 +28,8 @@ public class MetricReporterTest { private static class Fixture { final MetricReporter mockReporter = mock(MetricReporter.class); - final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0, CLUSTER_NAME); + final FakeTimer timer = new FakeTimer(); + final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, timer, 0, CLUSTER_NAME); final ClusterFixture clusterFixture; Fixture() { @@ -61,7 +65,7 @@ public class MetricReporterTest { Fixture f = new Fixture(); f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString("distributor:10 .1.s:d storage:9 .1.s:d .2.s:m .4.s:d"), - new ResourceUsageStats()); + new ResourceUsageStats(), Instant.ofEpochMilli(12345)); verify(f.mockReporter).set(eq("cluster-controller.up.count"), eq(9), argThat(hasMetricContext(withNodeTypeDimension("distributor")))); @@ -78,7 +82,7 @@ public class MetricReporterTest { private void doTestRatiosInState(String clusterState, double distributorRatio, double storageRatio) { Fixture f = new Fixture(); f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString(clusterState), - new ResourceUsageStats()); + new ResourceUsageStats(), Instant.ofEpochMilli(12345)); verify(f.mockReporter).set(eq("cluster-controller.available-nodes.ratio"), doubleThat(closeTo(distributorRatio, 0.0001)), @@ -115,7 +119,7 @@ public class MetricReporterTest { Fixture f = new Fixture(); f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString("distributor:10 storage:10"), - new ResourceUsageStats(0.5, 0.6, 5, 0.7, 0.8)); + new ResourceUsageStats(0.5, 0.6, 5, 0.7, 0.8), Instant.ofEpochMilli(12345)); verify(f.mockReporter).set(eq("cluster-controller.resource_usage.max_disk_utilization"), doubleThat(closeTo(0.5, 0.0001)), @@ -138,4 +142,73 @@ public class MetricReporterTest { argThat(hasMetricContext(withClusterDimension()))); } + private static class ConvergenceFixture extends Fixture { + + String stateString; + Instant stateBroadcastTime; + + ConvergenceFixture(String stateString) { + super(5); + this.stateString = stateString; + setUpFixturePendingVersions(); + + metricUpdater.setStateVersionConvergenceGracePeriod(Duration.ofSeconds(10)); + stateBroadcastTime = timer.getCurrentWallClockTime(); + } + + // Sets pending state versions for 5 distributors and storage nodes: + // - distributor: 2 converged, 3 not converged + // - storage: 3 converged, 2 not converged + private void setUpFixturePendingVersions() { + var pendingBundle = ClusterStateBundle.ofBaselineOnly(AnnotatedClusterState.withoutAnnotations( + ClusterState.stateFromString(stateString))); + for (int i = 0; i < 5; ++i) { + clusterFixture.cluster().getNodeInfo(Node.ofDistributor(i)).setClusterStateVersionBundleSent(pendingBundle); + clusterFixture.cluster().getNodeInfo(Node.ofStorage(i)).setClusterStateVersionBundleSent(pendingBundle); + } + clusterFixture.cluster().getNodeInfo(Node.ofDistributor(0)).setClusterStateBundleVersionAcknowledged(100, false); // NACK + clusterFixture.cluster().getNodeInfo(Node.ofDistributor(1)).setClusterStateBundleVersionAcknowledged(100, true); + clusterFixture.cluster().getNodeInfo(Node.ofDistributor(4)).setClusterStateBundleVersionAcknowledged(100, true); + // Heard nothing from distributors {2, 3} yet. + clusterFixture.cluster().getNodeInfo(Node.ofStorage(0)).setClusterStateBundleVersionAcknowledged(100, true); + clusterFixture.cluster().getNodeInfo(Node.ofStorage(1)).setClusterStateBundleVersionAcknowledged(100, true); + clusterFixture.cluster().getNodeInfo(Node.ofStorage(2)).setClusterStateBundleVersionAcknowledged(100, true); + // Heard nothing from storage {3, 4} yet. + } + + void advanceTimeAndVerifyMetrics(Duration delta, int expectedDistr, int expectedStorage) { + timer.advanceTime(delta.toMillis()); + metricUpdater.updateClusterStateMetrics(clusterFixture.cluster(), + ClusterState.stateFromString(stateString), + new ResourceUsageStats(), stateBroadcastTime); + + verify(mockReporter).set(eq("cluster-controller.nodes-not-converged"), eq(expectedDistr), + argThat(hasMetricContext(withNodeTypeDimension("distributor")))); + verify(mockReporter).set(eq("cluster-controller.nodes-not-converged"), eq(expectedStorage), + argThat(hasMetricContext(withNodeTypeDimension("storage")))); + } + + } + + @Test + void nodes_not_converged_metric_not_incremented_when_within_grace_period() { + var f = new ConvergenceFixture("version:100 distributor:5 storage:5"); + // 9 seconds passed since state broadcast; should not tag nodes as not converged + f.advanceTimeAndVerifyMetrics(Duration.ofMillis(9000), 0, 0); + } + + @Test + void nodes_not_converged_metric_incremented_when_outside_grace_period() { + var f = new ConvergenceFixture("version:100 distributor:5 storage:5"); + // 10+ seconds passed since state broadcast; _should_ tag nodes as not converged + f.advanceTimeAndVerifyMetrics(Duration.ofMillis(10001), 3, 2); + } + + @Test + void only_count_nodes_in_available_states_as_non_converging() { + var f = new ConvergenceFixture("version:100 distributor:5 .0.s:d .2.s:d .3.s:d storage:5 .3.s:m .4.s:d"); + // Should not count non-converged nodes, as they are not in an available state + f.advanceTimeAndVerifyMetrics(Duration.ofMillis(10001), 0, 0); + } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index 5f2046b1450..fe4c0af06d6 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -194,6 +194,10 @@ class OverrideProcessor implements PreProcessor { if ( ! elementTags.isEmpty()) { // match tags if ( ! elementTags.intersects(tags)) return false; + // Tags are set on instances. Having a tag match for a deployment to a non-prod environment + // disables the usual downscaling of the cluster, which is surprising. We therefore either + // require the tags match to either also match an environment directive, or the implicit prod. + if (elementEnvironments.isEmpty() && environment != Environment.prod) return false; } return true; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 28bf8e10a93..7b7f67b4ebb 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -52,6 +52,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.io.StringReader; +import java.nio.file.AccessDeniedException; +import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; import java.security.MessageDigest; import java.util.ArrayList; @@ -70,6 +72,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.text.Lowercase.toLowerCase; +import static java.util.logging.Level.INFO; /** @@ -613,24 +616,43 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { @Override public ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException { - IOUtils.recursiveDeleteDir(preprocessedDir); - IOUtils.copyDirectory(appDir, preprocessedDir, -1, + java.nio.file.Path tempDir = null; + try { + tempDir = Files.createTempDirectory(appDir.getParentFile().toPath(), "preprocess-tempdir"); + preprocess(appDir, tempDir.toFile(), zone); + IOUtils.recursiveDeleteDir(preprocessedDir); + // Use 'move' to make sure we do this atomically, important to avoid writing only partial content e.g. + // when shutting down. + // Temp directory needs to be on the same file system as appDir for 'move' to work, + // if it fails (with DirectoryNotEmptyException (!)) we need to use 'copy' instead + // (this will always be the case for the application package for a standalone container). + Files.move(tempDir, preprocessedDir.toPath()); + tempDir = null; + } catch (AccessDeniedException | DirectoryNotEmptyException e) { + preprocess(appDir, preprocessedDir, zone); + } finally { + if (tempDir != null) + IOUtils.recursiveDeleteDir(tempDir.toFile()); + } + FilesApplicationPackage preprocessedApp = fromFile(preprocessedDir, includeSourceFiles); + preprocessedApp.copyUserDefsIntoApplication(); + return preprocessedApp; + } + + private void preprocess(File appDir, File dir, Zone zone) throws IOException { + validateServicesFile(); + IOUtils.copyDirectory(appDir, dir, - 1, (__, name) -> ! List.of(preprocessed, SERVICES, HOSTS, CONFIG_DEFINITIONS_DIR).contains(name)); - File servicesFile = validateServicesFile(); - preprocessXML(applicationFile(preprocessedDir, SERVICES), servicesFile, zone); - preprocessXML(applicationFile(preprocessedDir, HOSTS), getHostsFile(), zone); - FilesApplicationPackage preprocessed = fromFile(preprocessedDir, includeSourceFiles); - preprocessed.copyUserDefsIntoApplication(); - return preprocessed; + preprocessXML(applicationFile(dir, SERVICES), getServicesFile(), zone); + preprocessXML(applicationFile(dir, HOSTS), getHostsFile(), zone); } - private File validateServicesFile() throws IOException { + private void validateServicesFile() throws IOException { File servicesFile = getServicesFile(); if ( ! servicesFile.exists()) throw new IllegalArgumentException(SERVICES + " does not exist in application package"); if (IOUtils.readFile(servicesFile).isEmpty()) throw new IllegalArgumentException(SERVICES + " in application package is empty"); - return servicesFile; } private void copyUserDefsIntoApplication() { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java index 0cdbed3999c..7190b25965f 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java @@ -22,7 +22,7 @@ public class HostedOverrideProcessorTagsTest { "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + " <container id='foo' version='1.0'>" + " <nodes count='5' deploy:tags='a' deploy:environment='perf'/>" + - " <nodes count='10' deploy:tags='a b'/>" + + " <nodes count='10' deploy:tags='a b' deploy:environment='prod dev'/>" + " <nodes count='20' deploy:tags='c'/>" + " <search deploy:tags='b'/>" + " <document-api deploy:tags='d'/>" + @@ -62,6 +62,38 @@ public class HostedOverrideProcessorTagsTest { } @Test + public void testParsingTagATest() throws TransformerException { + String expected = + "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + + "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + + " <container id='foo' version='1.0'>" + + " " + // (╭ರ_•́) + " </container>" + + "</services>"; + assertOverride(InstanceName.defaultName(), + Environment.test, + RegionName.defaultName(), + Tags.fromString("a"), + expected); + } + + @Test + public void testParsingTagADev() throws TransformerException { + String expected = + "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + + "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + + " <container id='foo' version='1.0'>" + + " <nodes count='10' required='true'/>" + + " </container>" + + "</services>"; + assertOverride(InstanceName.defaultName(), + Environment.dev, + RegionName.defaultName(), + Tags.fromString("a"), + expected); + } + + @Test public void testParsingTagB() throws TransformerException { String expected = "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + @@ -79,7 +111,7 @@ public class HostedOverrideProcessorTagsTest { } @Test - public void testParsingTagC() throws TransformerException { + public void testParsingTagCProd() throws TransformerException { String expected = "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + @@ -95,6 +127,22 @@ public class HostedOverrideProcessorTagsTest { } @Test + public void testParsingTagCDev() throws TransformerException { + String expected = + "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + + "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + + " <container id='foo' version='1.0'>" + + " " + // (╭ರ_•́) + " </container>" + + "</services>"; + assertOverride(InstanceName.defaultName(), + Environment.dev, + RegionName.defaultName(), + Tags.fromString("c"), + expected); + } + + @Test public void testParsingTagCAndD() throws TransformerException { String expected = "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 888a233c62a..a840950ea57 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -325,11 +325,11 @@ ], "methods" : [ "public void <init>(com.yahoo.config.provision.Environment)", - "public void <init>(com.yahoo.config.provision.Environment, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Map, java.util.Optional)", + "public void <init>(com.yahoo.config.provision.Environment, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Map, java.util.Optional)", "public com.yahoo.config.provision.Environment environment()", "public java.util.Optional region()", - "public boolean active()", "public java.util.Optional testerFlavor()", + "public java.util.Optional testerNodes()", "public java.util.List zones()", "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", "public boolean isTest()", @@ -1328,7 +1328,9 @@ "public int persistenceThreadMaxFeedOpBatchSize()", "public boolean logserverOtelCol()", "public com.yahoo.config.provision.SharedHosts sharedHosts()", - "public com.yahoo.config.provision.NodeResources$Architecture adminClusterArchitecture()" + "public com.yahoo.config.provision.NodeResources$Architecture adminClusterArchitecture()", + "public boolean symmetricPutAndActivateReplicaSelection()", + "public boolean enforceStrictlyIncreasingClusterStateVersions()" ], "fields" : [ ] }, @@ -1386,7 +1388,9 @@ "public java.util.Optional cloudAccount()", "public boolean allowUserFilters()", "public java.time.Duration endpointConnectionTtl()", - "public java.util.List dataplaneTokens()" + "public java.util.List dataplaneTokens()", + "public java.util.List requestPrefixForLoggingContent()", + "public boolean launchApplicationAthenzService()" ], "fields" : [ ] }, @@ -1860,7 +1864,6 @@ ], "fields" : [ "public static final enum com.yahoo.config.model.api.container.ContainerServiceType CONTAINER", - "public static final enum com.yahoo.config.model.api.container.ContainerServiceType QRSERVER", "public static final enum com.yahoo.config.model.api.container.ContainerServiceType CLUSTERCONTROLLER_CONTAINER", "public static final enum com.yahoo.config.model.api.container.ContainerServiceType LOGSERVER_CONTAINER", "public static final enum com.yahoo.config.model.api.container.ContainerServiceType METRICS_PROXY_CONTAINER", diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 4308e0c2a0e..fdaa7d57074 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -435,15 +435,16 @@ public class DeploymentSpec { private final Optional<RegionName> region; private final Optional<AthenzService> athenzService; private final Optional<String> testerFlavor; + private final Optional<String> testerNodes; private final Map<CloudName, CloudAccount> cloudAccounts; private final Optional<Duration> hostTTL; public DeclaredZone(Environment environment) { - this(environment, Optional.empty(), Optional.empty(), Optional.empty(), Map.of(), Optional.empty()); + this(environment, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Map.of(), Optional.empty()); } - public DeclaredZone(Environment environment, Optional<RegionName> region, - Optional<AthenzService> athenzService, Optional<String> testerFlavor, + public DeclaredZone(Environment environment, Optional<RegionName> region, Optional<AthenzService> athenzService, + Optional<String> testerFlavor, Optional<String> testerNodes, Map<CloudName, CloudAccount> cloudAccounts, Optional<Duration> hostTTL) { if (environment != Environment.prod && region.isPresent()) illegal("Non-prod environments cannot specify a region"); @@ -454,6 +455,7 @@ public class DeploymentSpec { this.region = Objects.requireNonNull(region); this.athenzService = Objects.requireNonNull(athenzService); this.testerFlavor = Objects.requireNonNull(testerFlavor); + this.testerNodes = Objects.requireNonNull(testerNodes); this.cloudAccounts = Map.copyOf(cloudAccounts); this.hostTTL = Objects.requireNonNull(hostTTL); } @@ -463,11 +465,12 @@ public class DeploymentSpec { /** The region name, or empty if not declared */ public Optional<RegionName> region() { return region; } - // TODO(mpolden): Remove after Vespa < 8.203 is no longer in use - public boolean active() { return true; } - + // TODO jonmv: remove after 8.350. public Optional<String> testerFlavor() { return testerFlavor; } + /** The XML <nodes> tag of the tester application for this zone, if specified. */ + public Optional<String> testerNodes() { return testerNodes; } + Optional<AthenzService> athenzService() { return athenzService; } Map<CloudName, CloudAccount> cloudAccounts() { return cloudAccounts; } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 23471264960..1f5fa228d8f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -92,6 +92,8 @@ public class DeploymentSpecXmlReader { private static final String athenzServiceAttribute = "athenz-service"; private static final String athenzDomainAttribute = "athenz-domain"; private static final String testerFlavorAttribute = "tester-flavor"; + private static final String testerTag = "tester"; + private static final String nodesTag = "nodes"; private static final String majorVersionAttribute = "major-version"; private static final String cloudAccountAttribute = "cloud-account"; private static final String hostTTLAttribute = "empty-host-ttl"; @@ -265,6 +267,7 @@ public class DeploymentSpecXmlReader { private List<Step> readNonInstanceSteps(Element stepTag, Map<String, String> prodAttributes, Element parentTag, Bcp defaultBcp) { Optional<AthenzService> athenzService = mostSpecificAttribute(stepTag, athenzServiceAttribute).map(AthenzService::from); Optional<String> testerFlavor = mostSpecificAttribute(stepTag, testerFlavorAttribute); + Optional<String> testerNodes = mostSpecificSibling(stepTag, testerTag).map(tester -> XML.getChild(tester, nodesTag)).map(XML::toString); switch (stepTag.getTagName()) { case testTag: @@ -273,7 +276,7 @@ public class DeploymentSpecXmlReader { return List.of(new DeclaredTest(RegionName.from(XML.getValue(stepTag).trim()), readHostTTL(stepTag))); // A production test } case devTag, perfTag, stagingTag: // Intentional fallthrough from test tag. - return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), athenzService, testerFlavor, readCloudAccounts(stepTag), readHostTTL(stepTag))); + return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), athenzService, testerFlavor, testerNodes, readCloudAccounts(stepTag), readHostTTL(stepTag))); case prodTag: // regions, delay and parallel may be nested within, but we can flatten them return XML.getChildren(stepTag).stream() .flatMap(child -> readNonInstanceSteps(child, prodAttributes, stepTag, defaultBcp).stream()) @@ -291,7 +294,7 @@ public class DeploymentSpecXmlReader { .flatMap(child -> readSteps(child, prodAttributes, parentTag, defaultBcp).stream()) .toList())); case regionTag: - return List.of(readDeclaredZone(Environment.prod, athenzService, testerFlavor, stepTag)); + return List.of(readDeclaredZone(Environment.prod, athenzService, testerFlavor, testerNodes, stepTag)); default: return List.of(); } @@ -680,9 +683,9 @@ public class DeploymentSpecXmlReader { } private DeclaredZone readDeclaredZone(Environment environment, Optional<AthenzService> athenzService, - Optional<String> testerFlavor, Element regionTag) { + Optional<String> testerFlavor, Optional<String> testerNodes, Element regionTag) { return new DeclaredZone(environment, Optional.of(RegionName.from(XML.getValue(regionTag).trim())), - athenzService, testerFlavor, + athenzService, testerFlavor, testerNodes, readCloudAccounts(regionTag), readHostTTL(regionTag)); } @@ -808,6 +811,16 @@ public class DeploymentSpecXmlReader { return mostSpecificAttribute(tag, attributeName, true); } + /** Returns the first encountered sibling with the given name, or sibling of parent, or sibling of grandparent, etc.. */ + private static Optional<Element> mostSpecificSibling(Element tag, String siblingName) { + return Stream.iterate(tag, Objects::nonNull, Node::getParentNode) + .filter(Element.class::isInstance) + .map(Element.class::cast) + .map(element -> XML.getChild(element, siblingName)) + .filter(Objects::nonNull) + .findFirst(); + } + /** * Returns a string consisting of a number followed by "m", "h" or "d" to a duration given in that unit, * or zero duration if null or blank. diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 67735329287..670c989cf5b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -99,7 +99,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default int heapSizePercentage() { return 0; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } - @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } + @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "8.350.x") default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return "-XX:-OmitStackTraceInFastThrow"; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.75; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } @@ -119,6 +119,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"olaa"}) default boolean logserverOtelCol() { return false; } @ModelFeatureFlag(owners = {"bratseth"}) default SharedHosts sharedHosts() { return SharedHosts.empty(); } @ModelFeatureFlag(owners = {"bratseth"}) default Architecture adminClusterArchitecture() { return Architecture.x86_64; } + @ModelFeatureFlag(owners = {"vekterli"}) default boolean symmetricPutAndActivateReplicaSelection() { return false; } + @ModelFeatureFlag(owners = {"vekterli"}) default boolean enforceStrictlyIncreasingClusterStateVersions() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ @@ -172,6 +174,10 @@ public interface ModelContext { default List<DataplaneToken> dataplaneTokens() { return List.of(); } + default List<String> requestPrefixForLoggingContent() { return List.of(); } + + default boolean launchApplicationAthenzService() { return false; } + } @Retention(RetentionPolicy.RUNTIME) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java index c45d69f02cb..23cf6846e26 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java @@ -35,7 +35,7 @@ public record OnnxMemoryStats(long vmSize, long vmRss, long mallocPeak, long mal } public static Path memoryStatsFilePath(Path modelPath) { - var fileName = modelPath.getRelative().replaceAll("[^\\w\\d\\$@_]", "_") + ".memory_stats"; + var fileName = modelPath.getRelative().replaceAll("[^\\w$@_]", "_") + ".memory_stats"; return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/container/ContainerServiceType.java b/config-model-api/src/main/java/com/yahoo/config/model/api/container/ContainerServiceType.java index 3283f5dd2c5..8c214574d46 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/container/ContainerServiceType.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/container/ContainerServiceType.java @@ -7,7 +7,6 @@ package com.yahoo.config.model.api.container; public enum ContainerServiceType { CONTAINER("container"), - QRSERVER("qrserver"), CLUSTERCONTROLLER_CONTAINER("container-clustercontroller"), LOGSERVER_CONTAINER("logserver-container"), METRICS_PROXY_CONTAINER("metricsproxy-container"); diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 05807ae6cc1..a3df216eea7 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -1128,7 +1128,7 @@ public class DeploymentSpecTest { } @Test - public void customTesterFlavor() { + public void customLegacyTesterFlavor() { DeploymentSpec spec = DeploymentSpec.fromXml(""" <deployment> <instance id='default'> @@ -1145,6 +1145,42 @@ public class DeploymentSpecTest { } @Test + public void customTesterFlavor() { + DeploymentSpec spec = DeploymentSpec.fromXml(""" + <deployment> + <instance id='default'> + <test> + <tester> + <nodes docker-image="foo"> + <resources vcpu="1" memory="3.5Gb" disk="30Gb" architecture="arm64" /> + </nodes> + </tester> + </test> + <staging /> + <prod> + <tester> + <nodes> + <resources vcpu="2" memory="7Gb" disk="30Gb" /> + </nodes> + </tester> + <region>us-north-7</region> + </prod> + </instance> + </deployment>"""); + assertEquals(Optional.of(""" + <nodes docker-image="foo"> + <resources architecture="arm64" disk="30Gb" memory="3.5Gb" vcpu="1"/> + </nodes>"""), + spec.requireInstance("default").steps().get(0).zones().get(0).testerNodes()); + assertEquals(Optional.empty(), spec.requireInstance("default").steps().get(1).zones().get(0).testerNodes()); + assertEquals(Optional.of(""" + <nodes> + <resources disk="30Gb" memory="7Gb" vcpu="2"/> + </nodes>"""), + spec.requireInstance("default").steps().get(2).zones().get(0).testerNodes()); + } + + @Test public void noEndpoints() { DeploymentSpec spec = DeploymentSpec.fromXml(""" <deployment> diff --git a/config-model-api/src/test/java/com/yahoo/config/model/api/container/ContainerServiceTypeTest.java b/config-model-api/src/test/java/com/yahoo/config/model/api/container/ContainerServiceTypeTest.java index e451598b71d..e2f23f60e4b 100644 --- a/config-model-api/src/test/java/com/yahoo/config/model/api/container/ContainerServiceTypeTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/model/api/container/ContainerServiceTypeTest.java @@ -7,7 +7,6 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERC import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static org.junit.Assert.assertEquals; /** @@ -17,13 +16,12 @@ public class ContainerServiceTypeTest { @Test public void new_values_are_not_added_without_updating_tests() { - assertEquals(5, ContainerServiceType.values().length); + assertEquals(4, ContainerServiceType.values().length); } @Test public void service_names_do_not_change() { assertEquals("container", CONTAINER.serviceName); - assertEquals("qrserver", QRSERVER.serviceName); assertEquals("container-clustercontroller", CLUSTERCONTROLLER_CONTAINER.serviceName); assertEquals("logserver-container", LOGSERVER_CONTAINER.serviceName); assertEquals("metricsproxy-container", METRICS_PROXY_CONTAINER.serviceName); diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index e2c6b788b02..4ef591cda9f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -84,7 +84,7 @@ public class AdminModel extends ConfigModel { } TreeConfigProducer<AnyConfigProducer> parent = modelContext.getParentProducer(); ModelContext.Properties properties = modelContext.getDeployState().getProperties(); - DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), + DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext, properties.multitenant(), properties.configServerSpecs()); model.admin = domBuilder.build(modelContext.getDeployState(), parent, adminElement); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3c45588a054..c2d6adddeed 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -55,7 +55,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedNiceness = 0.0; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = List.of(); - private String jvmOmitStackTraceInFastThrowOption; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = List.of(); private double resourceLimitDisk = 0.75; @@ -84,6 +83,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int contentLayerMetadataFeatureLevel = 0; private int persistenceThreadMaxFeedOpBatchSize = 1; private boolean logserverOtelCol = false; + private boolean symmetricPutAndActivateReplicaSelection = false; + private boolean enforceStrictlyIncreasingClusterStateVersions = false; + private boolean launchApplicationAthenzService = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -111,7 +113,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double feedNiceness() { return feedNiceness; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } - @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @@ -142,6 +143,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public int persistenceThreadMaxFeedOpBatchSize() { return persistenceThreadMaxFeedOpBatchSize; } @Override public boolean logserverOtelCol() { return logserverOtelCol; } + @Override public boolean symmetricPutAndActivateReplicaSelection() { return symmetricPutAndActivateReplicaSelection; } + @Override public boolean enforceStrictlyIncreasingClusterStateVersions() { return enforceStrictlyIncreasingClusterStateVersions; } + @Override public boolean launchApplicationAthenzService() { return launchApplicationAthenzService; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -276,11 +280,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setJvmOmitStackTraceInFastThrowOption(String value) { - this.jvmOmitStackTraceInFastThrowOption = value; - return this; - } - public TestProperties allowDisableMtls(boolean value) { this.allowDisableMtls = value; return this; @@ -382,6 +381,21 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setSymmetricPutAndActivateReplicaSelection(boolean symmetricReplicaSelection) { + this.symmetricPutAndActivateReplicaSelection = symmetricReplicaSelection; + return this; + } + + public TestProperties setEnforceStrictlyIncreasingClusterStateVersions(boolean enforce) { + this.enforceStrictlyIncreasingClusterStateVersions = enforce; + return this; + } + + public TestProperties setLaunchApplicationAthenzService(boolean launch) { + this.launchApplicationAthenzService = launch; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index da0fd265724..dcaa7acf823 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -305,8 +305,8 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public int compare(NodeResources a, NodeResources b) { - if (a.memoryGb() > b.memoryGb()) return 1; - if (a.memoryGb() < b.memoryGb()) return -1; + if (a.memoryGiB() > b.memoryGiB()) return 1; + if (a.memoryGiB() < b.memoryGiB()) return -1; if (a.diskGb() > b.diskGb()) return 1; if (a.diskGb() < b.diskGb()) return -1; return Double.compare(a.vcpu(), b.vcpu()); diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 60674b5487c..ed1a4e98b49 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -73,7 +73,8 @@ public class RankProfile implements Cloneable { /** The resolved inherited profiles, or null when not resolved. */ private List<RankProfile> inherited; - private MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhase = null; + private DiversitySettings diversity = null; protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); @@ -106,6 +107,7 @@ public class RankProfile implements Cloneable { /** The drop limit used to drop hits with rank score less than or equal to this value */ private double rankScoreDropLimit = -Double.MAX_VALUE; + private double secondPhaseRankScoreDropLimit = -Double.MAX_VALUE; private Set<ReferenceNode> summaryFeatures; private String inheritedSummaryFeaturesProfileName; @@ -224,7 +226,7 @@ public class RankProfile implements Cloneable { public boolean useSignificanceModel() { if (useSignificanceModel != null) return useSignificanceModel; - return uniquelyInherited(p -> p.useSignificanceModel(), "use-model") + return uniquelyInherited(RankProfile::useSignificanceModel, "use-model") .orElse(false); // Disabled by default } @@ -306,20 +308,28 @@ public class RankProfile implements Cloneable { return false; } - public void setMatchPhaseSettings(MatchPhaseSettings settings) { + public void setMatchPhase(MatchPhaseSettings settings) { settings.checkValid(); - this.matchPhaseSettings = settings; + this.matchPhase = settings; } - public MatchPhaseSettings getMatchPhaseSettings() { - if (matchPhaseSettings != null) return matchPhaseSettings; - return uniquelyInherited(p -> p.getMatchPhaseSettings(), "match phase settings").orElse(null); + public MatchPhaseSettings getMatchPhase() { + if (matchPhase != null) return matchPhase; + return uniquelyInherited(RankProfile::getMatchPhase, "match phase settings").orElse(null); + } + public void setDiversity(DiversitySettings value) { + value.checkValid(); + diversity = value; + } + public DiversitySettings getDiversity() { + if (diversity != null) return diversity; + return uniquelyInherited(RankProfile::getDiversity, "diversity settings").orElse(null); } /** Returns the uniquely determined property, where non-empty is defined as non-null */ private <T> Optional<T> uniquelyInherited(Function<RankProfile, T> propertyRetriever, String propertyDescription) { - return uniquelyInherited(propertyRetriever, p -> p != null, propertyDescription); + return uniquelyInherited(propertyRetriever, Objects::nonNull, propertyDescription); } /** @@ -334,8 +344,8 @@ public class RankProfile implements Cloneable { Predicate<T> nonEmptyValueFilter, String propertyDescription) { Set<T> uniqueProperties = inherited().stream() - .map(p -> propertyRetriever.apply(p)) - .filter(p -> nonEmptyValueFilter.test(p)) + .map(propertyRetriever) + .filter(nonEmptyValueFilter) .collect(Collectors.toSet()); if (uniqueProperties.isEmpty()) return Optional.empty(); if (uniqueProperties.size() == 1) return uniqueProperties.stream().findAny(); @@ -494,7 +504,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getFirstPhase() { if (firstPhaseRanking != null) return firstPhaseRanking; - return uniquelyInherited(p -> p.getFirstPhase(), "first-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getFirstPhase, "first-phase expression").orElse(null); } void setFirstPhaseRanking(RankingExpression rankingExpression) { @@ -521,7 +531,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - return uniquelyInherited(p -> p.getSecondPhase(), "second-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getSecondPhase, "second-phase expression").orElse(null); } public void setSecondPhaseRanking(String expression) { @@ -541,7 +551,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getGlobalPhase() { if (globalPhaseRanking != null) return globalPhaseRanking; - return uniquelyInherited(p -> p.getGlobalPhase(), "global-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getGlobalPhase, "global-phase expression").orElse(null); } public void setGlobalPhaseRanking(String expression) { @@ -600,7 +610,7 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableSet(combined); } if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); - return uniquelyInherited(p -> p.getSummaryFeatures(), f -> ! f.isEmpty(), "summary features") + return uniquelyInherited(RankProfile::getSummaryFeatures, f -> ! f.isEmpty(), "summary features") .orElse(Set.of()); } @@ -617,13 +627,13 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableSet(combined); } if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); - return uniquelyInherited(p -> p.getMatchFeatures(), f -> ! f.isEmpty(), "match features") + return uniquelyInherited(RankProfile::getMatchFeatures, f -> ! f.isEmpty(), "match features") .orElse(Set.of()); } public Set<ReferenceNode> getHiddenMatchFeatures() { if (hiddenMatchFeatures != null) return Collections.unmodifiableSet(hiddenMatchFeatures); - return uniquelyInherited(p -> p.getHiddenMatchFeatures(), f -> ! f.isEmpty(), "hidden match features") + return uniquelyInherited(RankProfile::getHiddenMatchFeatures, f -> ! f.isEmpty(), "hidden match features") .orElse(Set.of()); } @@ -661,7 +671,7 @@ public class RankProfile implements Cloneable { /** Returns a read-only view of the rank features to use in this profile. This is never null */ public Set<ReferenceNode> getRankFeatures() { if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); - return uniquelyInherited(p -> p.getRankFeatures(), f -> ! f.isEmpty(), "summary-features") + return uniquelyInherited(RankProfile::getRankFeatures, f -> ! f.isEmpty(), "summary-features") .orElse(Set.of()); } @@ -692,7 +702,7 @@ public class RankProfile implements Cloneable { if (rankProperties.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); - var inheritedProperties = uniquelyInherited(p -> p.getRankPropertyMap(), m -> ! m.isEmpty(), "rank-properties") + var inheritedProperties = uniquelyInherited(RankProfile::getRankPropertyMap, m -> ! m.isEmpty(), "rank-properties") .orElse(Map.of()); if (rankProperties.isEmpty()) return inheritedProperties; @@ -734,21 +744,21 @@ public class RankProfile implements Cloneable { public int getRerankCount() { if (rerankCount >= 0) return rerankCount; - return uniquelyInherited(p -> p.getRerankCount(), c -> c >= 0, "rerank-count").orElse(-1); + return uniquelyInherited(RankProfile::getRerankCount, c -> c >= 0, "rerank-count").orElse(-1); } public void setGlobalPhaseRerankCount(int count) { this.globalPhaseRerankCount = count; } public int getGlobalPhaseRerankCount() { if (globalPhaseRerankCount >= 0) return globalPhaseRerankCount; - return uniquelyInherited(p -> p.getGlobalPhaseRerankCount(), c -> c >= 0, "global-phase rerank-count").orElse(-1); + return uniquelyInherited(RankProfile::getGlobalPhaseRerankCount, c -> c >= 0, "global-phase rerank-count").orElse(-1); } public void setNumThreadsPerSearch(int numThreads) { this.numThreadsPerSearch = numThreads; } public int getNumThreadsPerSearch() { if (numThreadsPerSearch >= 0) return numThreadsPerSearch; - return uniquelyInherited(p -> p.getNumThreadsPerSearch(), n -> n >= 0, "num-threads-per-search") + return uniquelyInherited(RankProfile::getNumThreadsPerSearch, n -> n >= 0, "num-threads-per-search") .orElse(-1); } @@ -756,14 +766,14 @@ public class RankProfile implements Cloneable { public int getMinHitsPerThread() { if (minHitsPerThread >= 0) return minHitsPerThread; - return uniquelyInherited(p -> p.getMinHitsPerThread(), n -> n >= 0, "min-hits-per-search").orElse(-1); + return uniquelyInherited(RankProfile::getMinHitsPerThread, n -> n >= 0, "min-hits-per-search").orElse(-1); } public void setNumSearchPartitions(int numSearchPartitions) { this.numSearchPartitions = numSearchPartitions; } public int getNumSearchPartitions() { if (numSearchPartitions >= 0) return numSearchPartitions; - return uniquelyInherited(p -> p.getNumSearchPartitions(), n -> n >= 0, "num-search-partitions").orElse(-1); + return uniquelyInherited(RankProfile::getNumSearchPartitions, n -> n >= 0, "num-search-partitions").orElse(-1); } public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } @@ -773,7 +783,7 @@ public class RankProfile implements Cloneable { public OptionalDouble getTermwiseLimit() { if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); - return uniquelyInherited(p -> p.getTermwiseLimit(), l -> l.isPresent(), "termwise-limit") + return uniquelyInherited(RankProfile::getTermwiseLimit, OptionalDouble::isPresent, "termwise-limit") .orElse(OptionalDouble.empty()); } @@ -781,21 +791,21 @@ public class RankProfile implements Cloneable { if (postFilterThreshold != null) { return OptionalDouble.of(postFilterThreshold); } - return uniquelyInherited(p -> p.getPostFilterThreshold(), l -> l.isPresent(), "post-filter-threshold").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getPostFilterThreshold, OptionalDouble::isPresent, "post-filter-threshold").orElse(OptionalDouble.empty()); } public OptionalDouble getApproximateThreshold() { if (approximateThreshold != null) { return OptionalDouble.of(approximateThreshold); } - return uniquelyInherited(p -> p.getApproximateThreshold(), l -> l.isPresent(), "approximate-threshold").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getApproximateThreshold, OptionalDouble::isPresent, "approximate-threshold").orElse(OptionalDouble.empty()); } public OptionalDouble getTargetHitsMaxAdjustmentFactor() { if (targetHitsMaxAdjustmentFactor != null) { return OptionalDouble.of(targetHitsMaxAdjustmentFactor); } - return uniquelyInherited(p -> p.getTargetHitsMaxAdjustmentFactor(), l -> l.isPresent(), "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getTargetHitsMaxAdjustmentFactor, OptionalDouble::isPresent, "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); } /** Whether we should ignore the default rank features. Set to null to use inherited */ @@ -805,24 +815,34 @@ public class RankProfile implements Cloneable { public Boolean getIgnoreDefaultRankFeatures() { if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; - return uniquelyInherited(p -> p.getIgnoreDefaultRankFeatures(), "ignore-default-rank-features").orElse(false); + return uniquelyInherited(RankProfile::getIgnoreDefaultRankFeatures, "ignore-default-rank-features").orElse(false); } public void setKeepRankCount(int rerankArraySize) { this.keepRankCount = rerankArraySize; } public int getKeepRankCount() { if (keepRankCount >= 0) return keepRankCount; - return uniquelyInherited(p -> p.getKeepRankCount(), c -> c >= 0, "keep-rank-count").orElse(-1); + return uniquelyInherited(RankProfile::getKeepRankCount, c -> c >= 0, "keep-rank-count").orElse(-1); } public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } public double getRankScoreDropLimit() { if (rankScoreDropLimit > -Double.MAX_VALUE) return rankScoreDropLimit; - return uniquelyInherited(p -> p.getRankScoreDropLimit(), c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") + return uniquelyInherited(RankProfile::getRankScoreDropLimit, c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") .orElse(rankScoreDropLimit); } + public void setSecondPhaseRankScoreDropLimit(double limit) { this.secondPhaseRankScoreDropLimit = limit; } + + public double getSecondPhaseRankScoreDropLimit() { + if (secondPhaseRankScoreDropLimit > -Double.MAX_VALUE) { + return secondPhaseRankScoreDropLimit; + } + return uniquelyInherited(RankProfile::getSecondPhaseRankScoreDropLimit, c -> c > -Double.MAX_VALUE, "second-phase rank-score-drop-limit") + .orElse(secondPhaseRankScoreDropLimit); + } + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { addFunction(parseRankingExpression(name, arguments, expression), inline); @@ -947,7 +967,7 @@ public class RankProfile implements Cloneable { } private boolean needToUpdateFunctionCache() { - if (inherited().stream().anyMatch(profile -> profile.needToUpdateFunctionCache())) return true; + if (inherited().stream().anyMatch(RankProfile::needToUpdateFunctionCache)) return true; return allFunctionsCached == null; } @@ -955,7 +975,7 @@ public class RankProfile implements Cloneable { /** Returns all filter fields in this profile and any profile it inherits. */ public Set<String> allFilterFields() { - Set<String> inheritedFilterFields = uniquelyInherited(p -> p.allFilterFields(), fields -> ! fields.isEmpty(), + Set<String> inheritedFilterFields = uniquelyInherited(RankProfile::allFilterFields, fields -> ! fields.isEmpty(), "filter fields").orElse(Set.of()); if (inheritedFilterFields.isEmpty()) return Collections.unmodifiableSet(filterFields); @@ -966,7 +986,7 @@ public class RankProfile implements Cloneable { } private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { - if (expression.trim().length() == 0) + if (expression.trim().isEmpty()) throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { @@ -1008,7 +1028,8 @@ public class RankProfile implements Cloneable { try { RankProfile clone = (RankProfile)super.clone(); clone.rankSettings = new LinkedHashSet<>(this.rankSettings); - clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? + clone.matchPhase = this.matchPhase; // hmm? + clone.diversity = this.diversity; clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; clone.matchFeatures = matchFeatures != null ? new LinkedHashSet<>(this.matchFeatures) : null; clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; @@ -1189,7 +1210,7 @@ public class RankProfile implements Cloneable { private Map<Reference, TensorType> featureTypes() { Map<Reference, TensorType> featureTypes = inputs().values().stream() - .collect(Collectors.toMap(input -> input.name(), + .collect(Collectors.toMap(Input::name, input -> input.type().tensorType())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); @@ -1506,15 +1527,9 @@ public class RankProfile implements Cloneable { private boolean ascending = false; private int maxHits = 0; // try to get this many hits before degrading the match phase private double maxFilterCoverage = 0.2; // Max coverage of original corpus that will trigger the filter. - private DiversitySettings diversity = null; private double evaluationPoint = 0.20; private double prePostFilterTippingPoint = 1.0; - public void setDiversity(DiversitySettings value) { - value.checkValid(); - diversity = value; - } - public void setAscending(boolean value) { ascending = value; } public void setAttribute(String value) { attribute = value; } public void setMaxHits(int value) { maxHits = value; } @@ -1526,7 +1541,6 @@ public class RankProfile implements Cloneable { public String getAttribute() { return attribute; } public int getMaxHits() { return maxHits; } public double getMaxFilterCoverage() { return maxFilterCoverage; } - public DiversitySettings getDiversity() { return diversity; } public double getEvaluationPoint() { return evaluationPoint; } public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; } @@ -1690,7 +1704,7 @@ public class RankProfile implements Cloneable { } - public static record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { + public record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { @Override public String toString() { return "normalizer{name=" + name + ",input=" + input + ",algo=" + algo + ",k=" + kparam + "}"; @@ -1711,7 +1725,7 @@ public class RankProfile implements Cloneable { } } - private List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); + private final List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); public Map<String, RankFeatureNormalizer> getFeatureNormalizers() { Map<String, RankFeatureNormalizer> all = new LinkedHashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 3402ba31be9..127d12594b4 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -721,7 +721,7 @@ public class Schema implements ImmutableSchema { "', but this schema does not exist"); // Require schema and document type inheritance to be consistent to keep things simple - // And require it to be explicit so we have the option to support other possibilities later + // and require it to be explicit, so we have the option to support other possibilities later var parentDocument = owner.schemas().get(inherited.get()).getDocument(); if ( ! getDocument().inheritedTypes().containsKey(new DataTypeName(parentDocument.getName()))) throw new IllegalArgumentException(this + " inherits '" + inherited.get() + diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index c1b698df55f..3fb185e333d 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -106,15 +106,18 @@ public class IndexInfo extends Derived { } private static boolean isPositionField(ImmutableSDField field) { - return GeoPos.isAnyPos(field); + return (field != null) && GeoPos.isAnyPos(field); + } + private static boolean isMultivalueField(ImmutableSDField field) { + return (field != null) && field.getDataType().isMultivalue(); } @Override protected void derive(ImmutableSDField field, Schema schema) { - derive(field, schema, false); + derive(field, schema, null); } - protected void derive(ImmutableSDField field, Schema schema, boolean inPosition) { + protected void derive(ImmutableSDField field, Schema schema, ImmutableSDField parent) { if (field.getDataType().equals(DataType.PREDICATE)) { addIndexCommand(field, CMD_PREDICATE); Index index = field.getIndex(field.getName()); @@ -134,14 +137,13 @@ public class IndexInfo extends Derived { String name = e.getValue(); addIndexAlias(alias, name); } - boolean isPosition = isPositionField(field); if (field.usesStructOrMap()) { for (ImmutableSDField structField : field.getStructFields()) { - derive(structField, schema, isPosition); // Recursion + derive(structField, schema, field); // Recursion } } - if (isPosition) { + if (isPositionField(field)) { addIndexCommand(field.getName(), CMD_DEFAULT_POSITION); } @@ -153,12 +155,12 @@ public class IndexInfo extends Derived { addIndexCommand(field, CMD_LOWERCASE); } - if (field.getDataType().isMultivalue()) { + if (isMultivalueField(field) || isMultivalueField(parent)) { addIndexCommand(field, CMD_MULTIVALUE); } Attribute attribute = field.getAttribute(); - if ((field.doesAttributing() || (attribute != null && !inPosition)) && !field.doesIndexing()) { + if ((field.doesAttributing() || (attribute != null && !isPositionField(parent))) && !field.doesIndexing()) { addIndexCommand(field.getName(), CMD_ATTRIBUTE); if (attribute != null && attribute.isFastSearch()) addIndexCommand(field.getName(), CMD_FAST_SEARCH); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index b057624f055..15e5891a3e3 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -159,6 +159,7 @@ public class RawRankProfile { private final boolean ignoreDefaultRankFeatures; private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final RankProfile.DiversitySettings diversitySettings; private final int rerankCount; private final int keepRankCount; private final int numThreadsPerSearch; @@ -169,6 +170,7 @@ public class RawRankProfile { private final OptionalDouble approximateThreshold; private final OptionalDouble targetHitsMaxAdjustmentFactor; private final double rankScoreDropLimit; + private final double secondPhaseRankScoreDropLimit; private final boolean sortBlueprintsByCost; private final boolean alwaysMarkPhraseExpensive; @@ -207,7 +209,8 @@ public class RawRankProfile { rankFeatures = compiled.getRankFeatures(); rerankCount = compiled.getRerankCount(); globalPhaseRerankCount = compiled.getGlobalPhaseRerankCount(); - matchPhaseSettings = compiled.getMatchPhaseSettings(); + matchPhaseSettings = compiled.getMatchPhase(); + diversitySettings = compiled.getDiversity(); numThreadsPerSearch = compiled.getNumThreadsPerSearch(); minHitsPerThread = compiled.getMinHitsPerThread(); numSearchPartitions = compiled.getNumSearchPartitions(); @@ -219,6 +222,7 @@ public class RawRankProfile { targetHitsMaxAdjustmentFactor = compiled.getTargetHitsMaxAdjustmentFactor(); keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); + secondPhaseRankScoreDropLimit = compiled.getSecondPhaseRankScoreDropLimit(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(compiled.getRankProperties()); @@ -486,13 +490,12 @@ public class RawRankProfile { properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); - RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); - if (diversitySettings != null) { - properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); - properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); - } + } + if (diversitySettings != null) { + properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); } if (rerankCount > -1) { properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); @@ -506,6 +509,9 @@ public class RawRankProfile { if (rankScoreDropLimit > -Double.MAX_VALUE) { properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); } + if (secondPhaseRankScoreDropLimit > -Double.MAX_VALUE) { + properties.add(new Pair<>("vespa.hitcollector.secondphase.rankscoredroplimit", secondPhaseRankScoreDropLimit + "")); + } if (ignoreDefaultRankFeatures) { properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java index 77a10862f9c..ff78a4a3b60 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -38,8 +38,8 @@ public class ConvertParsedRanking { for (String name : parsed.getInherited()) profile.inherit(name); - parsed.isStrict().ifPresent(value -> profile.setStrict(value)); - parsed.isUseSignificanceModel().ifPresent(value -> profile.setUseSignificanceModel(value)); + parsed.isStrict().ifPresent(profile::setStrict); + parsed.isUseSignificanceModel().ifPresent(profile::setUseSignificanceModel); for (var constant : parsed.getConstants().values()) profile.add(constant); @@ -58,39 +58,26 @@ public class ConvertParsedRanking { profile.addFunction(name, parameters, expression, inline); } - parsed.getRankScoreDropLimit().ifPresent - (value -> profile.setRankScoreDropLimit(value)); - parsed.getTermwiseLimit().ifPresent - (value -> profile.setTermwiseLimit(value)); - parsed.getPostFilterThreshold().ifPresent - (value -> profile.setPostFilterThreshold(value)); - parsed.getApproximateThreshold().ifPresent - (value -> profile.setApproximateThreshold(value)); - parsed.getTargetHitsMaxAdjustmentFactor().ifPresent - (value -> profile.setTargetHitsMaxAdjustmentFactor(value)); - parsed.getKeepRankCount().ifPresent - (value -> profile.setKeepRankCount(value)); - parsed.getMinHitsPerThread().ifPresent - (value -> profile.setMinHitsPerThread(value)); - parsed.getNumSearchPartitions().ifPresent - (value -> profile.setNumSearchPartitions(value)); - parsed.getNumThreadsPerSearch().ifPresent - (value -> profile.setNumThreadsPerSearch(value)); - parsed.getReRankCount().ifPresent - (value -> profile.setRerankCount(value)); - - parsed.getMatchPhaseSettings().ifPresent - (value -> profile.setMatchPhaseSettings(value)); - - parsed.getFirstPhaseExpression().ifPresent - (value -> profile.setFirstPhaseRanking(value)); - parsed.getSecondPhaseExpression().ifPresent - (value -> profile.setSecondPhaseRanking(value)); - - parsed.getGlobalPhaseExpression().ifPresent - (value -> profile.setGlobalPhaseRanking(value)); - parsed.getGlobalPhaseRerankCount().ifPresent - (value -> profile.setGlobalPhaseRerankCount(value)); + parsed.getRankScoreDropLimit().ifPresent(profile::setRankScoreDropLimit); + parsed.getSecondPhaseRankScoreDropLimit().ifPresent(profile::setSecondPhaseRankScoreDropLimit); + parsed.getTermwiseLimit().ifPresent(profile::setTermwiseLimit); + parsed.getPostFilterThreshold().ifPresent(profile::setPostFilterThreshold); + parsed.getApproximateThreshold().ifPresent(profile::setApproximateThreshold); + parsed.getTargetHitsMaxAdjustmentFactor().ifPresent(profile::setTargetHitsMaxAdjustmentFactor); + parsed.getKeepRankCount().ifPresent(profile::setKeepRankCount); + parsed.getMinHitsPerThread().ifPresent(profile::setMinHitsPerThread); + parsed.getNumSearchPartitions().ifPresent(profile::setNumSearchPartitions); + parsed.getNumThreadsPerSearch().ifPresent(profile::setNumThreadsPerSearch); + parsed.getReRankCount().ifPresent(profile::setRerankCount); + + parsed.getMatchPhase().ifPresent(profile::setMatchPhase); + parsed.getDiversity().ifPresent(profile::setDiversity); + + parsed.getFirstPhaseExpression().ifPresent(profile::setFirstPhaseRanking); + parsed.getSecondPhaseExpression().ifPresent(profile::setSecondPhaseRanking); + + parsed.getGlobalPhaseExpression().ifPresent(profile::setGlobalPhaseRanking); + parsed.getGlobalPhaseRerankCount().ifPresent(profile::setGlobalPhaseRerankCount); for (var value : parsed.getMatchFeatures()) { profile.addMatchFeatures(value); @@ -102,10 +89,8 @@ public class ConvertParsedRanking { profile.addSummaryFeatures(value); } - parsed.getInheritedMatchFeatures().ifPresent - (value -> profile.setInheritedMatchFeatures(value)); - parsed.getInheritedSummaryFeatures().ifPresent - (value -> profile.setInheritedSummaryFeatures(value)); + parsed.getInheritedMatchFeatures().ifPresent(profile::setInheritedMatchFeatures); + parsed.getInheritedSummaryFeatures().ifPresent(profile::setInheritedSummaryFeatures); if (parsed.getIgnoreDefaultRankFeatures()) { profile.setIgnoreDefaultRankFeatures(true); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index 93319e82076..2a117a4af4b 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -4,6 +4,7 @@ package com.yahoo.schema.parser; import com.yahoo.schema.OnnxModel; import com.yahoo.schema.RankProfile; import com.yahoo.schema.RankProfile.MatchPhaseSettings; +import com.yahoo.schema.RankProfile.DiversitySettings; import com.yahoo.schema.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.Reference; @@ -26,6 +27,7 @@ class ParsedRankProfile extends ParsedBlock { private boolean ignoreDefaultRankFeatures = false; private Double rankScoreDropLimit = null; + private Double secondPhaseRankScoreDropLimit = null; private Double termwiseLimit = null; private Double postFilterThreshold = null; private Double approximateThreshold = null; @@ -38,7 +40,8 @@ class ParsedRankProfile extends ParsedBlock { private Integer numSearchPartitions = null; private Integer numThreadsPerSearch = null; private Integer reRankCount = null; - private MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhase = null; + private DiversitySettings diversity = null; private String firstPhaseExpression = null; private String inheritedSummaryFeatures = null; private String inheritedMatchFeatures = null; @@ -64,6 +67,7 @@ class ParsedRankProfile extends ParsedBlock { boolean getIgnoreDefaultRankFeatures() { return this.ignoreDefaultRankFeatures; } Optional<Double> getRankScoreDropLimit() { return Optional.ofNullable(this.rankScoreDropLimit); } + Optional<Double> getSecondPhaseRankScoreDropLimit() { return Optional.ofNullable(this.secondPhaseRankScoreDropLimit); } Optional<Double> getTermwiseLimit() { return Optional.ofNullable(this.termwiseLimit); } Optional<Double> getPostFilterThreshold() { return Optional.ofNullable(this.postFilterThreshold); } Optional<Double> getApproximateThreshold() { return Optional.ofNullable(this.approximateThreshold); } @@ -76,7 +80,8 @@ class ParsedRankProfile extends ParsedBlock { Optional<Integer> getNumSearchPartitions() { return Optional.ofNullable(this.numSearchPartitions); } Optional<Integer> getNumThreadsPerSearch() { return Optional.ofNullable(this.numThreadsPerSearch); } Optional<Integer> getReRankCount() { return Optional.ofNullable(this.reRankCount); } - Optional<MatchPhaseSettings> getMatchPhaseSettings() { return Optional.ofNullable(this.matchPhaseSettings); } + Optional<MatchPhaseSettings> getMatchPhase() { return Optional.ofNullable(this.matchPhase); } + Optional<DiversitySettings> getDiversity() { return Optional.ofNullable(this.diversity); } Optional<String> getFirstPhaseExpression() { return Optional.ofNullable(this.firstPhaseExpression); } Optional<String> getInheritedMatchFeatures() { return Optional.ofNullable(this.inheritedMatchFeatures); } List<ParsedRankFunction> getFunctions() { return List.copyOf(functions.values()); } @@ -171,9 +176,13 @@ class ParsedRankProfile extends ParsedBlock { this.keepRankCount = count; } - void setMatchPhaseSettings(MatchPhaseSettings settings) { - verifyThat(matchPhaseSettings == null, "already has match-phase"); - this.matchPhaseSettings = settings; + void setMatchPhase(MatchPhaseSettings settings) { + verifyThat(matchPhase == null, "already has match-phase"); + this.matchPhase = settings; + } + void setDiversity(DiversitySettings settings) { + verifyThat(diversity == null, "already has diversity"); + this.diversity = settings; } void setMinHitsPerThread(int minHits) { @@ -196,6 +205,11 @@ class ParsedRankProfile extends ParsedBlock { this.rankScoreDropLimit = limit; } + void setSecondPhaseRankScoreDropLimit(double limit) { + verifyThat(secondPhaseRankScoreDropLimit == null, "already has rank-score-drop-limit for second phase"); + this.secondPhaseRankScoreDropLimit = limit; + } + void setRerankCount(int count) { verifyThat(reRankCount == null, "already has rerank-count"); this.reRankCount = count; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java index 5c06ce25184..a8e0f86de8c 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java @@ -23,8 +23,8 @@ public class DiversitySettingsValidator extends Processor { if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { - validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity()); + if (rankProfile.getDiversity() != null) { + validate(rankProfile, rankProfile.getDiversity()); } } } @@ -32,6 +32,9 @@ public class DiversitySettingsValidator extends Processor { String attributeName = settings.getAttribute(); new AttributeValidator(schema.getName(), rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); + if ((rankProfile.getMatchPhase() == null) && (rankProfile.getSecondPhaseRanking() == null)) { + throw new IllegalArgumentException("In schema '" + schema.getName() + "', rank-profile '" + rankProfile.name() + "': 'diversity' requires either 'match-phase' or 'second-phase' to be specified."); + } } private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java index f3a8f7cee18..d29820e0d51 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java @@ -25,7 +25,7 @@ public class MatchPhaseSettingsValidator extends Processor { if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); + RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhase(); if (settings != null) { validateMatchPhaseSettings(rankProfile, settings); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java index 4feb065a90b..5a13267f507 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java @@ -18,6 +18,9 @@ import java.util.Optional; */ public class PagedAttributeValidator extends Processor { + private static final String disavantagesUrl = + "https://docs.vespa.ai/en/attributes.html#paged-attributes-disadvantages"; + public PagedAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, @@ -43,6 +46,10 @@ public class PagedAttributeValidator extends Processor { if (!isSupportedType(attribute)) { fail(schema, field, "The 'paged' attribute setting is not supported for fast-rank tensor and predicate types"); } + if (attribute.getType() == Attribute.Type.TENSOR && attribute.hnswIndexParams().isPresent()) { + warn(schema.getName(), field.getName(), "The 'paged' attribute setting in combination with " + + "HNSW indexing is strongly discouraged, see " + disavantagesUrl + " for details"); + } } private boolean isSupportedType(Attribute attribute) { @@ -58,8 +65,4 @@ public class PagedAttributeValidator extends Processor { return true; } - private boolean isDenseTensorType(TensorType type) { - return type.dimensions().stream().allMatch(d -> d.isIndexed()); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 10de46ae6d8..e48aa0eef7c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -9,7 +9,8 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; + +import static java.util.logging.Level.WARNING; /** * A document summary definition - a list of summary fields. @@ -121,13 +122,12 @@ public class DocumentSummary extends FieldView { public void validate(DeployLogger logger) { for (var inheritedName : inherited) { var inheritedSummary = owner.getSummary(inheritedName); - if (inheritedSummary == null) { - // TODO Vespa 9: Throw IllegalArgumentException instead - logger.logApplicationPackage(Level.WARNING, - this + " inherits '" + inheritedName + "' but this" + " is not present in " + owner); - } + // TODO: Throw when no one is doing this anymore + if (inheritedName.equals("default")) + logger.logApplicationPackage(WARNING, this + " inherits '" + inheritedName + "', which makes no sense. Remove this inheritance"); + else if (inheritedSummary == null ) + throw new IllegalArgumentException(this + " inherits '" + inheritedName + "', but this is not present in " + owner); } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index a8085919a98..f87f1382ffb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -16,7 +16,7 @@ import java.util.Objects; public final class Host extends TreeConfigProducer<AnyConfigProducer> implements SentinelConfig.Producer, Comparable<Host> { // Memory needed for auxiliary processes always running on the node (config-proxy, metrics-proxy). - // Keep in sync with node-repository/ClusterModel. + // Keep in sync with node-repository/ClusterModel and startup scripts (go and bash). public static final double memoryOverheadGb = 0.7; private ConfigSentinel configSentinel = null; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index e18ffea731e..12892cb12ac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -1,23 +1,18 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.container.Container; /** * Container that should be running on same host as the logserver. Sets up a handler for getting logs from logserver. - * Only in use in hosted Vespa. */ public class LogserverContainer extends Container { public LogserverContainer(TreeConfigProducer<?> parent, DeployState deployState) { super(parent, "" + 0, 0, deployState); - if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) - useDynamicPorts(); } @Override @@ -30,9 +25,4 @@ public class LogserverContainer extends Container { return ""; } - @Override - public String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index e702a29b640..be38298279f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -97,11 +97,6 @@ public class ClusterControllerContainer extends Container implements return ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; } - @Override - public String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - private void configureZooKeeperServer(boolean runStandaloneZooKeeper) { if (runStandaloneZooKeeper) ContainerModelBuilder.addReconfigurableZooKeeperServerComponents(this); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 2ff58438a07..8f262281edc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -10,7 +10,6 @@ import ai.vespa.metricsproxy.rpc.RpcConnector; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServices; import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; @@ -78,11 +77,6 @@ public class MetricsProxyContainer extends Container implements } @Override - public String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - - @Override public int getWantedPort() { return BASEPORT; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java index 03b96b12c03..364048ff261 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java @@ -19,11 +19,13 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem private final Zone zone; private final ApplicationId applicationId; + private final boolean isHostedVespa; public OpenTelemetryCollector(TreeConfigProducer<?> parent) { super(parent, "otelcol"); this.zone = null; this.applicationId = null; + this.isHostedVespa = false; setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -32,6 +34,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem super(parent, "otelcol"); this.zone = deployState.zone(); this.applicationId = deployState.getProperties().applicationId(); + this.isHostedVespa = deployState.isHosted(); setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -54,7 +57,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem @Override public void getConfig(OpenTelemetryConfig.Builder builder) { - var generator = new OpenTelemetryConfigGenerator(zone, applicationId); + var generator = new OpenTelemetryConfigGenerator(zone, applicationId, isHostedVespa); AnyConfigProducer pp = this; AnyConfigProducer p = pp.getParent(); while (p != null && p != pp) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java index 3f7ca7b46a7..67c45d58a95 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.admin.otel; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -11,6 +12,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; @@ -18,6 +20,9 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; +import java.util.Set; +import java.util.TreeSet; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -33,10 +38,12 @@ public class OpenTelemetryConfigGenerator { private List<StatePortInfo> statePorts = new ArrayList<>(); private final Zone zone; private final ApplicationId applicationId; + private final boolean isHostedVespa; - OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId) { + OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId, boolean isHostedVespa) { this.zone = zone; this.applicationId = applicationId; + this.isHostedVespa = isHostedVespa; boolean isCd = true; boolean isPublic = true; if (zone != null) { @@ -101,10 +108,29 @@ public class OpenTelemetryConfigGenerator { g.writeStringField(entry.getKey(), entry.getValue()); } } + String ph = findParentHost(statePort.hostName()); + if (isHostedVespa && ph != null) { + g.writeStringField("parentHostname", ph); + } g.writeEndObject(); } g.writeEndObject(); } + // note: this pattern should match entire node name + static private final Pattern expectedNodeName1 = Pattern.compile("[a-z0-9]+-v6-[0-9]+[.].+"); + static private final Pattern expectedNodeName2 = Pattern.compile("[a-z]*[0-9]+[a-z][.].+"); + // matches the part we want to replace with just a dot + static private final Pattern replaceNodeName1 = Pattern.compile("-v6-[0-9]+[.]"); + static private final Pattern replaceNodeName2 = Pattern.compile("[a-z][.]"); + static String findParentHost(String nodeName) { + if (expectedNodeName1.matcher(nodeName).matches()) { + return replaceNodeName1.matcher(nodeName).replaceFirst("."); + } + if (expectedNodeName2.matcher(nodeName).matches()) { + return replaceNodeName2.matcher(nodeName).replaceFirst("."); + } + return null; + } private void addTls(JsonGenerator g) throws java.io.IOException { g.writeFieldName("tls"); g.writeStartObject(); @@ -145,6 +171,31 @@ public class OpenTelemetryConfigGenerator { g.writeFieldName("processors"); g.writeStartObject(); addResourceProcessor(g); + addRenameProcessor(g); + addFilterProcessor(g); + g.writeEndObject(); + } + private void addRenameProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("metricstransform/rename"); + g.writeStartObject(); + g.writeFieldName("transforms"); + g.writeStartArray(); + var metrics = MetricsConsumer.vespa9.metrics(); + for (var metric : metrics.values()) { + if (! metric.name.equals(metric.outputName)) { + String oldName = PrometheusUtil.sanitize(metric.name); + String newName = PrometheusUtil.sanitize(metric.outputName); + addRenameAction(g, oldName, newName); + } + } + g.writeEndArray(); + g.writeEndObject(); + } + private void addRenameAction(JsonGenerator g, String oldName, String newName) throws java.io.IOException { + g.writeStartObject(); + g.writeStringField("include", oldName); + g.writeStringField("action", "update"); + g.writeStringField("new_name", newName); g.writeEndObject(); } private void addResourceProcessor(JsonGenerator g) throws java.io.IOException { @@ -172,6 +223,35 @@ public class OpenTelemetryConfigGenerator { g.writeStringField("action", "insert"); g.writeEndObject(); } + static private Set<String> wantedMetrics() { + Set<String> result = new TreeSet<>(); + var metrics = MetricsConsumer.vespa9.metrics(); + for (var metric : metrics.values()) { + String oldName = PrometheusUtil.sanitize(metric.name); + String newName = PrometheusUtil.sanitize(metric.outputName); + result.add(oldName); + result.add(newName); + } + return result; + } + private void addFilterProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("filter/metricset"); + g.writeStartObject(); + g.writeFieldName("metrics"); + g.writeStartObject(); + g.writeFieldName("include"); + g.writeStartObject(); + g.writeStringField("match_type", "strict"); + g.writeFieldName("metric_names"); + g.writeStartArray(); + for (String metricName : wantedMetrics()) { + g.writeString(metricName); + } + g.writeEndArray(); + g.writeEndObject(); + g.writeEndObject(); + g.writeEndObject(); + } private void addServiceBlock(JsonGenerator g) throws java.io.IOException { g.writeFieldName("service"); g.writeStartObject(); @@ -208,12 +288,14 @@ public class OpenTelemetryConfigGenerator { } g.writeFieldName("processors"); g.writeStartArray(); + g.writeString("metricstransform/rename"); + g.writeString("filter/metricset"); g.writeString("resource"); g.writeEndArray(); { g.writeFieldName("exporters"); g.writeStartArray(); - g.writeString("file"); + g.writeString("otlphttp/gw"); g.writeEndArray(); } g.writeEndObject(); // metrics @@ -296,16 +378,18 @@ public class OpenTelemetryConfigGenerator { private Map<String, String> serviceAttributes(Service svc) { Map<String, String> dimvals = new LinkedHashMap<>(); - dimvals.put("instance", svc.getServiceName()); // should maybe be "local_service_name" ? - dimvals.put("instanceType", svc.getServiceType()); // maybe "local_service_type", or remove + // currently unused - can be added if necessary: + // dimvals.put("local_service_name", svc.getServiceName()); + // dimvals.put("local_service_type", svc.getServiceType()); String cName = svc.getServicePropertyString("clustername", null); if (cName != null) { - // what about "clusterid" below, is it always the same? - dimvals.put("clustername", cName); + // overridden by cluster membership below (if available) + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cName); } String cType = svc.getServicePropertyString("clustertype", null); if (cType != null) { - dimvals.put("clustertype", cType); + // overridden by cluster membership below (if available) + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cType); } var hostResource = svc.getHost(); if (hostResource != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 9cf5fe84c21..4900b56801c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -23,21 +23,22 @@ public class JvmHeapSizeValidator implements Validator { context.model().getContainerClusters().forEach((clusterId, appCluster) -> { var mp = appCluster.getMemoryPercentage().orElse(null); if (mp == null) return; - if (mp.availableMemoryGb().isEmpty()) { + if (mp.asAbsoluteGb().isEmpty()) { context.deployState().getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); return; } long jvmModelCost = appCluster.onnxModelCostCalculator().aggregatedModelCostInBytes(); if (jvmModelCost > 0) { - double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); + double availableMemoryGb = mp.asAbsoluteGb().getAsDouble(); + int percentageOfTotal = mp.ofContainerTotal().getAsInt(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); context.deployState().getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", - mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); - if (mp.percentage() < percentLimit) { + percentageOfTotal, percentLimit, availableMemoryGb, gbLimit, modelCostGb)); + if (percentageOfTotal < percentLimit) { context.illegal(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", - clusterId, mp.percentage(), percentLimit, modelCostGb)); + clusterId, percentageOfTotal, percentLimit, modelCostGb)); } if (availableMemoryGb < gbLimit) { context.illegal( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 7f624032627..f073c0f27c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForLoc import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForOnnxModelChangesValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import com.yahoo.vespa.model.application.validation.first.RedundancyValidator; import com.yahoo.yolean.Exceptions; @@ -215,6 +216,9 @@ public class Validation { @Override public void require(ConfigChangeAction action) { + if (action instanceof VespaRestartAction && action.getServices().isEmpty()) + throw new IllegalStateException("restart actions must have services specified"); + actions.add(action); action.validationId().ifPresent(id -> invalid(id, action.getMessage())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java index bfd100f40c9..3722dd84e29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.function.Consumer; import java.util.logging.Level; import java.util.stream.Stream; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java index df8bf0e9b01..f86a3b31182 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -38,7 +39,8 @@ public class GlobalDocumentChangeValidator implements ChangeValidator { if ( ! context.deployState().validationOverrides().allows(ValidationId.globalDocumentChange, context.deployState().now())) context.invalid(ValidationId.globalDocumentChange, reason); else if (context.deployState().isHosted()) - context.require(new VespaRestartAction(ClusterSpec.Id.from(clusterName), reason)); + context.require(new VespaRestartAction(ClusterSpec.Id.from(clusterName), reason, + nextCluster.getSearch().getSearchNodes().stream().map(AbstractService::getServiceInfo).toList())); } } }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java index 0d4776ad00a..ce24d11121c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java @@ -36,7 +36,7 @@ public class NodeResourceChangeValidator implements ChangeValidator { } private boolean changeRequiresRestart(NodeResources currentResources, NodeResources nextResources) { - return currentResources.memoryGb() != nextResources.memoryGb(); + return currentResources.memoryGiB() != nextResources.memoryGiB(); } private Optional<NodeResources> resourcesOf(ClusterSpec.Id clusterId, VespaModel model) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 42410dc3acf..a1f8a97d47f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -42,7 +42,7 @@ public class ResourcesReductionValidator implements ChangeValidator { NodeResources currentResources = current.totalResources(); NodeResources nextResources = next.totalResources(); if (nextResources.vcpu() < 0.5 * currentResources.vcpu() || - nextResources.memoryGb() < 0.5 * currentResources.memoryGb() || + nextResources.memoryGiB() < 0.5 * currentResources.memoryGiB() || nextResources.diskGb() < 0.5 * currentResources.diskGb()) context.invalid(ValidationId.resourcesReduction, "Resource reduction in '" + clusterId.value() + "' is too large: " + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java index ccfc611c3dc..ba1f19ff68c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -33,7 +34,9 @@ public class RestartOnDeployForLocalLLMValidator implements ChangeValidator { // Only restart services if we use a local LLM in both the next and previous generation for (var clusterId : intersect(previousClustersWithLocalLLM, nextClustersWithLocalLLM)) { String message = "Need to restart services in %s due to use of local LLM".formatted(clusterId); - context.require(new VespaRestartAction(clusterId, message)); + context.require(new VespaRestartAction(clusterId, message, + context.model().getContainerClusters().get(clusterId.value()).getContainers() + .stream().map(AbstractService::getServiceInfo).toList())); log.log(INFO, message); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java index 008a3fc5547..f95f0a6c079 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainer; @@ -99,7 +100,7 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida log.log(INFO, message); cluster.onnxModelCostCalculator().setRestartOnDeploy(); cluster.onnxModelCostCalculator().store(); - actions.add(new VespaRestartAction(cluster.id(), message)); + actions.add(new VespaRestartAction(cluster.id(), message, cluster.getContainers().stream().map(AbstractService::getServiceInfo).toList())); } private static boolean enoughMemoryToAvoidRestart(ApplicationContainerCluster clusterInCurrentModel, @@ -111,11 +112,11 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida double currentModelCostInGb = onnxModelCostInGb(clusterInCurrentModel); double nextModelCostInGb = onnxModelCostInGb(cluster); - double totalMemory = containers.stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); + double totalMemory = containers.stream().mapToDouble(c -> c.getHostResource().realResources().memoryGiB()).min().orElseThrow(); double memoryUsedByModels = currentModelCostInGb + nextModelCostInGb; double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - memoryUsedByModels); - var availableMemoryPercentage = cluster.availableMemoryPercentage(); + var availableMemoryPercentage = cluster.heapSizePercentageOfAvailable(); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); var prefix = "Validating Onnx models memory usage for %s".formatted(cluster); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java index c0b55d856ab..e36fa5b7373 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java @@ -16,6 +16,7 @@ public class VespaRestartAction extends VespaConfigChangeAction implements Confi private final boolean ignoreForInternalRedeploy; + /** <strong>This does <em>not</em> trigger restarts; you <em>need</em> the {@code ServiceInfo}!</strong>*/ public VespaRestartAction(ClusterSpec.Id id, String message) { this(id, message, List.of()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 692de1769d3..34df7e9d963 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -12,15 +12,18 @@ import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; import com.yahoo.vespa.model.admin.Logserver; +import com.yahoo.vespa.model.admin.LogserverContainer; +import com.yahoo.vespa.model.admin.LogserverContainerCluster; import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; import com.yahoo.vespa.model.admin.otel.OpenTelemetryCollector; -import com.yahoo.vespa.model.admin.otel.OpenTelemetryConfigGenerator; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilderBase; import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerModel; import org.w3c.dom.Element; + import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -35,10 +38,13 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private static final String ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK = "standalone-zookeeper"; - public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, + private final ConfigModelContext context; + + public DomAdminV2Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { - super(applicationType, multitenant, configServerSpecs); + super(context.getApplicationType(), multitenant, configServerSpecs); + this.context = context; } private void addOtelcol(TreeConfigProducer<?> parent, DeployState deployState, HostResource hostResource) { @@ -52,6 +58,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { List<Configserver> configservers = parseConfigservers(deployState, admin, adminE); var logserver = parseLogserver(deployState, admin, adminE); admin.setLogserver(logserver); + createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); if (deployState.featureFlags().logserverOtelCol()) { // for manual testing addOtelcol(admin, deployState, logserver.getHostResource()); @@ -65,6 +72,21 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { addLoggingSpecs(new ModelElement(adminE).child("logging"), admin); } + private void createContainerOnLogserverHost(DeployState deployState, Admin admin, HostResource hostResource) { + LogserverContainerCluster logServerCluster = new LogserverContainerCluster(admin, "logs", deployState); + ContainerModel logserverClusterModel = new ContainerModel(context.withParent(admin).withId(logServerCluster.getSubId())); + logserverClusterModel.setCluster(logServerCluster); + + LogserverContainer container = new LogserverContainer(logServerCluster, deployState); + container.useDynamicPorts(); + container.setHostResource(hostResource); + container.initService(deployState); + logServerCluster.addContainer(container); + admin.addAndInitializeService(deployState, hostResource, container); + admin.setLogserverContainerCluster(logServerCluster); + context.getConfigModelRepoAdder().add(logserverClusterModel); + } + private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) { List<Configserver> configservers; if (multitenant) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 347bb504857..c6086327a1c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -111,6 +111,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { logserverClusterModel.setCluster(logServerCluster); LogserverContainer container = new LogserverContainer(logServerCluster, deployState); + if (deployState.getProperties().applicationId().instance().isTester()) + container.useDynamicPorts(); // TODO: read current version in ApplicationRepository, and always use this. container.setHostResource(hostResource); container.initService(deployState); logServerCluster.addContainer(container); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index d877600db13..11f4c9794aa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -137,7 +137,7 @@ public class NodesSpecification { int defaultMinGroups = nodes.from().orElse(1) / groupSize.to().orElse(nodes.from().orElse(1)); int defaultMaxGroups = groupSize.isEmpty() ? 1 : nodes.to().orElse(1) / groupSize.from().orElse(1); - var min = new ClusterResources(nodes.from().orElse(1), groups.from().orElse(defaultMinGroups), nodeResources(nodesElement).getFirst()); + var min = new ClusterResources(nodes.from().orElse(1), groups.from().orElse(defaultMinGroups), nodeResources(nodesElement).getFirst()); var max = new ClusterResources(nodes.to().orElse(1), groups.to().orElse(defaultMaxGroups), nodeResources(nodesElement).getSecond()); return new ResourceConstraints(min, max, groupSize); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index ed7646b3066..014d183071d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -209,22 +209,27 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage)); if (isHostedVespa()) { - int availableMemoryPercentage = availableMemoryPercentage(); - if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known - - // Node memory is known so convert available memory percentage to node memory percentage - double totalMemory = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); - double jvmHeapDeductionGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); - double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); - int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); - logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" - .formatted(id(), memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); - return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); + int heapSizePercentageOfAvailable = heapSizePercentageOfAvailable(); + if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(heapSizePercentageOfAvailable)); // Node memory is not known + + // Node memory is known, so compute heap size as a percentage of available memory (excluding overhead, which the startup scripts also account for) + double totalMemoryGb = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGiB()).min().orElseThrow(); + double totalMemoryMinusOverhead = Math.max(0, totalMemoryGb - Host.memoryOverheadGb); + double onnxModelCostGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); + double availableMemoryGb = Math.max(0, totalMemoryMinusOverhead - onnxModelCostGb); + int memoryPercentageOfAvailable = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryMinusOverhead); + int memoryPercentageOfTotal = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryGb); + logger.log(FINE, () -> ("cluster id '%s': memoryPercentageOfAvailable=%d, memoryPercentageOfTotal=%d, " + + "availableMemoryGb=%f, totalMemoryGb=%f, heapSizePercentageOfAvailable=%d, onnxModelCostGb=%f") + .formatted(id(), memoryPercentageOfAvailable, memoryPercentageOfTotal, + availableMemoryGb, totalMemoryGb, heapSizePercentageOfAvailable, onnxModelCostGb)); + return Optional.of(JvmMemoryPercentage.of(memoryPercentageOfAvailable, memoryPercentageOfTotal, + availableMemoryGb * heapSizePercentageOfAvailable * 1e-2)); } return Optional.empty(); } - public int availableMemoryPercentage() { + public int heapSizePercentageOfAvailable() { return getHostClusterId().isPresent() ? heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : heapSizePercentageOfAvailableMemory; @@ -310,14 +315,23 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); var memoryPct = getMemoryPercentage().orElse(null); - int heapsize = memoryPct != null && memoryPct.availableMemoryGb().isPresent() - ? (int) (memoryPct.availableMemoryGb().getAsDouble() * 1024) : 1536; + int heapsize = truncateTo4SignificantBits(memoryPct != null && memoryPct.asAbsoluteGb().isPresent() + ? (int) (memoryPct.asAbsoluteGb().getAsDouble() * 1024) : 1536); builder.jvm.verbosegc(true) .availableProcessors(0) .compressedClassSpaceSize(0) - .minHeapsize(heapsize) + .minHeapsize(heapsize) // These cause restarts when changed, so we try to keep them stable. .heapsize(heapsize); - if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.percentage()); + if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.ofContainerAvailable()); + } + + static int truncateTo4SignificantBits(int i) { + if (i == Integer.MIN_VALUE) return i; + if (i < 0) return -truncateTo4SignificantBits(-i); + if (i <= 16) return i; + int mask = Integer.highestOneBit(i); + mask += mask - (mask >> 3); + return i & mask; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 864cbc8691b..c2368fd29a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -108,10 +108,6 @@ public abstract class Container extends AbstractService implements addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x1000000"); // 16M } - public String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); - } - void setOwner(ContainerCluster<?> owner) { this.owner = owner; } /** True if this container is retired (slated for removal) */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index aeb6c030a49..00ab47e8ddb 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -73,6 +73,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; +import java.util.OptionalInt; import java.util.Set; import java.util.TreeSet; @@ -721,10 +722,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> * Returns the percentage of host physical memory this application has specified for nodes in this cluster, * or empty if this is not specified by the application. */ - public record JvmMemoryPercentage(int percentage, OptionalDouble availableMemoryGb) { - static JvmMemoryPercentage of(int percentage) { return new JvmMemoryPercentage(percentage, OptionalDouble.empty()); } - static JvmMemoryPercentage of(int percentage, double availableMemoryGb) { - return new JvmMemoryPercentage(percentage, OptionalDouble.of(availableMemoryGb)); + public record JvmMemoryPercentage(int ofContainerAvailable, OptionalInt ofContainerTotal, OptionalDouble asAbsoluteGb) { // optionalInt pctOfTotal < int pctOfAvailable + static JvmMemoryPercentage of(int percentageOfAvailable) { return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.empty(), OptionalDouble.empty()); } + static JvmMemoryPercentage of(int percentageOfAvailable, int percentageOfTotal, double absoluteMemoryGb) { + return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.of(percentageOfTotal), OptionalDouble.of(absoluteMemoryGb)); } } public Optional<JvmMemoryPercentage> getMemoryPercentage() { return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 5f824950ecd..9c9e20062f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http.ssl; +import ai.vespa.utils.BytesQuantity; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.security.tls.TlsContext; @@ -21,6 +22,8 @@ import java.util.TreeSet; */ public class HostedSslConnectorFactory extends ConnectorFactory { + private record EntityLoggingEntry(String prefix, double sampleRate, BytesQuantity maxEntitySize) {} + private final SslClientAuth clientAuth; private final List<String> tlsCiphersOverride; private final boolean proxyProtocolEnabled; @@ -28,6 +31,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final List<String> remoteAddressHeaders; private final List<String> remotePortHeaders; private final Set<String> knownServerNames; + private final List<EntityLoggingEntry> entityLoggingEntries; public static Builder builder(String name, int listenPort) { return new Builder(name, listenPort); } @@ -40,6 +44,22 @@ public class HostedSslConnectorFactory extends ConnectorFactory { this.remoteAddressHeaders = List.copyOf(builder.remoteAddressHeaders); this.remotePortHeaders = List.copyOf(builder.remotePortHeaders); this.knownServerNames = Collections.unmodifiableSet(new TreeSet<>(builder.knownServerNames)); + this.entityLoggingEntries = builder.requestPrefixForLoggingContent.stream() + .map(prefix -> { + var parts = prefix.split(":"); + if (parts.length != 3) { + throw new IllegalArgumentException("Expected string of format 'prefix:sample-rate:max-entity-size', got '%s'".formatted(prefix)); + } + var pathPrefix = parts[0]; + if (pathPrefix.isBlank()) + throw new IllegalArgumentException("Path prefix must not be blank"); + var sampleRate = Double.parseDouble(parts[1]); + if (sampleRate < 0 || sampleRate > 1) + throw new IllegalArgumentException("Sample rate must be in range [0, 1], got '%s'".formatted(sampleRate)); + var maxEntitySize = BytesQuantity.fromString(parts[2]); + return new EntityLoggingEntry(pathPrefix, sampleRate, maxEntitySize); + }) + .toList(); } private static SslProvider createSslProvider(Builder builder) { @@ -72,8 +92,14 @@ public class HostedSslConnectorFactory extends ConnectorFactory { .idleTimeout(Duration.ofSeconds(30).toSeconds()) .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0) .accessLog(new ConnectorConfig.AccessLog.Builder() - .remoteAddressHeaders(remoteAddressHeaders) - .remotePortHeaders(remotePortHeaders)) + .remoteAddressHeaders(remoteAddressHeaders) + .remotePortHeaders(remotePortHeaders) + .content(entityLoggingEntries.stream() + .map(e -> new ConnectorConfig.AccessLog.Content.Builder() + .pathPrefix(e.prefix) + .sampleRate(e.sampleRate) + .maxSize(e.maxEntitySize.toBytes())) + .toList())) .serverName.known(knownServerNames); } @@ -93,6 +119,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { String tlsCaCertificatesPath; boolean tokenEndpoint; Set<String> knownServerNames = Set.of(); + Set<String> requestPrefixForLoggingContent = Set.of(); private Builder(String name, int port) { this.name = name; this.port = port; } public Builder clientAuth(SslClientAuth auth) { clientAuth = auth; return this; } @@ -106,6 +133,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { public Builder remoteAddressHeader(String header) { this.remoteAddressHeaders.add(header); return this; } public Builder remotePortHeader(String header) { this.remotePortHeaders.add(header); return this; } public Builder knownServerNames(Set<String> knownServerNames) { this.knownServerNames = Set.copyOf(knownServerNames); return this; } + public Builder requestPrefixForLoggingContent(Collection<String> v) { this.requestPrefixForLoggingContent = Set.copyOf(v); return this; } public HostedSslConnectorFactory build() { return new HostedSslConnectorFactory(this); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index dbe28d48f9e..4995c20b985 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -140,6 +140,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Default path to vip status file for container in Hosted Vespa. static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().underVespaHome("var/vespa/load-balancer/status.html"); + static final String HOSTED_VESPA_TENANT_PARENT_DOMAIN = "vespa.tenant."; + //Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE private static final String HOSTED_VESPA_STATUS_FILE_SETTING = "VESPA_LB_STATUS_FILE"; @@ -235,6 +237,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Must be added after nodes: addDeploymentSpecConfig(cluster, context, deployState.getDeployLogger()); addZooKeeper(cluster, spec); + addAthenzServiceIdentityProvider(cluster, context, deployState.getDeployLogger()); addParameterStoreValidationHandler(cluster, deployState); } @@ -344,6 +347,20 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addComponent(cloudSecretStore); } + private void addAthenzServiceIdentityProvider(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { + if ( ! context.getDeployState().isHosted()) return; + if ( ! context.getDeployState().zone().system().isPublic()) return; // Non-public is handled by deployment spec config. + if ( ! context.properties().launchApplicationAthenzService()) return; + addIdentityProvider(cluster, + context.getDeployState().getProperties().configServerSpecs(), + context.getDeployState().getProperties().loadBalancerName(), + context.getDeployState().getProperties().ztsUrl(), + context.getDeployState().getProperties().athenzDnsSuffix(), + context.getDeployState().zone(), + AthenzDomain.from(HOSTED_VESPA_TENANT_PARENT_DOMAIN + context.properties().applicationId().tenant().value()), + AthenzService.from(context.properties().applicationId().application().value())); + } + private void addDeploymentSpecConfig(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { if ( ! context.getDeployState().isHosted()) return; DeploymentSpec deploymentSpec = app.getDeploymentSpec(); @@ -607,7 +624,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { var builder = HostedSslConnectorFactory.builder(serverName, getMtlsDataplanePort(state)) .proxyProtocol(state.zone().cloud().useProxyProtocol()) .tlsCiphersOverride(state.getProperties().tlsCiphersOverride()) - .endpointConnectionTtl(state.getProperties().endpointConnectionTtl()); + .endpointConnectionTtl(state.getProperties().endpointConnectionTtl()) + .requestPrefixForLoggingContent(state.getProperties().requestPrefixForLoggingContent()); var endpointCert = state.endpointCertificateSecrets().orElse(null); if (endpointCert != null) { builder.endpointCertificate(endpointCert); @@ -670,6 +688,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .remotePortHeader("X-Forwarded-Port") .clientAuth(SslClientAuth.NEED) .knownServerNames(tokenEndpoints) + .requestPrefixForLoggingContent(state.getProperties().requestPrefixForLoggingContent()) .build(); server.addConnector(connector); @@ -815,8 +834,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { !container.getHostResource().realResources().gpuResources().isZero()); onnxModel.setGpuDevice(gpuDevice, hasGpu); } - cluster.onnxModelCostCalculator().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); } + for (OnnxModel onnxModel : models.asMap().values()) + cluster.onnxModelCostCalculator().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles, models)); } @@ -986,8 +1006,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { AbstractService.distributeCpuSocketAffinity(nodes); cluster.addContainers(nodes); } - // Must be done after setting Jvm options from services.xml (#extractJvmOptions()), otherwise those options will not be set - cluster.getContainers().forEach(container -> container.appendJvmOptions(container.jvmOmitStackTraceInFastThrowOption(context.featureFlags()))); } private ZoneEndpoint zoneEndpoint(ConfigModelContext context, ClusterSpec.Id cluster) { @@ -1039,7 +1057,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } catch (NumberFormatException e) { throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster + - " must be an integer percentage ending by the '%' sign", e); + " must be given as an integer followe by '%'", e); } } @@ -1074,9 +1092,21 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return List.of(node); } + private static void requireFixedSizeSingularNodeIfTester(ConfigModelContext context, NodesSpecification nodes) { + if ( ! context.properties().hostedVespa() || ! context.properties().applicationId().instance().isTester()) + return; + + if ( ! nodes.maxResources().equals(nodes.minResources())) + throw new IllegalArgumentException("tester resources must be absolute, but min and max resources differ: " + nodes); + + if (nodes.maxResources().nodes() > 1) + throw new IllegalArgumentException("tester cannot run on more than 1 node, but " + nodes.maxResources().nodes() + " nodes were specified"); + } + private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { try { var nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + requireFixedSizeSingularNodeIfTester(context, nodesSpecification); var clusterId = ClusterSpec.Id.from(cluster.name()); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, @@ -1282,37 +1312,36 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private void addIdentityProvider(ApplicationContainerCluster cluster, - List<ConfigServerSpec> configServerSpecs, - HostName loadBalancerName, - URI ztsUrl, - String athenzDnsSuffix, - Zone zone, - DeploymentSpec spec) { - spec.athenzDomain() - .ifPresent(domain -> { - AthenzService service = spec.athenzService(app.getApplicationId().instance(), zone.environment(), zone.region()) - .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + - app.getApplicationId().instance() + "'")); - String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; - IdentityProvider identityProvider = new IdentityProvider(domain, - service, - getLoadBalancerName(loadBalancerName, configServerSpecs), - ztsUrl, - zoneDnsSuffix, - zone); - - // Replace AthenzIdentityProviderProvider - cluster.removeComponent(ComponentId.fromString("com.yahoo.container.jdisc.AthenzIdentityProviderProvider")); - cluster.addComponent(identityProvider); - - var serviceIdentityProviderProvider = "com.yahoo.vespa.athenz.identityprovider.client.ServiceIdentityProviderProvider"; - cluster.addComponent(new SimpleComponent(new ComponentModel(serviceIdentityProviderProvider, serviceIdentityProviderProvider, "vespa-athenz"))); - - cluster.getContainers().forEach(container -> { - container.setProp("identity.domain", domain.value()); - container.setProp("identity.service", service.value()); - }); + private void addIdentityProvider(ApplicationContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, + URI ztsUrl, String athenzDnsSuffix, Zone zone, DeploymentSpec spec) { + spec.athenzDomain().ifPresent(domain -> { + AthenzService service = spec.athenzService(app.getApplicationId().instance(), zone.environment(), zone.region()) + .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + + app.getApplicationId().instance() + "'")); + addIdentityProvider(cluster, configServerSpecs, loadBalancerName, ztsUrl, athenzDnsSuffix, zone, domain, service); + }); + } + + private void addIdentityProvider(ApplicationContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, + URI ztsUrl, String athenzDnsSuffix, Zone zone, AthenzDomain domain, AthenzService service) { + String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; + IdentityProvider identityProvider = new IdentityProvider(domain, + service, + getLoadBalancerName(loadBalancerName, configServerSpecs), + ztsUrl, + zoneDnsSuffix, + zone); + + // Replace AthenzIdentityProviderProvider + cluster.removeComponent(ComponentId.fromString("com.yahoo.container.jdisc.AthenzIdentityProviderProvider")); + cluster.addComponent(identityProvider); + + var serviceIdentityProviderProvider = "com.yahoo.vespa.athenz.identityprovider.client.ServiceIdentityProviderProvider"; + cluster.addComponent(new SimpleComponent(new ComponentModel(serviceIdentityProviderProvider, serviceIdentityProviderProvider, "vespa-athenz"))); + + cluster.getContainers().forEach(container -> { + container.setProp("identity.domain", domain.value()); + container.setProp("identity.service", service.value()); }); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index fbac9e9d710..994837a2dbe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -237,24 +236,17 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> NodeSpec spec = getNextSearchNodeSpec(parentGroup); SearchNode searchNode; - TransactionLogServer tls; if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags(), syncTransactionLog); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); - - tls = new TransactionLogServer(searchNode, clusterName, syncTransactionLog); - tls.setHostResource(searchNode.getHostResource()); - tls.initService(deployState); } else { - searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, - tuning, resourceLimits, fractionOfMemoryReserved) + searchNode = new SearchNode.Builder("" + node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, + tuning, resourceLimits, fractionOfMemoryReserved, syncTransactionLog) .build(deployState, parent, element.getXml()); - tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } - searchNode.setTls(tls); if (searchCluster != null) { searchCluster.addSearcher(searchNode); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index c182f0e0507..f218cedb74e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -35,6 +35,9 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen private final boolean hasIndexedDocumentType; private final int maxActivationInhibitedOutOfSyncGroups; private final int contentLayerMetadataFeatureLevel; + private final boolean symmetricPutAndActivateReplicaSelection; + private final boolean enforceStrictlyIncreasingClusterStateVersions; + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<DistributorCluster> { ContentCluster parent; @@ -97,19 +100,25 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen var featureFlags = deployState.getProperties().featureFlags(); int maxInhibitedGroups = featureFlags.maxActivationInhibitedOutOfSyncGroups(); int contentLayerMetadataFeatureLevel = featureFlags.contentLayerMetadataFeatureLevel(); + boolean symmetricPutAndActivateReplicaSelection = featureFlags.symmetricPutAndActivateReplicaSelection(); + boolean enforceStrictlyIncreasingClusterStateVersions = featureFlags.enforceStrictlyIncreasingClusterStateVersions(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, maxInhibitedGroups, - contentLayerMetadataFeatureLevel); + contentLayerMetadataFeatureLevel, + symmetricPutAndActivateReplicaSelection, + enforceStrictlyIncreasingClusterStateVersions); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, int maxActivationInhibitedOutOfSyncGroups, - int contentLayerMetadataFeatureLevel) + int contentLayerMetadataFeatureLevel, + boolean symmetricPutAndActivateReplicaSelection, + boolean enforceStrictlyIncreasingClusterStateVersions) { super(parent, "distributor"); this.parent = parent; @@ -118,6 +127,8 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen this.hasIndexedDocumentType = hasIndexedDocumentType; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.contentLayerMetadataFeatureLevel = contentLayerMetadataFeatureLevel; + this.symmetricPutAndActivateReplicaSelection = symmetricPutAndActivateReplicaSelection; + this.enforceStrictlyIncreasingClusterStateVersions = enforceStrictlyIncreasingClusterStateVersions; } @Override @@ -132,6 +143,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen if (contentLayerMetadataFeatureLevel > 0) { builder.enable_operation_cancellation(true); } + builder.symmetric_put_and_activate_replica_selection(symmetricPutAndActivateReplicaSelection); bucketSplitting.getConfig(builder); } @@ -152,6 +164,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen builder.root_folder(""); builder.cluster_name(parent.getName()); builder.is_distributor(true); + builder.require_strictly_increasing_cluster_state_versions(enforceStrictlyIncreasingClusterStateVersions); } public String getClusterName() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index bac86e37e8f..d37e5d5382a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -46,6 +46,7 @@ import com.yahoo.vespa.model.content.IndexedHierarchicDistributionValidator; import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.content.ReservedDocumentTypeNameValidator; import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; import com.yahoo.vespa.model.content.engines.PersistenceEngine; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; @@ -137,6 +138,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, c, c.search.isStreaming()); c.clusterControllerConfig = createClusterControllerConfig(contentElement, deployState, c, resourceLimits); validateThatGroupSiblingsAreUnique(c.clusterId, c.rootGroup); + warnIfDistributionKeyRangeIsSuboptimal(c.clusterId, c.rootGroup, deployState); c.search.handleRedundancy(c.redundancy); setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); @@ -247,7 +249,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem for (ContainerModel containerModel : containers) { Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId(); if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { - return containerModel.getCluster().getMemoryPercentage().get().percentage() * 0.01; + return containerModel.getCluster().getMemoryPercentage().get().ofContainerAvailable() * 0.01; } } return 0.0; @@ -275,6 +277,38 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem } } + private static class HighestDistributionKeyAggregator { + public int nodeCount = 0; + public int highestNodeDistributionKey = 0; + + void aggregateNodeStats(StorageGroup group) { + for (StorageNode n : group.getNodes()) { + nodeCount++; + highestNodeDistributionKey = Math.max(highestNodeDistributionKey, n.getDistributionKey()); + } + for (StorageGroup g : group.getSubgroups()) { + aggregateNodeStats(g); + } + } + } + + private void warnIfDistributionKeyRangeIsSuboptimal(String clusterId, StorageGroup rootGroup, DeployState deployState) { + if (rootGroup == null) { + return; // Unit testing case + } + var aggr = new HighestDistributionKeyAggregator(); + aggr.aggregateNodeStats(rootGroup); + int warnThreshold = 100; // ... Not scientifically chosen + if ((aggr.highestNodeDistributionKey - aggr.nodeCount) >= warnThreshold) { + deployState.getDeployLogger().logApplicationPackage(WARNING, + ("Content cluster '%s' has %d node(s), but the highest distribution key is %d. " + + "Having much higher distribution keys than the number of nodes is not recommended, " + + "as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node") + .formatted(clusterId, aggr.nodeCount, aggr.highestNodeDistributionKey)); + } + } + private void addClusterControllers(ConfigModelContext context, ModelElement contentElement, ContentCluster contentCluster, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index 701da93a329..8bde038b160 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -31,13 +31,15 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> protected StorageCluster doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { final ModelElement clusterElem = new ModelElement(producerSpec); final ContentCluster cluster = (ContentCluster)ancestor; + var featureFlags = deployState.getProperties().featureFlags(); return new StorageCluster(ancestor, ContentCluster.getClusterId(clusterElem), new FileStorProducer.Builder().build(deployState.getProperties(), cluster, clusterElem), new StorServerProducer.Builder().build(deployState.getProperties(), clusterElem), new StorVisitorProducer.Builder().build(clusterElem), - new PersistenceProducer.Builder().build(clusterElem)); + new PersistenceProducer.Builder().build(clusterElem), + featureFlags.enforceStrictlyIncreasingClusterStateVersions()); } } @@ -46,19 +48,22 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> private final StorServerProducer storServerProducer; private final StorVisitorProducer storVisitorProducer; private final PersistenceProducer persistenceProducer; + private final boolean enforceStrictlyIncreasingClusterStateVersions; StorageCluster(TreeConfigProducer<?> parent, String clusterName, FileStorProducer fileStorProducer, StorServerProducer storServerProducer, StorVisitorProducer storVisitorProducer, - PersistenceProducer persistenceProducer) { + PersistenceProducer persistenceProducer, + boolean enforceStrictlyIncreasingClusterStateVersions) { super(parent, "storage"); this.clusterName = clusterName; this.fileStorProducer = fileStorProducer; this.storServerProducer = storServerProducer; this.storVisitorProducer = storVisitorProducer; this.persistenceProducer = persistenceProducer; + this.enforceStrictlyIncreasingClusterStateVersions = enforceStrictlyIncreasingClusterStateVersions; } @Override @@ -85,6 +90,7 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> @Override public void getConfig(StorServerConfig.Builder builder) { storServerProducer.getConfig(builder); + builder.require_strictly_increasing_cluster_state_versions(enforceStrictlyIncreasingClusterStateVersions); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 2616dd8a93c..d60e6ac2df6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.content.Redundancy; import static java.lang.Long.min; import static java.lang.Long.max; @@ -21,8 +20,9 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08; private final static double MIN_MEMORY_PER_FLUSH_THREAD_GB = 11.0; private final static double TLS_SIZE_FRACTION = 0.02; - final static long MB = 1024 * 1024; - public final static long GB = MB * 1024; + final static long MiB = 1024 * 1024; + public final static long GiB = MiB * 1024; + public final static long GB = 1_000_000_000; private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; @@ -50,14 +50,14 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { - long memoryLimitBytes = (long) ((usableMemoryGb() * SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() * SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY) * GiB); builder.maxbytes(memoryLimitBytes); } private void setHwInfo(ProtonConfig.Builder builder) { builder.hwinfo.disk.shared(true); builder.hwinfo.cpu.cores((int)resources.vcpu()); - builder.hwinfo.memory.size((long)(usableMemoryGb() * GB)); + builder.hwinfo.memory.size((long)(usableMemoryGb() * GiB)); builder.hwinfo.disk.size((long)(resources.diskGb() * GB)); } @@ -68,12 +68,12 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) { - long fileSizeBytes = (long) Math.max(256*MB, usableMemoryGb()*GB*SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY); + long fileSizeBytes = (long) Math.max(256* MiB, usableMemoryGb()* GiB *SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY); builder.maxfilesize(fileSizeBytes); } private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) { - long memoryLimitBytes = (long) ((usableMemoryGb() * MEMORY_GAIN_AS_FRACTION_OF_MEMORY) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() * MEMORY_GAIN_AS_FRACTION_OF_MEMORY) * GiB); builder.maxmemory(memoryLimitBytes); builder.each.maxmemory(memoryLimitBytes); } @@ -89,7 +89,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { long tlsSizeBytes = (long) ((resources.diskGb() * TLS_SIZE_FRACTION) * GB); - tlsSizeBytes = max(2*GB, min(tlsSizeBytes, 100 * GB)); + tlsSizeBytes = max(2* GB, min(tlsSizeBytes, 100 * GB)); builder.maxtlssize(tlsSizeBytes); } @@ -114,7 +114,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { /** Returns the memory we can expect will be available for the content node processes */ private double usableMemoryGb() { - double usableMemoryGb = resources.memoryGb() - Host.memoryOverheadGb; + double usableMemoryGb = resources.memoryGiB() - Host.memoryOverheadGb; return usableMemoryGb * (1 - fractionOfMemoryReserved); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 61933c10504..08743290ae3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -55,17 +55,18 @@ public class SearchNode extends AbstractService implements private static final int UNUSED_2 = 2; private static final int UNUSED_3 = 3; private static final int HEALTH_PORT = 4; + private static final int TLS_PORT = 5; private final boolean isHostedVespa; private final boolean flushOnShutdown; private final NodeSpec nodeSpec; private final int distributionKey; private final String clusterName; - private TransactionLogServer tls; private final AbstractService serviceLayerService; private final Tuning tuning; private final ResourceLimits resourceLimits; private final double fractionOfMemoryReserved; + private final Boolean syncTransactionLog; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<SearchNode> { @@ -77,10 +78,11 @@ public class SearchNode extends AbstractService implements private final Tuning tuning; private final ResourceLimits resourceLimits; private final double fractionOfMemoryReserved; + private final Boolean syncTransactionLog; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Boolean syncTransactionLog) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -89,6 +91,7 @@ public class SearchNode extends AbstractService implements this.tuning = tuning; this.resourceLimits = resourceLimits; this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.syncTransactionLog = syncTransactionLog; } @Override @@ -96,7 +99,7 @@ public class SearchNode extends AbstractService implements Element producerSpec) { return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags(), syncTransactionLog); } } @@ -105,9 +108,9 @@ public class SearchNode extends AbstractService implements String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, boolean isHostedVespa, double fractionOfMemoryReserved, - ModelContext.FeatureFlags featureFlags) { + ModelContext.FeatureFlags featureFlags, Boolean syncTransactionLog) { SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, syncTransactionLog); if (featureFlags.loadCodeAsHugePages()) { node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true); } @@ -120,7 +123,7 @@ public class SearchNode extends AbstractService implements private SearchNode(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Boolean syncTransactionLog) { super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; @@ -134,9 +137,11 @@ public class SearchNode extends AbstractService implements portsMeta.on(UNUSED_2).tag("unused"); portsMeta.on(UNUSED_3).tag("unused"); portsMeta.on(HEALTH_PORT).tag("http").tag("json").tag("health").tag("state"); + portsMeta.on(TLS_PORT).tag("tls"); // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; + this.syncTransactionLog = syncTransactionLog; setPropertiesElastic(clusterName, distributionKey); addEnvironmentVariable("OMP_NUM_THREADS", 1); } @@ -172,6 +177,7 @@ public class SearchNode extends AbstractService implements from.allocatePort("unused/2"); from.allocatePort("unused/3"); from.allocatePort("health"); + from.allocatePort("tls"); } /** @@ -181,7 +187,7 @@ public class SearchNode extends AbstractService implements */ @Override public int getPortCount() { - return 5; + return 6; } /** @@ -198,6 +204,8 @@ public class SearchNode extends AbstractService implements return getHttpPort(); } + int getTlsPort() { return getRelativePort(TLS_PORT); } + @Override public String getServiceType() { return "searchnode"; @@ -219,7 +227,10 @@ public class SearchNode extends AbstractService implements builder.usefsync(false); } } - tls.getConfig(builder); + builder.listenport(getTlsPort()) + .basedir(getTlsDir()); + if (syncTransactionLog != null) + builder.usefsync(syncTransactionLog); } @Override @@ -227,14 +238,6 @@ public class SearchNode extends AbstractService implements return getHostName(); } - private TransactionLogServer getTransactionLogServer() { - return tls; - } - - public void setTls(TransactionLogServer tls) { - this.tls = tls; - } - public AbstractService getServiceLayerService() { return serviceLayerService; } @@ -260,7 +263,7 @@ public class SearchNode extends AbstractService implements httpport(getHttpPort()). clustername(getClusterName()). basedir(getBaseDir()). - tlsspec("tcp/" + getHost().getHostname() + ":" + getTransactionLogServer().getTlsPort()). + tlsspec("tcp/" + getHost().getHostname() + ":" + getTlsPort()). tlsconfigid(getConfigId()). slobrokconfigid(getClusterConfigId()). routingconfigid(getClusterConfigId()). @@ -305,6 +308,8 @@ public class SearchNode extends AbstractService implements new MetricsmanagerConfig.Consumer.Builder().name("log").tags("logdefault")); } + private String getTlsDir() { return "tls";} + @Override public Optional<String> getPreShutdownCommand() { if (flushOnShutdown) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java deleted file mode 100644 index 5617fd15cbc..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchlib.TranslogserverConfig; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.PortAllocBridge; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import org.w3c.dom.Element; - -/** - * @author hmusum - */ -public class TransactionLogServer extends AbstractService { - - private final Boolean useFsync; - - public TransactionLogServer(TreeConfigProducer<?> searchNode, String clusterName, Boolean useFsync) { - super(searchNode, "transactionlogserver"); - portsMeta.on(0).tag("tls"); - this.useFsync = useFsync; - setProp("clustername", clusterName); - setProp("clustertype", "search"); - } - - public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<TransactionLogServer> { - - private final String clusterName; - private final Boolean useFsync; - public Builder(String clusterName, Boolean useFsync) { - this.clusterName = clusterName; - this.useFsync = useFsync; - } - - @Override - protected TransactionLogServer doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { - return new TransactionLogServer(ancestor, clusterName, useFsync); - } - - } - - public int getPortCount() { - return 1; - } - - @Override - public void allocatePorts(int start, PortAllocBridge from) { - // NB: ignore "start" - from.allocatePort("tls"); - } - - /** - * Returns the port used by the TLS. - * - * @return The port. - */ - int getTlsPort() { - return getRelativePort(0); - } - - /** - * Returns the directory used by the TLS. - * - * @return The directory. - */ - private String getTlsDir() { - return "tls"; - } - - public void getConfig(TranslogserverConfig.Builder builder) { - builder.listenport(getTlsPort()) - .basedir(getTlsDir()); - if (useFsync != null) { - builder.usefsync(useFsync); - } - } - -} diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 1365c133932..c9eff88764f 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -1756,6 +1756,7 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | fieldRankFilter(profile) | firstPhase(profile) | matchPhase(profile) + | diversity(profile) | function(profile) | mutate(profile) | ignoreRankFeatures(profile) @@ -1875,14 +1876,14 @@ void matchPhase(ParsedRankProfile profile) : MatchPhaseSettings settings = new MatchPhaseSettings(); } { - <MATCH_PHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE> + <MATCH_PHASE> lbrace() (matchPhaseItem(profile, settings) (<NL>)*)* <RBRACE> { settings.checkValid(); - profile.setMatchPhaseSettings(settings); + profile.setMatchPhase(settings); } } -void matchPhaseItem(MatchPhaseSettings settings) : +void matchPhaseItem(ParsedRankProfile profile, MatchPhaseSettings settings) : { String str; int num; @@ -1891,7 +1892,7 @@ void matchPhaseItem(MatchPhaseSettings settings) : } { ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); } - | diversity(settings) + | diversityDeprecated(profile) | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } | <DESCENDING> { settings.setAscending(false); } ) | <MAX_HITS> <COLON> num = integer() { settings.setMaxHits(num); } @@ -1906,7 +1907,7 @@ void matchPhaseItem(MatchPhaseSettings settings) : * * @param profile The rank profile to modify. */ -void diversity(MatchPhaseSettings profile) : +void diversity(ParsedRankProfile profile) : { DiversitySettings settings = new DiversitySettings(); } @@ -1917,6 +1918,18 @@ void diversity(MatchPhaseSettings profile) : } } +void diversityDeprecated(ParsedRankProfile profile) : +{ + DiversitySettings settings = new DiversitySettings(); +} +{ + <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE> + { + profile.setDiversity(settings); + deployLogger.logApplicationPackage(Level.WARNING, "'diversity is deprecated inside 'match-phase'. Specify it at 'rank-profile' level."); + } +} + void diversityItem(DiversitySettings settings) : { String str; @@ -1981,10 +1994,12 @@ void secondPhaseItem(ParsedRankProfile profile) : { String expression; int rerankCount; + double dropLimit; } { ( expression = expression() { profile.setSecondPhaseRanking(expression); } | (<RERANK_COUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } + | (<RANK_SCORE_DROP_LIMIT> <COLON> dropLimit = floatValue()) { profile.setSecondPhaseRankScoreDropLimit(dropLimit); } ) } diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 3491d868f20..f79fc614a53 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -2,6 +2,8 @@ # RELAX NG Compact Syntax # Vespa Deployment file +include "common.rnc" + start = element deployment { attribute version { "1.0" } & attribute major-version { text }? & @@ -100,7 +102,7 @@ Test = element test { attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & attribute empty-host-ttl { xsd:string }? & - text + Tester? } Staging = element staging { @@ -108,7 +110,7 @@ Staging = element staging { attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & attribute empty-host-ttl { xsd:string }? & - text + Tester? } Dev = element dev { @@ -129,7 +131,8 @@ Prod = element prod { Region* & Delay* & ProdTest* & - ParallelSteps* + ParallelSteps* & + Tester? } ProdTest = element test { @@ -197,3 +200,7 @@ MemberRegion = element region { attribute fraction { xsd:double }? & text } + +Tester = element tester { + Nodes? +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/ml_serving/services.xml b/config-model/src/test/cfg/application/ml_serving/services.xml index 3a5a4438c78..b1271b1297f 100644 --- a/config-model/src/test/cfg/application/ml_serving/services.xml +++ b/config-model/src/test/cfg/application/ml_serving/services.xml @@ -3,7 +3,13 @@ <services version="1.0"> <container version="1.0"> - <model-evaluation/> + <model-evaluation> + <onnx> + <models> + <model name="sqrt" /> <!-- list one of the models --> + </models> + </onnx> + </model-evaluation> <nodes> <node hostalias="node1" /> </nodes> diff --git a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg index 0927045528d..a4fbe6f6c39 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "fast-search" @@ -16,6 +18,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "word" indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "numerical" diff --git a/config-model/src/test/derived/exactmatch/index-info.cfg b/config-model/src/test/derived/exactmatch/index-info.cfg index 4a9ba85ad38..73a80d3510e 100644 --- a/config-model/src/test/derived/exactmatch/index-info.cfg +++ b/config-model/src/test/derived/exactmatch/index-info.cfg @@ -22,6 +22,8 @@ indexinfo[].command[].command "exact *!A!!*" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "string" @@ -30,6 +32,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "exact *!B!!*" indexinfo[].command[].indexname "string_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "string_map.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "string_map.value" indexinfo[].command[].command "type string" @@ -38,6 +42,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "string_map" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_map.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "elem_map.key" indexinfo[].command[].command "type string" @@ -58,6 +64,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "elem_map" indexinfo[].command[].command "multivalue" @@ -66,6 +74,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "string" @@ -74,6 +84,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "exact @@" indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "integer" @@ -84,6 +96,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "elem_array" indexinfo[].command[].command "type Array<elem>" indexinfo[].command[].indexname "another_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "another_map.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "another_map.key" indexinfo[].command[].command "type string" @@ -104,6 +118,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "another_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "another_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "another_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "another_map" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/imported_struct_fields/index-info.cfg b/config-model/src/test/derived/imported_struct_fields/index-info.cfg index 2b8a6fc344d..7c808c932b2 100644 --- a/config-model/src/test/derived/imported_struct_fields/index-info.cfg +++ b/config-model/src/test/derived/imported_struct_fields/index-info.cfg @@ -12,6 +12,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "fast-search" @@ -22,6 +24,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_array.weight" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_array.weight" indexinfo[].command[].command "numerical" @@ -54,10 +58,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "my_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "my_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "fast-search" @@ -74,6 +82,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_str_int_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "fast-search" @@ -84,6 +94,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_str_int_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_str_int_map.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_str_int_map.value" indexinfo[].command[].command "numerical" diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg index 8c2349e37ea..e764fea8d1f 100644 --- a/config-model/src/test/derived/indexschema/index-info.cfg +++ b/config-model/src/test/derived/indexschema/index-info.cfg @@ -260,6 +260,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "f10.text" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "normalize" @@ -270,6 +272,8 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "f10.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "f10.name" indexinfo[].command[].command "string" indexinfo[].command[].indexname "f10.name" indexinfo[].command[].command "type string" diff --git a/config-model/src/test/derived/map_attribute/index-info.cfg b/config-model/src/test/derived/map_attribute/index-info.cfg index 64a51db1edd..be9612ba293 100644 --- a/config-model/src/test/derived/map_attribute/index-info.cfg +++ b/config-model/src/test/derived/map_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "fast-search" @@ -18,6 +20,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "string" @@ -30,6 +34,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "str_map" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "int_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "numerical" @@ -38,6 +44,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "int_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_map.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "int_map.value" indexinfo[].command[].command "integer" diff --git a/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg index b649fa2aa8a..5f414f14420 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "fast-search" @@ -34,12 +36,16 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "str_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "str_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "str_elem_map" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "str_elem_map" indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "int_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "int_elem_map.key" indexinfo[].command[].command "numerical" @@ -66,6 +72,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "int_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "int_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "int_elem_map" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_after/index-info.cfg b/config-model/src/test/derived/matchsettings_map_after/index-info.cfg index e91784db387..f96bc5ad99f 100644 --- a/config-model/src/test/derived/matchsettings_map_after/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_after/index-info.cfg @@ -4,6 +4,8 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sddocname" indexinfo[].command[].command "word" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -24,6 +26,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_def/index-info.cfg b/config-model/src/test/derived/matchsettings_map_def/index-info.cfg index 1304354a722..22214e83698 100644 --- a/config-model/src/test/derived/matchsettings_map_def/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_def/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "normalize" @@ -18,6 +20,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "normalize" @@ -36,6 +40,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -52,6 +58,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -60,6 +68,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -88,6 +98,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg b/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg index 75273abefa1..c0d16de6393 100644 --- a/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg @@ -16,6 +16,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "string" @@ -26,6 +28,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "string" @@ -50,6 +54,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "string" @@ -60,6 +66,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "string" @@ -88,6 +96,8 @@ indexinfo[].command[].command "exact @combi@" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "string" @@ -98,6 +108,8 @@ indexinfo[].command[].command "exact @combi@" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "string" @@ -124,6 +136,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "string" @@ -134,6 +148,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "string" diff --git a/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg b/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg index 2ef04a85a4f..51c13848a05 100644 --- a/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "type string" @@ -14,6 +16,8 @@ indexinfo[].command[].command "exact @mss3_key@" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "type string" @@ -28,6 +32,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -48,6 +54,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -56,6 +64,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -84,6 +94,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg b/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg index cf6a2fc5992..c4cf9cd2f1e 100644 --- a/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "normalize" @@ -18,6 +20,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "normalize" @@ -36,6 +40,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -54,6 +60,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -62,6 +70,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -90,6 +100,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/position_array/index-info.cfg b/config-model/src/test/derived/position_array/index-info.cfg index 5484964149e..65587820bc1 100644 --- a/config-model/src/test/derived/position_array/index-info.cfg +++ b/config-model/src/test/derived/position_array/index-info.cfg @@ -4,12 +4,16 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sddocname" indexinfo[].command[].command "word" indexinfo[].command[].indexname "pos.x" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "pos.y" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "pos.y" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "pos.y" indexinfo[].command[].command "integer" diff --git a/config-model/src/test/derived/structandfieldset/index-info.cfg b/config-model/src/test/derived/structandfieldset/index-info.cfg index 87a95f5a908..0a423530cfa 100644 --- a/config-model/src/test/derived/structandfieldset/index-info.cfg +++ b/config-model/src/test/derived/structandfieldset/index-info.cfg @@ -16,6 +16,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "people.first_name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "string" @@ -26,6 +28,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "people.last_name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "string" diff --git a/config-model/src/test/derived/structanyorder/index-info.cfg b/config-model/src/test/derived/structanyorder/index-info.cfg index 6ea63818572..43dc6312d3e 100644 --- a/config-model/src/test/derived/structanyorder/index-info.cfg +++ b/config-model/src/test/derived/structanyorder/index-info.cfg @@ -180,10 +180,14 @@ indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structfield" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s1" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structarrayfield.s2" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "type string" @@ -344,6 +348,8 @@ indexinfo[].command[].command "type int" indexinfo[].command[].indexname "structarrayfield.s3.s4" indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structarrayfield.s3" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s3" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s4.s1" indexinfo[].command[].command "numerical" @@ -352,6 +358,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "structarrayfield.s4.s1" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "structarrayfield.s4" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s4" indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structarrayfield" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index cc49e006f98..6b39e4d1924 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -96,10 +96,14 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structfield" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s1" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structarrayfield.s2" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "type string" @@ -110,6 +114,8 @@ indexinfo[].command[].command "type Array<sct>" indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stringmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "normalize" @@ -122,6 +128,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stringmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "normalize" @@ -140,10 +148,14 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "stringmapfield" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "intmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "intmapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "intmapfield.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "intmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "intmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "intmapfield.value" indexinfo[].command[].command "integer" @@ -154,10 +166,14 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "intmapfield" indexinfo[].command[].command "type Map<string,int>" indexinfo[].command[].indexname "floatmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "floatmapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "floatmapfield.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "floatmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "floatmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "floatmapfield.value" indexinfo[].command[].command "type float" @@ -166,12 +182,16 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "floatmapfield" indexinfo[].command[].command "type Map<string,float>" indexinfo[].command[].indexname "longmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "longmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "longmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "longmapfield.value" indexinfo[].command[].command "integer" @@ -182,12 +202,16 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "longmapfield" indexinfo[].command[].command "type Map<int,long>" indexinfo[].command[].indexname "doublemapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "doublemapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "doublemapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "doublemapfield.value" indexinfo[].command[].command "type double" @@ -196,6 +220,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "doublemapfield" indexinfo[].command[].command "type Map<int,double>" indexinfo[].command[].indexname "arraymapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "arraymapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "arraymapfield.key" indexinfo[].command[].command "type string" @@ -232,10 +258,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructfield.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructfield.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructfield.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructfield.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructfield.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructfield.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructfield.mymap.value" indexinfo[].command[].command "type string" @@ -254,6 +284,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructfield" indexinfo[].command[].command "type mystruct" indexinfo[].command[].indexname "mystructmap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "mystructmap.key" indexinfo[].command[].command "integer" @@ -268,10 +300,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructmap.value.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructmap.value.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructmap.value.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.mymap.value" indexinfo[].command[].command "type string" @@ -288,6 +324,8 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.structfield" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructmap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value" indexinfo[].command[].command "type mystruct" indexinfo[].command[].indexname "mystructmap" indexinfo[].command[].command "multivalue" @@ -302,10 +340,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructarr.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructarr.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructarr.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.mymap.value" indexinfo[].command[].command "type string" @@ -314,10 +356,14 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructarr.mymap" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mystructarr.title" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.title" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.title" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructarr.structfield" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.structfield" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.structfield" indexinfo[].command[].command "type string" @@ -326,6 +372,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructarr" indexinfo[].command[].command "type Array<mystruct>" indexinfo[].command[].indexname "Folders.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.key" indexinfo[].command[].command "integer" @@ -342,10 +390,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -366,10 +418,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -390,10 +446,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -414,10 +474,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -438,10 +502,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -462,10 +530,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -486,10 +558,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -510,10 +586,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -534,10 +614,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -566,6 +650,8 @@ indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders.value.anotherfolder" indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value" indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java index f9f74546a00..8a184e414d7 100644 --- a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java @@ -62,7 +62,7 @@ public class SystemModelTestCase { VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "simpleconfig/"); assertNotNull(vespaModel); - assertEquals(4, vespaModel.configModelRepo().asMap().size(), "There are two instances of the simple model + Routing and AdminModel (set up implicitly)"); + assertEquals(5, vespaModel.configModelRepo().asMap().size(), "There are two instances of the simple model + Routing, Logs and AdminModel (set up implicitly)"); assertNotNull(vespaModel.configModelRepo().asMap().get("simple"), "One gets the default name as there is no explicit id"); assertNotNull(vespaModel.configModelRepo().asMap().get("second"), "The other gets the explicit id as name"); @@ -116,9 +116,9 @@ public class SystemModelTestCase { assertEquals(host1, host2); assertEquals(host2, host3); - // all three host aliases are for the same host, so the number of services should be 3 + 8 - // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, logserver and metricsproxy) - assertEquals(10, host1.getServices().size()); + // all three host aliases are for the same host, so the number of services should be 3 + 9 + // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, logserver, logserver-container and metricsproxy) + assertEquals(11, host1.getServices().size()); assertNotNull(host1.getService("simpleservice")); assertNotNull(host1.getService("simpleservice2")); @@ -145,7 +145,7 @@ public class SystemModelTestCase { assertNotNull(vespaModel); ApplicationConfigProducerRoot root = vespaModel.getVespa(); - assertEquals(5, vespaModel.configModelRepo().asMap().size()); + assertEquals(6, vespaModel.configModelRepo().asMap().size()); assertTrue(vespaModel.configModelRepo().asMap().containsKey("simple")); assertTrue(vespaModel.configModelRepo().asMap().containsKey("api")); assertTrue(root.getConfigIds().contains("simple/simpleservice.0")); @@ -155,6 +155,8 @@ public class SystemModelTestCase { // Verify that configModelRegistry iterates in dependency order Iterator<ConfigModel> i = vespaModel.configModelRepo().iterator(); ConfigModel plugin = i.next(); + assertEquals("logs", plugin.getId()); + plugin = i.next(); assertEquals("admin", plugin.getId()); plugin = i.next(); assertEquals("simple", plugin.getId()); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 5ead9812b56..737ac4c248c 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -60,7 +60,7 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed; import static com.yahoo.config.provision.NodeResources.StorageType; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.model.Host.memoryOverheadGb; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GiB; import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -151,7 +151,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().ofContainerAvailable()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); @@ -234,8 +234,8 @@ public class ModelProvisioningTest { assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(1, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); assertEquals(2, model.getContentClusters().get("content").getRootGroup().getNodes().size(), "Nodes in cluster without ID"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); - assertEquals(84, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.content, model); assertProvisioned(1, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content"), ClusterSpec.Type.content, model); @@ -287,8 +287,8 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1").deployLogger(logger)); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(18, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); - assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); + assertEquals(24, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); + assertEquals(1876900708, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); @@ -356,7 +356,7 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); assertEquals((long) ((3 - memoryOverheadGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is normal"); } @@ -2580,7 +2580,7 @@ public class ModelProvisioningTest { ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId()); assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override assertEquals(1000, cfg.flush().memory().maxmemory()); // from explicit tuning - assertEquals((long) ((128 - memoryOverheadGb) * GB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning + assertEquals((long) ((128 - memoryOverheadGb) * GiB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning } private static ProtonConfig getProtonConfig(VespaModel model, String configId) { diff --git a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java index df71e30c7d9..d9355e2df7d 100644 --- a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java @@ -3,49 +3,58 @@ package com.yahoo.schema; import com.yahoo.search.query.ranking.Diversity; import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; /** * @author baldersheim */ public class DiversityTestCase { - @Test - void testDiversity() throws ParseException { + private static void verifyDiversity(DeployLoggerStub logger, boolean atRankProfile, boolean atMatchPhase) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(logger, rankProfileRegistry); + String diversitySpec = """ + diversity { + attribute: b + min-groups: 74 + cutoff-factor: 17.3 + cutoff-strategy: strict + } + """; builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - " field b type int {\n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " cutoff-factor: 17.3\n" + - " cutoff-strategy: strict" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " max-filter-coverage: 0.065" + - " }\n" + - " }\n" + - "}\n"); + """ + search test { + document test { + field a type int { + indexing: attribute + attribute: fast-search + } + field b type int { + indexing: attribute + } + } + rank-profile parent { + match-phase {""" + + (atMatchPhase ? diversitySpec : "") + + """ + attribute: a + max-hits: 120 + max-filter-coverage: 0.065 + }""" + + (atRankProfile ? diversitySpec : "") + + """ + } + } + """); builder.build(true); Schema s = builder.getSchema(); - RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); - RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); + RankProfile parent = rankProfileRegistry.get(s, "parent"); + RankProfile.MatchPhaseSettings matchPhase = parent.getMatchPhase(); + RankProfile.DiversitySettings diversity = parent.getDiversity(); assertEquals("b", diversity.getAttribute()); assertEquals(74, diversity.getMinGroups()); assertEquals(17.3, diversity.getCutoffFactor(), 1e-16); @@ -54,6 +63,44 @@ public class DiversityTestCase { assertEquals("a", matchPhase.getAttribute()); assertEquals(0.065, matchPhase.getMaxFilterCoverage(), 1e-16); } + @Test + void testDiversity() throws ParseException { + DeployLoggerStub logger = new DeployLoggerStub(); + verifyDiversity(logger, true, false); + assertTrue(logger.entries.isEmpty()); + verifyDiversity(logger, false, true); + assertEquals(1, logger.entries.size()); + assertEquals("'diversity is deprecated inside 'match-phase'. Specify it at 'rank-profile' level.", logger.entries.get(0).message); + try { + verifyDiversity(logger, true, true); + fail("Should throw."); + } catch (Exception e) { + assertEquals("rank-profile 'parent' error: already has diversity", e.getMessage()); + } + } + + @Test + void requireMatchPhaseOrSecondPhase() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(""" + search test { + document test { + field b type int { indexing: attribute } + } + rank-profile parent { + diversity { + attribute: b + min-groups: 74 + } + } + }"""); + try { + builder.build(true); + fail("Should throw."); + } catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank-profile 'parent': 'diversity' requires either 'match-phase' or 'second-phase' to be specified.", e.getMessage()); + } + } private static String getMessagePrefix() { return "In search definition 'test', rank-profile 'parent': diversity attribute 'b' "; @@ -82,30 +129,29 @@ public class DiversityTestCase { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array<int>'", e.getMessage()); } } - private ApplicationBuilder getSearchBuilder(String diversity) throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - diversity + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " }\n" + - " }\n" + - "}\n"); + private ApplicationBuilder getSearchBuilder(String diversityField) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(""" + search test { + document test { + field a type int { + indexing: attribute + attribute: fast-search + }""" + + diversityField + + """ + } + rank-profile parent { + match-phase { + attribute: a + max-hits: 120 + } + diversity { + attribute: b + min-groups: 74 + } + } + }"""); return builder; } } diff --git a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java index 59887bfd57e..564bfd4a990 100644 --- a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java @@ -530,4 +530,28 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { "}"); } + @Test + public void secondPhaseRankScoreDropLimitIsAddedToRankProperties() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry); + String input = """ + schema test { + document test { + } + rank-profile test inherits default { + second-phase { + rank-score-drop-limit: 17.0 + } + } + } + """; + builder.addSchema(input); + builder.build(true); + Schema schema = builder.getSchema(); + + assertEquals(3, registry.all().size()); + RawRankProfile rawProfile = createRawRankProfile(registry.get(schema, "test"), schema); + assertEquals("17.0", findProperty(rawProfile.configProperties(), "vespa.hitcollector.secondphase.rankscoredroplimit").get()); + } + } diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 8ffbab84fd7..539ec91ee5f 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -6,16 +6,13 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.yolean.Exceptions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import java.util.Collection; import java.util.List; -import java.util.Optional; import java.util.logging.Level; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -233,16 +230,12 @@ public class SummaryTestCase { " document-summary test_summary inherits nonesuch {" + " }" + "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromStrings(logger, schema); - assertEquals("document-summary 'test_summary' inherits 'nonesuch' but this is not present in schema 'test'", - logger.entries.get(0).message); - // fail("Expected failure"); + ApplicationBuilder.createFromString(schema); + fail("Expected failure"); } catch (IllegalArgumentException e) { - fail(); - // assertEquals("document-summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", - // e.getMessage()); + assertEquals("document-summary 'test_summary' inherits 'nonesuch', but this is not present in schema 'test'", + e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java index 409e3b5df22..3f7c87ae37e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.schema.derived.TestableDeployLogger; import com.yahoo.schema.parser.ParseException; import org.junit.jupiter.api.Test; @@ -90,6 +91,31 @@ public class PagedAttributeValidatorTestCase { assertPagedSupported("reference<parent>", Optional.of(getSd("parent", "int", false))); } + @Test + void hnsw_index_triggers_warning_with_paged_setting() throws ParseException { + var logger = new TestableDeployLogger(); + var sd = """ + schema test { + document test { + field pos type tensor(x[2]) { + indexing: attribute | index + attribute: paged + index { + hnsw { + } + } + } + } + } + """; + var schema = createFromString(sd, logger); + assertEquals(1, logger.warnings.size()); + assertEquals("For schema 'test', field 'pos': " + + "The 'paged' attribute setting in combination with HNSW indexing is strongly discouraged, see " + + "https://docs.vespa.ai/en/attributes.html#paged-attributes-disadvantages for details", + logger.warnings.get(0)); + } + private void assertPagedSettingNotSupported(String fieldType) throws ParseException { assertPagedSettingNotSupported(fieldType, false); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java index 9eca2106c5e..5ea097f13fb 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class SummaryConsistencyTestCase { @@ -42,4 +43,24 @@ public class SummaryConsistencyTestCase { Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); } + + @Test + void testDocumentSummaryWithInheritanceOfNonExistingSummary() { + String schemaString = """ + schema foo { + document foo { + field foo type string { + indexing: summary + } + } + document-summary foo_summary inherits non-existent { + summary foo { + source: foo + } + } + } + """; + assertThrows(IllegalArgumentException.class, () -> ApplicationBuilder.createFromString(schemaString).getSchema()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index adf570c4664..96c6b1f99eb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -50,8 +50,8 @@ public class AdminTestCase { void testAdmin20() { VespaModel vespaModel = getVespaModel(TESTDIR + "adminconfig20"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -81,10 +81,10 @@ public class AdminTestCase { StateserverConfig.Builder ssb = new StateserverConfig.Builder(); vespaModel.getConfig(ssb, "admin/slobrok.0"); - assertEquals(19100, new StateserverConfig(ssb).httpport()); + assertEquals(19103, new StateserverConfig(ssb).httpport()); vespaModel.getConfig(ssb, "admin/slobrok.1"); - assertEquals(19102, new StateserverConfig(ssb).httpport()); + assertEquals(19105, new StateserverConfig(ssb).httpport()); LogdConfig.Builder lb = new LogdConfig.Builder(); vespaModel.getConfig(lb, "admin/slobrok.0"); @@ -98,12 +98,13 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertEquals(5, sentinelConfig.service().size()); + assertEquals(6, sentinelConfig.service().size()); assertEquals("logserver", sentinelConfig.service(0).name()); - assertEquals("slobrok", sentinelConfig.service(1).name()); - assertEquals("slobrok2", sentinelConfig.service(2).name()); - assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(3).name()); - assertEquals("logd", sentinelConfig.service(4).name()); + assertEquals("logserver-container", sentinelConfig.service(1).name()); + assertEquals("slobrok", sentinelConfig.service(2).name()); + assertEquals("slobrok2", sentinelConfig.service(3).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(4).name()); + assertEquals("logd", sentinelConfig.service(5).name()); } /** @@ -114,8 +115,8 @@ public class AdminTestCase { void testOnlyAdminserver() { VespaModel vespaModel = getVespaModel(TESTDIR + "simpleadminconfig20"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -134,11 +135,12 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertEquals(4, sentinelConfig.service().size()); + assertEquals(5, sentinelConfig.service().size()); assertEquals("logserver", sentinelConfig.service(0).name()); - assertEquals("slobrok", sentinelConfig.service(1).name()); - assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(2).name()); - assertEquals("logd", sentinelConfig.service(3).name()); + assertEquals("logserver-container", sentinelConfig.service(1).name()); + assertEquals("slobrok", sentinelConfig.service(2).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(3).name()); + assertEquals("logd", sentinelConfig.service(4).name()); assertEquals(-1, sentinelConfig.service(0).affinity().cpuSocket()); assertTrue(sentinelConfig.service(0).preShutdownCommand().isEmpty()); @@ -175,8 +177,8 @@ public class AdminTestCase { void testMultipleConfigServers() { VespaModel vespaModel = getVespaModel(TESTDIR + "multipleconfigservers"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index ac431f081ed..0acacc340eb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -150,7 +150,7 @@ public class MetricsProxyContainerTest { @Test void vespa_services_config_has_all_services() { VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(hostedServicesWithContent()); - assertEquals(10, vespaServicesConfig.service().size()); + assertEquals(11, vespaServicesConfig.service().size()); for (var service : vespaServicesConfig.service()) { if (service.configId().equals("admin/cluster-controllers/0")) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java index c24fcb27dc9..efe9cfe5060 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java @@ -31,7 +31,7 @@ public class OpenTelemetryConfigGeneratorTest { void testBuildsYaml() { var mockZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("mock")); var app = ApplicationId.from("mytenant", "myapp", "myinstance"); - var generator = new OpenTelemetryConfigGenerator(mockZone, app); + var generator = new OpenTelemetryConfigGenerator(mockZone, app, true); var root = new MockRoot(); var mockHost = new Host(root, "localhost2.local"); @@ -50,12 +50,15 @@ public class OpenTelemetryConfigGeneratorTest { var mockSvc2 = new MockService(root, "searchnode"); mockSvc2.setProp("clustername", "mycluster"); mockSvc2.setProp("clustertype", "mockup"); - var mockPort2 = new StatePortInfo("other.host.local", 19102, mockSvc2); + var mockPort2 = new StatePortInfo("other123x.host.local", 19102, mockSvc2); generator.addStatePorts(List.of(mockPort1, mockPort2)); String yaml = generator.generate(); // System.err.println(">>>\n" + yaml + "\n<<<"); assertTrue(yaml.contains("sentinel")); + String want = """ + "parentHostname":"other123.host.local"""; + assertTrue(yaml.contains(want)); } static class MockService extends AbstractService { @@ -70,4 +73,22 @@ public class OpenTelemetryConfigGeneratorTest { @Override public void allocatePorts(int start, PortAllocBridge from) { } } + @Test + void testFindParentHost() { + String result; + result = OpenTelemetryConfigGenerator.findParentHost("n1234c.foo.bar.some.cloud"); + assertEquals("n1234.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("n1234-v6-7.foo.bar.some.cloud"); + assertEquals("n1234.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("2000a.foo.bar.some.cloud"); + assertEquals("2000.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("2000-v6-10.foo.bar.some.cloud"); + assertEquals("2000.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("foobar.some.cloud"); + assertNull(result); + result = OpenTelemetryConfigGenerator.findParentHost("foo123bar.some.cloud"); + assertNull(result); + result = OpenTelemetryConfigGenerator.findParentHost("foo123.some.cloud"); + assertNull(result); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 45125c8eb68..340968f89d1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -45,10 +45,10 @@ class JvmHeapSizeValidatorTest { @Test void fails_on_too_low_heap_size() throws IOException, SAXException { - var deployState = createDeployState(2.2, 1024L * 1024 * 1024); + var deployState = createDeployState(2.3, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationTester.expect(new JvmHeapSizeValidator(), model, deployState, - "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); + "Allocated memory to JVM in cluster 'container' is too low (0.51GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index f68a1da7dfb..0f7113748dd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -57,10 +57,11 @@ public class ConfigValueChangeValidatorTest { createVespaModel(createQrStartConfigSegment(true, 2096)), createVespaModel(createQrStartConfigSegment(false, 2096)) ); - assertEquals(3, changes.size()); + assertEquals(4, changes.size()); assertComponentsEquals(changes, "default/container.0", 0); - assertComponentsEquals(changes, "admin/cluster-controllers/0", 1); - assertComponentsEquals(changes, "admin/metrics/localhost", 2); + assertComponentsEquals(changes, "admin/logs/0", 1); + assertComponentsEquals(changes, "admin/cluster-controllers/0", 2); + assertComponentsEquals(changes, "admin/metrics/localhost", 3); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java index b7e612127c1..36839e72e10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java @@ -209,7 +209,7 @@ public class ResourcesReductionValidatorTest { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb()); + resources.vcpu(), resources.memoryGiB(), resources.diskGb()); return "<services version='1.0'>" + " <container id='default' version='1.0'>" + " <nodes count='" + nodes + "'>" + @@ -223,7 +223,7 @@ public class ResourcesReductionValidatorTest { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb()); + resources.vcpu(), resources.memoryGiB(), resources.diskGb()); return "<services version='1.0'>" + " <content id='default' version='1.0'>" + " <redundancy>1</redundancy>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java index 4c0786ea879..484a938476d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java @@ -33,7 +33,7 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { // Must be so large that changing model set or options requires restart (due to using more memory than available), // but not so large that deployment will not work at all with one model - private static final long defaultCost = 723456789; + private static final long defaultCost = 635241309; private static final long defaultHash = 0; @@ -47,8 +47,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost() { - VespaModel current = createModel(onnxModelCost(70000000, defaultHash)); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash)); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash)); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash)); List<ConfigChangeAction> result = validateModel(current, next); assertEquals(1, result.size()); assertTrue(result.get(0).validationId().isEmpty()); @@ -58,8 +58,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost_non_hosted() { boolean hosted = false; - VespaModel current = createModel(onnxModelCost(70000000, defaultHash), hosted); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash), hosted); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash), hosted); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash), hosted); List<ConfigChangeAction> result = validateModel(current, next, hosted); assertEquals(0, result.size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 924419daeae..052d32269ef 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -157,7 +157,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. assertEquals(1, c.getRoot().hostSystem().getHosts().size()); HostResource h = c.getRoot().hostSystem().getHost("mockhost"); - String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor", "transactionlogserver"}; + String [] expectedServices = {"configserver", "logserver", "logserver-container", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor"}; assertServices(h, expectedServices); assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); assertEquals("clu/search/cluster.clu/0", h.getService("searchnode").getConfigId()); @@ -205,7 +205,7 @@ public class ContentBuilderTest extends DomBuilderTest { HostResource h = cluster.getRoot().hostSystem().getHost("mockhost"); String [] expectedServices = { "logd", "configproxy", "config-sentinel", "configserver", "container", "logserver", - "slobrok", "storagenode", "distributor", "searchnode", "transactionlogserver", + "logserver-container", "slobrok", "storagenode", "distributor", "searchnode", CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName }; assertServices(h, expectedServices); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 0e616661191..feeff452a91 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; + import java.util.ArrayList; import java.util.List; @@ -196,14 +197,21 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { assertEquals(1, admin.getConfigservers().size()); } + @Test + void containerOnLogserver() { + Admin admin = buildAdmin(servicesAdminNoAdminServerOrConfigServer()); + assertTrue(admin.getLogServerContainerCluster().isPresent()); + assertEquals("logs", admin.getLogServerContainerCluster().get().getName()); + } + private Admin buildAdmin(Element xml) { return buildAdmin(xml, false, new ArrayList<>()); } private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { DeployState deployState = DeployState.createTestState(); - final DomAdminV2Builder domAdminBuilder = - new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, multitenant, configServerSpecs); + ConfigModelContext context = ConfigModelContext.create(deployState, null, (m) -> {}, root, "foo"); + DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(context, multitenant, configServerSpecs); Admin admin = domAdminBuilder.build(deployState, root, xml); admin.addPerHostServices(root.hostSystem().getHosts(), deployState); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java index 344471cada0..e0193d10e23 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java @@ -42,8 +42,8 @@ public class NodesSpecificationTest { assertEquals(2, spec.minResources().nodeResources().vcpu(), 1e-9); assertEquals(2, spec.maxResources().nodeResources().vcpu(), 1e-9); - assertEquals(3e15, spec.minResources().nodeResources().memoryGb(), 1e-9); - assertEquals(3e15, spec.maxResources().nodeResources().memoryGb(), 1e-9); + assertEquals(3e15, spec.minResources().nodeResources().memoryGiB(), 1e-9); + assertEquals(3e15, spec.maxResources().nodeResources().memoryGiB(), 1e-9); assertEquals(4e3, spec.minResources().nodeResources().diskGb(), 1e-9); assertEquals(4e3, spec.maxResources().nodeResources().diskGb(), 1e-9); @@ -54,8 +54,8 @@ public class NodesSpecificationTest { assertEquals(1 << 30, spec.minResources().nodeResources().gpuResources().count()); assertEquals(1 << 30, spec.maxResources().nodeResources().gpuResources().count()); - assertEquals(3e-9, spec.minResources().nodeResources().gpuResources().memoryGb(), 1e-12); - assertEquals(3e-9, spec.maxResources().nodeResources().gpuResources().memoryGb(), 1e-12); + assertEquals(3e-9, spec.minResources().nodeResources().gpuResources().memoryGiB(), 1e-12); + assertEquals(3e-9, spec.maxResources().nodeResources().gpuResources().memoryGiB(), 1e-12); assertEquals(DiskSpeed.fast, spec.minResources().nodeResources().diskSpeed()); assertEquals(DiskSpeed.fast, spec.maxResources().nodeResources().diskSpeed()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 9cf34b55ebf..93362e144a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -49,6 +49,7 @@ import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -498,4 +499,28 @@ public class ContainerClusterTest { (extra.isEmpty() ? "" : " " + extra); } + @Test + void testTruncationTo4Bits() { + assertEquals(0, ApplicationContainerCluster.truncateTo4SignificantBits(0)); + assertEquals(1, ApplicationContainerCluster.truncateTo4SignificantBits(1)); + assertEquals(15, ApplicationContainerCluster.truncateTo4SignificantBits(15)); + assertEquals(16, ApplicationContainerCluster.truncateTo4SignificantBits(16)); + assertEquals(16, ApplicationContainerCluster.truncateTo4SignificantBits(17)); + assertEquals(18, ApplicationContainerCluster.truncateTo4SignificantBits(18)); + assertEquals(18, ApplicationContainerCluster.truncateTo4SignificantBits(19)); + assertEquals(30, ApplicationContainerCluster.truncateTo4SignificantBits(30)); + assertEquals(30, ApplicationContainerCluster.truncateTo4SignificantBits(31)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(32)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(33)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(34)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(35)); + assertEquals(36, ApplicationContainerCluster.truncateTo4SignificantBits(36)); + assertEquals(0x78000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x78000000)); + assertEquals(0x78000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x7fffffff)); + assertEquals(0x80000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x80000000)); + assertEquals(0b10001000000000000000000000000000, ApplicationContainerCluster.truncateTo4SignificantBits(0b10000000000000000000000000000001)); + assertEquals(0b11000100000000000000000000000000, ApplicationContainerCluster.truncateTo4SignificantBits(0b11000000000000000000000000000001)); + assertEquals(0b11111111111111111111111111100010, ApplicationContainerCluster.truncateTo4SignificantBits(0b11111111111111111111111111100001)); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java index ea43f5c8124..b782366655f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java @@ -142,7 +142,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertTrue(chain.phases().isEmpty()); assertEquals(1, chain.inherits().size()); assertEquals("native", chain.inherits(0)); - assertEquals(10, chain.components().size()); + assertEquals(11, chain.components().size()); assertEquals("com.yahoo.prelude.querytransform.PhrasingSearcher@vespa", chain.components(0)); assertEquals("com.yahoo.prelude.searcher.FieldCollapsingSearcher@vespa", chain.components(1)); assertEquals("com.yahoo.search.yql.MinimalQueryInserter@vespa", chain.components(2)); @@ -153,13 +153,14 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertEquals("com.yahoo.prelude.semantics.SemanticSearcher@vespa", chain.components(7)); assertEquals("com.yahoo.search.grouping.GroupingQueryParser@vespa", chain.components(8)); assertEquals("com.yahoo.search.querytransform.WeakAndReplacementSearcher@vespa", chain.components(9)); + assertEquals("com.yahoo.search.searchers.OpportunisticWeakAndSearcher@vespa", chain.components(10)); assertTrue(chain.excludes().isEmpty()); assertEquals(ChainsConfig.Chains.Type.SEARCH, chain.type()); } @Test public void require_all_default_chains_are_correct() { - assertEquals(63, chainsConfig.components().size()); + assertEquals(64, chainsConfig.components().size()); assertEquals(10, chainsConfig.chains().size()); validateVespaPhasesChain(findChain("vespaPhases")); validateNativeChain(findChain("native")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 0e8ce4748b4..60ea37cad3f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -17,6 +17,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -49,6 +50,7 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -741,6 +743,29 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + void testerContainer() { + createModelWithTesterNodes("<nodes count='1' docker-image='foo/bar/baz'><resources vcpu='0.1' memory='1Gb' disk='1Gb'/></nodes>"); + + assertEquals("In container cluster 'default': tester cannot run on more than 1 node, but 2 nodes were specified", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes count='2'/>")))); + + assertEquals("In container cluster 'default': tester resources must be absolute, but min and max resources differ: specification of dedicated " + + "min 1 nodes with [vcpu: 0.0, memory: 1.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any] " + + "max 1 nodes with [vcpu: 0.0, memory: 2.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any]", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes><resources memory='[1Gb, 2Gb]'/></nodes>")))); + } + + void createModelWithTesterNodes(String testerNodesXml) { + String containerXml = "<container id='default' version='1.0'>%s</container>".formatted(testerNodesXml); + VespaModelTester tester = new VespaModelTester(); + tester.setApplicationId("t", "a", "i-t"); + tester.addHosts(3); + tester.createModel(containerXml, true); + } + + @Test void cluster_with_zookeeper() { Function<Integer, String> servicesXml = (nodeCount) -> "<container version='1.0' id='default'>" + "<nodes count='" + nodeCount + "'/>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 64afd444989..46097da434e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.container.xml; -import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; @@ -206,29 +205,6 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { assertEquals("-XX:+UseParNewGC", qrStartConfig.jvm().gcopts()); } - @Test - void verify_jvm_option_and_value_from_feature_flag_are_both_included() throws IOException, SAXException { - String servicesXml = """ - <container version='1.0'> - <search/> - <nodes> - <jvm options="-Xms1024m -Xmx2048m" /> - <node hostalias="node1" /> - </nodes> - </container> - """; - ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); - // Need to create VespaModel to make deploy properties have effect - VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() - .applicationPackage(applicationPackage) - .properties(new TestProperties().setJvmOmitStackTraceInFastThrowOption("-XX:-OmitStackTraceInFastThrow")) - .build()); - SentinelConfig.Builder builder = new SentinelConfig.Builder(); - model.getConfig(builder, "hosts/localhost"); - SentinelConfig config = builder.build(); - assertEquals("PRELOAD=/opt/vespa/lib64/vespa/malloc/libvespamalloc.so exec ${VESPA_HOME}/libexec/vespa/vespa-wrapper vespa-start-container-daemon -Xms1024m -Xmx2048m -XX:-OmitStackTraceInFastThrow ", config.service().get(0).command()); - } - private void verifyLoggingOfJvmGcOptions(boolean isHosted, String override, String... invalidOptions) throws IOException, SAXException { verifyLogMessage(isHosted, "gc-options", override, invalidOptions); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 786caa4b317..a890f35caa8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; @@ -48,6 +49,8 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.function.Consumer; +import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -941,8 +944,19 @@ public class ContentClusterTest extends ContentBaseTest { } private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, Optional<Flavor> flavor) throws Exception { + return createOneNodeCluster(clusterXml, props, flavor, new StringBuffer()); + } + + private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, + Optional<Flavor> flavor, StringBuffer deployWarningsBuffer) throws Exception { + DeployLogger logger = (level, message) -> { + if (level == Level.WARNING) { // only care about warnings + deployWarningsBuffer.append("%s\n".formatted(message)); + } + }; DeployState.Builder deployStateBuilder = new DeployState.Builder() - .properties(props); + .properties(props) + .deployLogger(logger); MockRoot root = flavor.isPresent() ? ContentClusterUtils.createMockRoot(new SingleNodeProvisioner(flavor.get()), List.of(), deployStateBuilder) : @@ -1264,7 +1278,7 @@ public class ContentClusterTest extends ContentBaseTest { void verify_max_tls_size() throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty())); - assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor))); + assertEquals(2_000_000_000, resolveMaxTLSSize(Optional.of(flavor))); } void assertZookeeperServerImplementation(String expectedClassName, @@ -1471,13 +1485,18 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(expectedGroupsAllowedDown, config.max_number_of_groups_allowed_to_be_down()); } - private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + private StorDistributormanagerConfig resolveDistributorConfig(Consumer<TestProperties> propertyMutator) throws Exception { var properties = new TestProperties(); - if (featureLevel != null) { - properties.setContentLayerMetadataFeatureLevel(featureLevel); - } - var cfg = resolveStorDistributormanagerConfig(properties); - return cfg.enable_operation_cancellation(); + propertyMutator.accept(properties); + return resolveStorDistributormanagerConfig(properties); + } + + private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + return resolveDistributorConfig((props) -> { + if (featureLevel != null) { + props.setContentLayerMetadataFeatureLevel(featureLevel); + } + }).enable_operation_cancellation(); } @Test @@ -1488,6 +1507,21 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveDistributorOperationCancellationConfig(2)); } + private boolean resolveDistributorSymmetricReplicaSelectionConfig(Boolean flagValue) throws Exception { + return resolveDistributorConfig((props) -> { + if (flagValue != null) { + props.setSymmetricPutAndActivateReplicaSelection(flagValue); + } + }).symmetric_put_and_activate_replica_selection(); + } + + @Test + void distributor_symmetric_replica_selection_config_controlled_by_properties() throws Exception { + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(null)); // defaults to false + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(false)); + assertTrue(resolveDistributorSymmetricReplicaSelectionConfig(true)); + } + @Test void node_distribution_key_outside_legal_range_is_disallowed() { // Only [0, UINT16_MAX - 1] is a valid range. UINT16_MAX is a special content layer-internal @@ -1506,6 +1540,68 @@ public class ContentClusterTest extends ContentBaseTest { } } + private String createClusterAndGetDeploymentWarnings(String xml) { + var warningBuf = new StringBuffer(); + try { + createOneNodeCluster(xml, new TestProperties(), Optional.empty(), warningBuf); + } catch (Exception e) { + throw new RuntimeException(e); + } + return warningBuf.toString(); + }; + + private static String clusterXmlWithNodeDistributionKey(int key) { + return "<content version=\"1.0\" id=\"mockcluster\">" + + " <redundancy>1</redundancy>" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"%d\" hostalias=\"mockhost\"/>".formatted(key) + + " </group>" + + "</content>"; + } + + @Test + void distribution_key_much_higher_than_node_count_logs_deployment_warning() { + // "much higher" is somewhat arbitrary, but needs to be kept in track with threshold in `ContentCluster`. + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(101)); + assertEquals(warnings, "Content cluster 'mockcluster' has 1 node(s), but the highest distribution " + + "key is 101. Having much higher distribution keys than the number of nodes " + + "is not recommended, as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node\n"); + } + + @Test + void distribution_key_not_much_higher_than_node_count_does_not_log_deployment_warning() { + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(100)); + assertEquals(warnings, ""); + } + + private void checkStrictlyIncreasingClusterStateVersionConfig(Boolean flagValue, boolean expected) throws Exception { + var props = new TestProperties(); + if (flagValue != null) { + props.setEnforceStrictlyIncreasingClusterStateVersions(flagValue); + } + var cc = createOneNodeCluster(props); + + // stor-server config should be the same for both distributors and storage nodes + var builder = new StorServerConfig.Builder(); + cc.getStorageCluster().getConfig(builder); + var cfg = builder.build(); + assertEquals(expected, cfg.require_strictly_increasing_cluster_state_versions()); + + builder = new StorServerConfig.Builder(); + cc.getDistributorNodes().getConfig(builder); + cfg = builder.build(); + assertEquals(expected, cfg.require_strictly_increasing_cluster_state_versions()); + } + + @Test + void strictly_increasing_cluster_state_versions_config_controlled_by_feature_flag() throws Exception { + checkStrictlyIncreasingClusterStateVersionConfig(null, false); // TODO change default + checkStrictlyIncreasingClusterStateVersionConfig(false, false); + checkStrictlyIncreasingClusterStateVersionConfig(true, true); + } + private String servicesWithGroups(int groupCount, double minGroupUpRatio) { String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index e9e96d8b0cf..920b2f7a868 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -131,7 +131,7 @@ public class IndexedTest extends ContentBaseTest { // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", // "container", "fleetcontroller", - // "storagenode", "searchnode", "distributor", "transactionlogserver"}; + // "storagenode", "searchnode", "distributor"}; // DomContentBuilderTest.assertServices(h, expectedServices); Routing routing = model.getRouting(); assertNotNull(routing); @@ -159,8 +159,7 @@ public class IndexedTest extends ContentBaseTest { // HostResource h = model.getHostSystem().getHosts().get(0); // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", - // "container", "storagenode", "searchnode", "distributor", - // "transactionlogserver"}; + // "container", "storagenode", "searchnode", "distributor"}; // DomContentBuilderTest.assertServices(h, expectedServices); ContentCluster s = model.getContentClusters().get("test"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 8b83c941631..2a81f2da286 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -111,9 +111,9 @@ public class ClusterTest { assertEquals(1, nodesConfig.node(1).key()); assertEquals(2, nodesConfig.node(2).key()); - assertEquals(19106, nodesConfig.node(0).port()); - assertEquals(19118, nodesConfig.node(1).port()); - assertEquals(19130, nodesConfig.node(2).port()); + assertEquals(19109, nodesConfig.node(0).port()); + assertEquals(19121, nodesConfig.node(1).port()); + assertEquals(19133, nodesConfig.node(2).port()); assertEquals(0, nodesConfig.node(0).group()); assertEquals(0, nodesConfig.node(1).group()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index 4fd61f59ed7..6114cc7d8bf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -8,9 +8,14 @@ import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.ApplicationPackageTester; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.api.OnnxModelCost.Calculator; +import com.yahoo.config.model.api.OnnxModelCost.ModelInfo; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; @@ -22,7 +27,10 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.URI; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -47,6 +55,7 @@ public class ImportedModelTester { private final String modelName; private final Path applicationDir; private final DeployState deployState; + public final Calculator calculator = new MockCalculator(); public ImportedModelTester(String modelName, Path applicationDir) { this(modelName, applicationDir, new DeployState.Builder()); @@ -58,6 +67,7 @@ public class ImportedModelTester { deployState = deployStateBuilder.applicationPackage(ApplicationPackageTester.create(applicationDir.toString()).app()) .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .modelImporters(importers) + .onnxModelCost((pkg, app, cluster) -> calculator) .build(); } @@ -98,4 +108,19 @@ public class ImportedModelTester { } } + public static class MockCalculator implements OnnxModelCost.Calculator { + private final Map<String, ModelInfo> models = new HashMap<>(); + @Override public long aggregatedModelCostInBytes() { return models.size(); } + @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) { + models.put(path.toString(), new ModelInfo(path.toString(), 1, 1, onnxModelOptions)); + } + @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) { + models.put(uri.toString(), new ModelInfo(uri.toString(), 1, 1, onnxModelOptions)); + } + @Override public Map<String, ModelInfo> models() { return Map.copyOf(models); } + @Override public void setRestartOnDeploy() { } + @Override public boolean restartOnDeploy() { return false; } + @Override public void store() { } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index cc33c8561fc..bced4c546c6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -55,6 +55,7 @@ public class ModelEvaluationTest { RankProfilesConfig config = new RankProfilesConfig(b); assertEquals(0, config.rankprofile().size()); + assertEquals(0, tester.calculator.aggregatedModelCostInBytes()); } finally { IOUtils.recursiveDeleteDir(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); @@ -69,6 +70,7 @@ public class ModelEvaluationTest { try { ImportedModelTester tester = new ImportedModelTester("ml_serving", appDir); assertHasMlModels(tester.createVespaModel(), appDir); + assertEquals(3, tester.calculator.aggregatedModelCostInBytes()); // At this point the expression is stored - copy application to another location which do not have a models dir storedAppDir.toFile().mkdirs(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index fbc4f6768a2..998437b64f5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -9,7 +9,8 @@ import org.junit.jupiter.api.Test; import static com.yahoo.vespa.model.Host.memoryOverheadGb; import static org.junit.jupiter.api.Assertions.assertEquals; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.MiB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GiB; import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; /** @@ -28,7 +29,7 @@ public class NodeResourcesTuningTest { @Test void require_that_hwinfo_memory_size_is_set() { - assertEquals(24 * GB, configFromMemorySetting(24 + memoryOverheadGb, 0).hwinfo().memory().size()); + assertEquals(24 * GiB, configFromMemorySetting(24 + memoryOverheadGb, 0).hwinfo().memory().size()); assertEquals(1.9585050869E10, configFromMemorySetting(24 + memoryOverheadGb, ApplicationContainerCluster.heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster * 0.01).hwinfo().memory().size(), 1000); } @@ -84,26 +85,26 @@ public class NodeResourcesTuningTest { @Test void require_that_document_store_maxfilesize_is_set_based_on_available_memory() { - assertDocumentStoreMaxFileSize(256 * MB, 4); - assertDocumentStoreMaxFileSize(256 * MB, 6); - assertDocumentStoreMaxFileSize(256 * MB, 8); - assertDocumentStoreMaxFileSize(256 * MB, 12); - assertDocumentStoreMaxFileSize((long) (16 * GB * 0.02), 16); - assertDocumentStoreMaxFileSize((long) (24 * GB * 0.02), 24); - assertDocumentStoreMaxFileSize((long) (32 * GB * 0.02), 32); - assertDocumentStoreMaxFileSize((long) (48 * GB * 0.02), 48); - assertDocumentStoreMaxFileSize((long) (64 * GB * 0.02), 64); - assertDocumentStoreMaxFileSize((long) (128 * GB * 0.02), 128); - assertDocumentStoreMaxFileSize((long) (256 * GB * 0.02), 256); - assertDocumentStoreMaxFileSize((long) (512 * GB * 0.02), 512); + assertDocumentStoreMaxFileSize(256 * MiB, 4); + assertDocumentStoreMaxFileSize(256 * MiB, 6); + assertDocumentStoreMaxFileSize(256 * MiB, 8); + assertDocumentStoreMaxFileSize(256 * MiB, 12); + assertDocumentStoreMaxFileSize((long) (16 * GiB * 0.02), 16); + assertDocumentStoreMaxFileSize((long) (24 * GiB * 0.02), 24); + assertDocumentStoreMaxFileSize((long) (32 * GiB * 0.02), 32); + assertDocumentStoreMaxFileSize((long) (48 * GiB * 0.02), 48); + assertDocumentStoreMaxFileSize((long) (64 * GiB * 0.02), 64); + assertDocumentStoreMaxFileSize((long) (128 * GiB * 0.02), 128); + assertDocumentStoreMaxFileSize((long) (256 * GiB * 0.02), 256); + assertDocumentStoreMaxFileSize((long) (512 * GiB * 0.02), 512); } @Test void require_that_flush_strategy_memory_limits_are_set_based_on_available_memory() { - assertFlushStrategyMemory((long) (4 * GB * DEFAULT_MEMORY_GAIN), 4); - assertFlushStrategyMemory((long) (8 * GB * DEFAULT_MEMORY_GAIN), 8); - assertFlushStrategyMemory((long) (24 * GB * DEFAULT_MEMORY_GAIN), 24); - assertFlushStrategyMemory((long) (64 * GB * DEFAULT_MEMORY_GAIN), 64); + assertFlushStrategyMemory((long) (4 * GiB * DEFAULT_MEMORY_GAIN), 4); + assertFlushStrategyMemory((long) (8 * GiB * DEFAULT_MEMORY_GAIN), 8); + assertFlushStrategyMemory((long) (24 * GiB * DEFAULT_MEMORY_GAIN), 24); + assertFlushStrategyMemory((long) (64 * GiB * DEFAULT_MEMORY_GAIN), 64); } @Test @@ -129,8 +130,8 @@ public class NodeResourcesTuningTest { @Test void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1 * GB / 25, configFromMemorySetting(1 + memoryOverheadGb, 0).summary().cache().maxbytes()); - assertEquals(256 * GB / 25, configFromMemorySetting(256 + memoryOverheadGb, 0).summary().cache().maxbytes()); + assertEquals(1 * GiB / 25, configFromMemorySetting(1 + memoryOverheadGb, 0).summary().cache().maxbytes()); + assertEquals(256 * GiB / 25, configFromMemorySetting(256 + memoryOverheadGb, 0).summary().cache().maxbytes()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index bc981c3de7c..499df4e5669 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -31,13 +30,8 @@ public class SearchNodeTest { assertEquals(expected, cfg.basedir()); } - private void prepare(MockRoot root, SearchNode node, Boolean useFsync) { + private void prepare(MockRoot root, SearchNode node) { Host host = new Host(root, "mockhost"); - TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); - tls.setHostResource(new HostResource(host)); - tls.setBasePort(100); - tls.initService(root.getDeployState()); - node.setTls(tls); node.setHostResource(new HostResource(host)); node.setBasePort(200); node.initService(root.getDeployState()); @@ -45,13 +39,15 @@ public class SearchNodeTest { } private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, - boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) { + boolean flushOnShutDown, boolean isHosted, + ModelContext.FeatureFlags featureFlags, + Boolean syncTransactionLog) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - null, null, isHosted, 0.0, featureFlags); + null, null, isHosted, 0.0, featureFlags, syncTransactionLog); } - private static SearchNode createSearchNode(MockRoot root) { - return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties()); + private static SearchNode createSearchNode(MockRoot root, Boolean syncTransactionLog) { + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties(), syncTransactionLog); } @Test @@ -64,15 +60,17 @@ public class SearchNodeTest { @Test void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); - prepare(root, node, true); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); + prepare(root, node); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @Test void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -81,7 +79,8 @@ public class SearchNodeTest { @Test void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); @@ -90,7 +89,8 @@ public class SearchNodeTest { private void verifyCodePlacement(boolean hugePages) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().loadCodeAsHugePages(hugePages)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().loadCodeAsHugePages(hugePages), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(hugePages, node.getEnvVars().get("VESPA_LOAD_CODE_AS_HUGEPAGES") != null); @@ -104,7 +104,8 @@ public class SearchNodeTest { private void verifySharedStringRepoReclaim(boolean sharedStringRepoNoReclaim) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(sharedStringRepoNoReclaim, node.getEnvVars().get("VESPA_SHARED_STRING_REPO_NO_RECLAIM") != null); @@ -120,10 +121,10 @@ public class SearchNodeTest { return new MockRoot("", new DeployState.Builder().properties(properties).build()); } - private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean useFsync) { + private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean syncTransactionLog) { MockRoot root = createRoot(properties); - SearchNode node = createSearchNode(root); - prepare(root, node, useFsync); + SearchNode node = createSearchNode(root, syncTransactionLog); + prepare(root, node); TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); node.getConfig(tlsBuilder); return tlsBuilder.build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index c7d9ae6b818..2fda76d0d82 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -20,9 +20,6 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.net.HostName; @@ -37,6 +34,7 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; + import java.io.File; import java.io.IOException; import java.io.StringReader; @@ -265,9 +263,9 @@ public class VespaModelTestCase { assertEquals(1, admin.getConfigservers().size()); Set<HostInfo> hosts = model.getHosts(); assertEquals(1, hosts.size()); - //logd, config proxy, sentinel, config server, slobrok, log server + // logd, config proxy, sentinel, config server, slobrok, logserver, logserver container HostInfo host = hosts.iterator().next(); - assertEquals(7, host.getServices().size()); + assertEquals(8, host.getServices().size()); new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 7037480f853..887db9c9cc9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -88,7 +88,7 @@ public class VespaModelTester { // (for e.g cluster controllers and slobrok nodes) String hostname = String.format("%s-%02d", "node" + "-" + Math.round(resources.vcpu()) + - "-" + Math.round(resources.memoryGb()) + + "-" + Math.round(resources.memoryGiB()) + "-" + Math.round(resources.diskGb()), count - i); hosts.add(new Host(hostname, List.of(), flavor)); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/CapacityPolicies.java b/config-provisioning/src/main/java/com/yahoo/config/provision/CapacityPolicies.java index 818a448187c..7d44c4bb8e1 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/CapacityPolicies.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/CapacityPolicies.java @@ -51,7 +51,7 @@ public class CapacityPolicies { return switch (zone.environment()) { case dev, test -> 1; case perf -> Math.min(requested, 3); - case staging -> requested <= 1 ? requested : Math.max(2, requested / 10); + case staging -> requested <= 1 ? requested : Math.max(2, (int)(0.05 * requested)); case prod -> requested; }; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java index 4f34c010aae..b70f098a153 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java @@ -56,7 +56,7 @@ public class ClusterResources { /** Returns the total resources of this, that is the number of nodes times the node resources */ public NodeResources totalResources() { return nodeResources.withVcpu(nodeResources.vcpu() * nodes) - .withMemoryGb(nodeResources.memoryGb() * nodes) + .withMemoryGiB(nodeResources.memoryGiB() * nodes) .withDiskGb(nodeResources.diskGb() * nodes) .withBandwidthGbps(nodeResources.bandwidthGbps() * nodes); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java index a8674d220d1..d7325a5fe92 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java @@ -4,16 +4,20 @@ package com.yahoo.config.provision; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; +import javax.naming.NameNotFoundException; import javax.naming.NamingException; import javax.naming.directory.Attribute; import javax.naming.directory.Attributes; +import javax.naming.directory.DirContext; import javax.naming.directory.InitialDirContext; import java.net.InetAddress; -import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.Collections; -import java.util.Enumeration; +import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * @author jonmv @@ -35,9 +39,7 @@ public interface EndpointsChecker { public static final Availability ready = new Availability(Status.available, "Endpoints are ready."); } - interface HostNameResolver { Optional<InetAddress> resolve(DomainName hostName); } - - interface CNameResolver { Optional<DomainName> resolve(DomainName hostName); } + interface NameResolver { List<String> resolve(NameType nameType, DomainName name); } interface HealthChecker { Availability healthy(Endpoint endpoint); } @@ -46,55 +48,54 @@ public interface EndpointsChecker { } static EndpointsChecker of(HealthChecker healthChecker) { - return zoneEndpoints -> endpointsAvailable(zoneEndpoints, EndpointsChecker::resolveHostName, EndpointsChecker::resolveCname, healthChecker); + return zoneEndpoints -> endpointsAvailable(zoneEndpoints, EndpointsChecker::resolveAll, healthChecker); } - static EndpointsChecker mock(HostNameResolver hostNameResolver, CNameResolver cNameResolver, HealthChecker healthChecker) { - return zoneEndpoints -> endpointsAvailable(zoneEndpoints, hostNameResolver, cNameResolver, healthChecker); + static EndpointsChecker mock(NameResolver resolver, HealthChecker healthChecker) { + return zoneEndpoints -> endpointsAvailable(zoneEndpoints, resolver, healthChecker); } Availability endpointsAvailable(List<Endpoint> zoneEndpoints); private static Availability endpointsAvailable(List<Endpoint> zoneEndpoints, - HostNameResolver hostNameResolver, - CNameResolver cNameResolver, + NameResolver nameResolver, HealthChecker healthChecker) { if (zoneEndpoints.isEmpty()) return new Availability(Status.endpointsUnavailable, "Endpoints not yet ready."); for (Endpoint endpoint : zoneEndpoints) { - Optional<InetAddress> resolvedIpAddress = hostNameResolver.resolve(endpoint.url().domain()); - if (resolvedIpAddress.isEmpty()) + Set<String> resolvedIpAddresses = resolveIpAddresses(endpoint.url().domain(), nameResolver); + if (resolvedIpAddresses.isEmpty()) return new Availability(Status.endpointsUnavailable, "DNS lookup yielded no IP address for '" + endpoint.url().domain() + "'."); - if (resolvedIpAddress.equals(endpoint.ipAddress())) // We expect a certain IP address, and that's what we got, so we're good. - continue; - - if (endpoint.ipAddress().isPresent()) // We expect a certain IP address, but that's not what we got. + if (endpoint.ipAddress().isPresent()) { + if (resolvedIpAddresses.contains(endpoint.ipAddress().get().getHostAddress())) { + continue; // Resolved addresses contain the expected endpoint IP address + } return new Availability(Status.endpointsUnavailable, - "IP address of '" + endpoint.url().domain() + "' (" + - resolvedIpAddress.get().getHostAddress() + ") and load balancer " + - "' (" + endpoint.ipAddress().get().getHostAddress() + ") are not equal"); + "IP address(es) of '" + endpoint.url().domain() + "' (" + + resolvedIpAddresses + ") do not include load balancer IP " + + "' (" + endpoint.ipAddress().get().getHostAddress() + ")"); + } if (endpoint.canonicalName().isEmpty()) // We have no expected IP address, and no canonical name, so there's nothing more to check. continue; - Optional<DomainName> cNameValue = cNameResolver.resolve(endpoint.url().domain()); - if (cNameValue.filter(endpoint.canonicalName().get()::equals).isEmpty()) { + List<String> cnameAnswers = nameResolver.resolve(NameType.CNAME, endpoint.url().domain()); + if (!cnameAnswers.contains(endpoint.canonicalName().get().value())) { return new Availability(Status.endpointsUnavailable, "CNAME '" + endpoint.url().domain() + "' points at " + - cNameValue.map(name -> "'" + name + "'").orElse("nothing") + + cnameAnswers + " but should point at load balancer " + endpoint.canonicalName().map(name -> "'" + name + "'").orElse("nothing")); } - Optional<InetAddress> loadBalancerAddress = hostNameResolver.resolve(endpoint.canonicalName().get()); - if ( ! loadBalancerAddress.equals(resolvedIpAddress)) { + Set<String> loadBalancerAddresses = resolveIpAddresses(endpoint.canonicalName().get(), nameResolver); + if ( ! loadBalancerAddresses.equals(resolvedIpAddresses)) { return new Availability(Status.endpointsUnavailable, - "IP address of CNAME '" + endpoint.url().domain() + "' (" + - resolvedIpAddress.get().getHostAddress() + ") and load balancer '" + - endpoint.canonicalName().get() + "' (" + - loadBalancerAddress.map(InetAddress::getHostAddress).orElse("empty") + ") are not equal"); + "IP address(es) of CNAME '" + endpoint.url().domain() + "' (" + + resolvedIpAddresses + ") and load balancer '" + + endpoint.canonicalName().get() + "' (" + loadBalancerAddresses + ") are not equal"); } } @@ -107,38 +108,43 @@ public interface EndpointsChecker { return availability; } - /** Returns the IP address of the given host name, if any. */ - private static Optional<InetAddress> resolveHostName(DomainName hostname) { - try { - return Optional.of(InetAddress.getByName(hostname.value())); - } - catch (UnknownHostException ignored) { - return Optional.empty(); - } + private static Set<String> resolveIpAddresses(DomainName name, NameResolver nameResolver) { + Set<String> answers = new HashSet<>(); + answers.addAll(nameResolver.resolve(NameType.A, name)); + answers.addAll(nameResolver.resolve(NameType.AAAA, name)); + return answers; + } + + enum NameType { + A, AAAA, CNAME } - /** Returns the host name of the given CNAME, if any. */ - private static Optional<DomainName> resolveCname(DomainName endpoint) { + /** Returns all answers for given type and name. An empty list is returned if name does not exist (NXDOMAIN) */ + private static List<String> resolveAll(NameType type, DomainName name) { try { - InitialDirContext ctx = new InitialDirContext(); + DirContext ctx = new InitialDirContext(); try { - Attributes attrs = ctx.getAttributes("dns:/" + endpoint.value(), new String[]{ "CNAME" }); - for (Attribute attribute : Collections.list(attrs.getAll())) { - Enumeration<?> vals = attribute.getAll(); - if (vals.hasMoreElements()) { - String hostname = vals.nextElement().toString(); - return Optional.of(hostname.substring(0, hostname.length() - 1)).map(DomainName::of); - } + String entryType = type.name(); + Attributes attributes = ctx.getAttributes("dns:/" + name, new String[]{entryType}); + Attribute attribute = attributes.get(entryType); + if (attribute == null) { + return List.of(); } - } - finally { + List<String> results = new ArrayList<>(); + attribute.getAll().asIterator().forEachRemaining(value -> { + String answer = Objects.toString(value); + answer = answer.endsWith(".") ? answer.substring(0, answer.length() - 1) : answer; // Trim trailing dot + results.add(answer); + }); + return Collections.unmodifiableList(results); + } finally { ctx.close(); } - } - catch (NamingException e) { + } catch (NameNotFoundException ignored) { + return List.of(); + } catch (NamingException e) { throw new RuntimeException(e); } - return Optional.empty(); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index a0c48200ff6..9cc23b1f508 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -121,21 +121,24 @@ public class NodeResources { } - public record GpuResources(int count, double memoryGb) { + public record GpuResources(int count, double memoryGiB) { private static final GpuResources zero = new GpuResources(0, 0); public GpuResources { - validate(memoryGb, "memory"); + validate(memoryGiB, "memory"); } private boolean lessThan(GpuResources other) { return this.count < other.count || - this.memoryGb < other.memoryGb; + this.memoryGiB < other.memoryGiB; } public boolean isZero() { return this.equals(zero); } + @Deprecated(forRemoval = true) + public double memoryGb() { return memoryGiB; } // Remove after 8.355.18 is gone + public static GpuResources zero() { return zero; } public boolean isDefault() { return this.equals(getDefault()); } @@ -145,20 +148,20 @@ public class NodeResources { public GpuResources plus(GpuResources other) { if (other.isZero()) return this; - var thisMem = this.count() * this.memoryGb(); - var otherMem = other.count() * other.memoryGb(); + var thisMem = this.count() * this.memoryGiB(); + var otherMem = other.count() * other.memoryGiB(); return new NodeResources.GpuResources(1, thisMem + otherMem); } public GpuResources minus(GpuResources other) { if (other.isZero()) return this; - var thisMem = this.count() * this.memoryGb(); - var otherMem = other.count() * other.memoryGb(); + var thisMem = this.count() * this.memoryGiB(); + var otherMem = other.count() * other.memoryGiB(); return new NodeResources.GpuResources(1, thisMem - otherMem); } public GpuResources multipliedBy(double factor) { - return new GpuResources(this.count, this.memoryGb * factor); + return new GpuResources(this.count, this.memoryGiB * factor); } @Override @@ -166,17 +169,17 @@ public class NodeResources { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; GpuResources that = (GpuResources) o; - return count == that.count && equal(this.memoryGb, that.memoryGb); + return count == that.count && equal(this.memoryGiB, that.memoryGiB); } @Override public int hashCode() { - return Objects.hash(count, memoryGb); + return Objects.hash(count, memoryGiB); } } private final double vcpu; - private final double memoryGb; + private final double memoryGiB; private final double diskGb; private final double bandwidthGbps; private final GpuResources gpuResources; @@ -184,25 +187,25 @@ public class NodeResources { private final StorageType storageType; private final Architecture architecture; - public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps) { - this(vcpu, memoryGb, diskGb, bandwidthGbps, DiskSpeed.getDefault()); + public NodeResources(double vcpu, double memoryGiB, double diskGb, double bandwidthGbps) { + this(vcpu, memoryGiB, diskGb, bandwidthGbps, DiskSpeed.getDefault()); } - public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed) { - this(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, StorageType.getDefault(), Architecture.getDefault(), GpuResources.getDefault()); + public NodeResources(double vcpu, double memoryGiB, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed) { + this(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, StorageType.getDefault(), Architecture.getDefault(), GpuResources.getDefault()); } - public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed, StorageType storageType) { - this(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, Architecture.getDefault(), GpuResources.getDefault()); + public NodeResources(double vcpu, double memoryGiB, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed, StorageType storageType) { + this(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, Architecture.getDefault(), GpuResources.getDefault()); } - public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed, StorageType storageType, Architecture architecture) { - this(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, GpuResources.getDefault()); + public NodeResources(double vcpu, double memoryGiB, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed, StorageType storageType, Architecture architecture) { + this(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, GpuResources.getDefault()); } - public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed, StorageType storageType, Architecture architecture, GpuResources gpuResources) { + public NodeResources(double vcpu, double memoryGiB, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed, StorageType storageType, Architecture architecture, GpuResources gpuResources) { this.vcpu = validate(vcpu, "vcpu"); - this.memoryGb = validate(memoryGb, "memory"); + this.memoryGiB = validate(memoryGiB, "memory"); this.diskGb = validate(diskGb, "disk"); this.bandwidthGbps = validate(bandwidthGbps, "bandwidth"); this.gpuResources = gpuResources; @@ -212,7 +215,9 @@ public class NodeResources { } public double vcpu() { return vcpu; } - public double memoryGb() { return memoryGb; } + @Deprecated(forRemoval = true) + public double memoryGb() { return memoryGiB; } // Remove after 8.355.18 is gone + public double memoryGiB() { return memoryGiB; } public double diskGb() { return diskGb; } public double bandwidthGbps() { return bandwidthGbps; } public DiskSpeed diskSpeed() { return diskSpeed; } @@ -221,70 +226,70 @@ public class NodeResources { public GpuResources gpuResources() { return gpuResources; } public boolean vcpuIsUnspecified() { return vcpu == 0; } - public boolean memoryGbIsUnspecified() { return memoryGb == 0; } - public boolean diskGbIsUnspecified() { return diskGb == 0; } + public boolean memoryIsUnspecified() { return memoryGiB == 0; } + public boolean diskIsUnspecified() { return diskGb == 0; } public boolean bandwidthGbpsIsUnspecified() { return bandwidthGbps == 0; } /** Returns the standard cost of these resources, in dollars per hour */ public double cost() { return (vcpu * cpuUnitCost) + - (memoryGb * memoryUnitCost) + + (memoryGiB * memoryUnitCost) + (diskGb * diskUnitCost) + - (gpuResources.count * gpuResources.memoryGb * gpuUnitCost); + (gpuResources.count * gpuResources.memoryGiB * gpuUnitCost); } public NodeResources withVcpu(double vcpu) { if (vcpu == this.vcpu) return this; - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + return new NodeResources(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } - public NodeResources withMemoryGb(double memoryGb) { - if (memoryGb == this.memoryGb) return this; - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + public NodeResources withMemoryGiB(double memoryGiB) { + if (memoryGiB == this.memoryGiB) return this; + return new NodeResources(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } public NodeResources withDiskGb(double diskGb) { if (diskGb == this.diskGb) return this; - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + return new NodeResources(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } public NodeResources withBandwidthGbps(double bandwidthGbps) { ensureSpecified(); if (bandwidthGbps == this.bandwidthGbps) return this; - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + return new NodeResources(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } public NodeResources with(DiskSpeed diskSpeed) { ensureSpecified(); if (diskSpeed == this.diskSpeed) return this; - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + return new NodeResources(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } public NodeResources with(StorageType storageType) { ensureSpecified(); if (storageType == this.storageType) return this; - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + return new NodeResources(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } public NodeResources with(Architecture architecture) { ensureSpecified(); if (architecture == this.architecture) return this; - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + return new NodeResources(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } public NodeResources with(GpuResources gpuResources) { ensureSpecified(); if (this.gpuResources.equals(gpuResources)) return this; - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); + return new NodeResources(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } public NodeResources withUnspecifiedFieldsFrom(NodeResources fullySpecified) { var resources = this; if (resources.vcpuIsUnspecified()) resources = resources.withVcpu(fullySpecified.vcpu()); - if (resources.memoryGbIsUnspecified()) - resources = resources.withMemoryGb(fullySpecified.memoryGb()); - if (resources.diskGbIsUnspecified()) + if (resources.memoryIsUnspecified()) + resources = resources.withMemoryGiB(fullySpecified.memoryGiB()); + if (resources.diskIsUnspecified()) resources = resources.withDiskGb(fullySpecified.diskGb()); if (resources.bandwidthGbpsIsUnspecified()) resources = resources.withBandwidthGbps(fullySpecified.bandwidthGbps()); @@ -307,7 +312,7 @@ public class NodeResources { /** Returns this with all numbers set to 0 */ public NodeResources justNonNumbers() { if (isUnspecified()) return unspecified(); - return withVcpu(0).withMemoryGb(0).withDiskGb(0).withBandwidthGbps(0).with(GpuResources.zero()); + return withVcpu(0).withMemoryGiB(0).withDiskGb(0).withBandwidthGbps(0).with(GpuResources.zero()); } public NodeResources subtract(NodeResources other) { @@ -316,7 +321,7 @@ public class NodeResources { if ( ! this.isInterchangeableWith(other)) throw new IllegalArgumentException(this + " and " + other + " are not interchangeable"); return new NodeResources(vcpu - other.vcpu, - memoryGb - other.memoryGb, + memoryGiB - other.memoryGiB, diskGb - other.diskGb, bandwidthGbps - other.bandwidthGbps, this.diskSpeed.combineWith(other.diskSpeed), @@ -330,7 +335,7 @@ public class NodeResources { if ( ! this.isInterchangeableWith(other)) throw new IllegalArgumentException(this + " and " + other + " are not interchangeable"); return new NodeResources(vcpu + other.vcpu, - memoryGb + other.memoryGb, + memoryGiB + other.memoryGiB, diskGb + other.diskGb, bandwidthGbps + other.bandwidthGbps, this.diskSpeed.combineWith(other.diskSpeed), @@ -342,7 +347,7 @@ public class NodeResources { public NodeResources multipliedBy(double factor) { if (isUnspecified()) return this; return this.withVcpu(vcpu * factor) - .withMemoryGb(memoryGb * factor) + .withMemoryGiB(memoryGiB * factor) .withDiskGb(diskGb * factor) .withBandwidthGbps(bandwidthGbps * factor) .with(gpuResources.multipliedBy(factor)); @@ -365,7 +370,7 @@ public class NodeResources { if (o == this) return true; if ( ! (o instanceof NodeResources other)) return false; if ( ! equal(this.vcpu, other.vcpu)) return false; - if ( ! equal(this.memoryGb, other.memoryGb)) return false; + if ( ! equal(this.memoryGiB, other.memoryGiB)) return false; if ( ! equal(this.diskGb, other.diskGb)) return false; if ( ! equal(this.bandwidthGbps, other.bandwidthGbps)) return false; if ( ! this.gpuResources.equals(other.gpuResources)) return false; @@ -377,7 +382,7 @@ public class NodeResources { @Override public int hashCode() { - return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture); + return Objects.hash(vcpu, memoryGiB, diskGb, bandwidthGbps, diskSpeed, storageType, architecture); } private static StringBuilder appendDouble(StringBuilder sb, double d) { @@ -394,7 +399,7 @@ public class NodeResources { StringBuilder sb = new StringBuilder("[vcpu: "); appendDouble(sb, vcpu); sb.append(", memory: "); - appendDouble(sb, memoryGb); + appendDouble(sb, memoryGiB); sb.append(" Gb, disk: "); appendDouble(sb, diskGb); sb.append(" Gb"); @@ -413,7 +418,7 @@ public class NodeResources { if ( !gpuResources.isDefault()) { sb.append(", gpu count: ").append(gpuResources.count()); sb.append(", gpu memory: "); - appendDouble(sb, gpuResources.memoryGb()); + appendDouble(sb, gpuResources.memoryGiB()); sb.append(" Gb"); } sb.append(']'); @@ -425,7 +430,7 @@ public class NodeResources { ensureSpecified(); other.ensureSpecified(); if (this.vcpu < other.vcpu) return false; - if (this.memoryGb < other.memoryGb) return false; + if (this.memoryGiB < other.memoryGiB) return false; if (this.diskGb < other.diskGb) return false; if (this.bandwidthGbps < other.bandwidthGbps) return false; if (this.gpuResources.lessThan(other.gpuResources)) return false; @@ -452,7 +457,7 @@ public class NodeResources { */ public boolean compatibleWith(NodeResources requested) { if ( ! equal(this.vcpu, requested.vcpu)) return false; - if ( ! equal(this.memoryGb, requested.memoryGb)) return false; + if ( ! equal(this.memoryGiB, requested.memoryGiB)) return false; if (this.storageType == StorageType.local || requested.storageType == StorageType.local) { if ( ! equal(this.diskGb, requested.diskGb)) return false; } @@ -474,7 +479,7 @@ public class NodeResources { */ public boolean equalsWhereSpecified(NodeResources other) { if ( ! equal(this.vcpu, other.vcpu)) return false; - if ( ! equal(this.memoryGb, other.memoryGb)) return false; + if ( ! equal(this.memoryGiB, other.memoryGiB)) return false; if ( ! equal(this.diskGb, other.diskGb)) return false; if ( ! equal(this.bandwidthGbps, other.bandwidthGbps)) return false; if ( ! this.gpuResources.equals(other.gpuResources)) return false; @@ -499,7 +504,7 @@ public class NodeResources { if ( ! this.diskSpeed().compatibleWith(other.diskSpeed())) return Double.MAX_VALUE; if ( ! this.storageType().compatibleWith(other.storageType())) return Double.MAX_VALUE; - double distance = Math.pow(this.vcpu() - other.vcpu(), 2) + Math.pow(this.memoryGb() - other.memoryGb(), 2); + double distance = Math.pow(this.vcpu() - other.vcpu(), 2) + Math.pow(this.memoryGiB() - other.memoryGiB(), 2); if (this.storageType() == StorageType.local || other.storageType() == StorageType.local) distance += Math.pow(this.diskGb() - other.diskGb(), 2); return distance; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 094a7c5c003..ec2ad641677 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -78,6 +78,6 @@ public enum SystemName { return Stream.of(values()).filter(predicate).collect(Collectors.toUnmodifiableSet()); } - public static Set<SystemName> hostedVespa() { return EnumSet.of(main, cd, Public, PublicCd); } + public static Set<SystemName> hostedVespa() { return EnumSet.of(main, cd, Public, PublicCd, PublicCdMigration); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 9c20a42ed76..a017e6cc653 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -104,7 +104,7 @@ public class AllocatedHostsSerializer { private static void toSlime(NodeResources resources, Cursor resourcesObject) { resourcesObject.setDouble(vcpuKey, resources.vcpu()); - resourcesObject.setDouble(memoryKey, resources.memoryGb()); + resourcesObject.setDouble(memoryKey, resources.memoryGiB()); resourcesObject.setDouble(diskKey, resources.diskGb()); resourcesObject.setDouble(bandwidthKey, resources.bandwidthGbps()); resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); @@ -113,7 +113,7 @@ public class AllocatedHostsSerializer { if (!resources.gpuResources().isDefault()) { Cursor gpuObject = resourcesObject.setObject(gpuKey); gpuObject.setLong(gpuCountKey, resources.gpuResources().count()); - gpuObject.setDouble(gpuMemoryKey, resources.gpuResources().memoryGb()); + gpuObject.setDouble(gpuMemoryKey, resources.gpuResources().memoryGiB()); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java index 65ec070d744..473f7685e86 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java @@ -3,7 +3,6 @@ package com.yahoo.config.provision; import com.yahoo.config.provision.NodeResources.Architecture; import com.yahoo.config.provision.NodeResources.DiskSpeed; -import com.yahoo.config.provision.NodeResources.GpuResources; import com.yahoo.config.provision.NodeResources.StorageType; import org.junit.jupiter.api.Test; @@ -93,9 +92,9 @@ class NodeResourcesTest { assertEquals(3, empty.withUnspecifiedFieldsFrom(empty.withVcpu(3)).vcpu()); assertEquals(2, empty.withVcpu(2).withUnspecifiedFieldsFrom(empty.withVcpu(3)).vcpu()); - assertEquals(0, empty.withUnspecifiedFieldsFrom(empty).memoryGb()); - assertEquals(3, empty.withUnspecifiedFieldsFrom(empty.withMemoryGb(3)).memoryGb()); - assertEquals(2, empty.withMemoryGb(2).withUnspecifiedFieldsFrom(empty.withMemoryGb(3)).memoryGb()); + assertEquals(0, empty.withUnspecifiedFieldsFrom(empty).memoryGiB()); + assertEquals(3, empty.withUnspecifiedFieldsFrom(empty.withMemoryGiB(3)).memoryGiB()); + assertEquals(2, empty.withMemoryGiB(2).withUnspecifiedFieldsFrom(empty.withMemoryGiB(3)).memoryGiB()); assertEquals(0, empty.withUnspecifiedFieldsFrom(empty).diskGb()); assertEquals(3, empty.withUnspecifiedFieldsFrom(empty.withDiskGb(3)).diskGb()); @@ -132,7 +131,7 @@ class NodeResourcesTest { var other = resources.with(new NodeResources.GpuResources(4, 32)); var expected = resources.withVcpu(2) - .withMemoryGb(4) + .withMemoryGiB(4) .withDiskGb(6) .withBandwidthGbps(2) .with(new NodeResources.GpuResources(1, 192)); diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt index ce3ec92c38b..fe5b3329c28 100644 --- a/config/CMakeLists.txt +++ b/config/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_define_module( DEPENDS vespalib vespalog - fnet + vespa_fnet EXTERNAL_DEPENDS lz4 diff --git a/config/src/apps/vespa-get-config/CMakeLists.txt b/config/src/apps/vespa-get-config/CMakeLists.txt index 365e94087ad..eca34928870 100644 --- a/config/src/apps/vespa-get-config/CMakeLists.txt +++ b/config/src/apps/vespa-get-config/CMakeLists.txt @@ -5,5 +5,5 @@ vespa_add_executable(config_getvespaconfig_app OUTPUT_NAME vespa-get-config-bin INSTALL bin DEPENDS - config_cloudconfig + vespa_config ) diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index bbad22e52e8..66614a3da4e 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -293,7 +293,11 @@ public class ConfigSubscriber implements AutoCloseable { if (applyOnRestartOnly && ! isInitializing) { // disable any reconfig until restart synchronized (monitor) { - applyOnRestart = applyOnRestartOnly; + if ( ! applyOnRestart) { + log.log(Level.INFO, "Config generation " + generation + " requires restart; " + + "further config changes will not take effect until restart"); + applyOnRestart = true; + } } } diff --git a/config/src/tests/api/CMakeLists.txt b/config/src/tests/api/CMakeLists.txt index 0585d7351d0..5bca0bf8bf1 100644 --- a/config/src/tests/api/CMakeLists.txt +++ b/config/src/tests/api/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_api_test_app TEST SOURCES api.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_api_test_app COMMAND config_api_test_app) vespa_generate_config(config_api_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configagent/CMakeLists.txt b/config/src/tests/configagent/CMakeLists.txt index 9bb1ea4a51a..459caf2af6c 100644 --- a/config/src/tests/configagent/CMakeLists.txt +++ b/config/src/tests/configagent/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_configagent_test_app TEST SOURCES configagent.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configagent_test_app COMMAND config_configagent_test_app) vespa_generate_config(config_configagent_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configfetcher/CMakeLists.txt b/config/src/tests/configfetcher/CMakeLists.txt index 550ae6d9637..e25b2e6be27 100644 --- a/config/src/tests/configfetcher/CMakeLists.txt +++ b/config/src/tests/configfetcher/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_configfetcher_test_app TEST SOURCES configfetcher.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configfetcher_test_app COMMAND config_configfetcher_test_app) vespa_generate_config(config_configfetcher_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configformat/CMakeLists.txt b/config/src/tests/configformat/CMakeLists.txt index 725292a3728..f660cb9b151 100644 --- a/config/src/tests/configformat/CMakeLists.txt +++ b/config/src/tests/configformat/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_configformat_test_app TEST SOURCES configformat.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configformat_test_app COMMAND config_configformat_test_app) vespa_generate_config(config_configformat_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configgen/CMakeLists.txt b/config/src/tests/configgen/CMakeLists.txt index 8b6d0a33d2f..de5901c442e 100644 --- a/config/src/tests/configgen/CMakeLists.txt +++ b/config/src/tests/configgen/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_configgen_test_app TEST SOURCES configgen.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configgen_test_app COMMAND config_configgen_test_app) vespa_generate_config(config_configgen_test_app ../../test/resources/configdefinitions/motd.def) @@ -11,20 +11,20 @@ vespa_add_executable(config_vector_inserter_test_app TEST SOURCES vector_inserter.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_vector_inserter_test_app COMMAND config_vector_inserter_test_app) vespa_add_executable(config_map_inserter_test_app TEST SOURCES map_inserter.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_map_inserter_test_app COMMAND config_map_inserter_test_app) vespa_add_executable(config_value_converter_test_app TEST SOURCES value_converter.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_value_converter_test_app COMMAND config_value_converter_test_app) diff --git a/config/src/tests/configholder/CMakeLists.txt b/config/src/tests/configholder/CMakeLists.txt index 3892c02d108..c086f18d366 100644 --- a/config/src/tests/configholder/CMakeLists.txt +++ b/config/src/tests/configholder/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(config_configholder_test_app TEST SOURCES configholder.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configholder_test_app COMMAND config_configholder_test_app) diff --git a/config/src/tests/configmanager/CMakeLists.txt b/config/src/tests/configmanager/CMakeLists.txt index 348ef940279..7a2285a5061 100644 --- a/config/src/tests/configmanager/CMakeLists.txt +++ b/config/src/tests/configmanager/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_configmanager_test_app TEST SOURCES configmanager.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configmanager_test_app COMMAND config_configmanager_test_app) vespa_generate_config(config_configmanager_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configparser/CMakeLists.txt b/config/src/tests/configparser/CMakeLists.txt index ae71336552d..d14a07c54ae 100644 --- a/config/src/tests/configparser/CMakeLists.txt +++ b/config/src/tests/configparser/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_configparser_test_app TEST SOURCES configparser.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configparser_test_app COMMAND config_configparser_test_app) vespa_generate_config(config_configparser_test_app ../../test/resources/configdefinitions/foo.def) diff --git a/config/src/tests/configretriever/CMakeLists.txt b/config/src/tests/configretriever/CMakeLists.txt index 72d84b243ae..ce2d9d6f1f3 100644 --- a/config/src/tests/configretriever/CMakeLists.txt +++ b/config/src/tests/configretriever/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_configretriever_test_app TEST SOURCES configretriever.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configretriever_test_app COMMAND config_configretriever_test_app) vespa_generate_config(config_configretriever_test_app ../../test/resources/configdefinitions/bootstrap.def) diff --git a/config/src/tests/configuri/CMakeLists.txt b/config/src/tests/configuri/CMakeLists.txt index 8e910ea6c4b..5fa9610b7bf 100644 --- a/config/src/tests/configuri/CMakeLists.txt +++ b/config/src/tests/configuri/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_configuri_test_app TEST SOURCES configuri_test.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_configuri_test_app COMMAND config_configuri_test_app) vespa_generate_config(config_configuri_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/failover/CMakeLists.txt b/config/src/tests/failover/CMakeLists.txt index 0bbea7a4d2b..6f95b32049d 100644 --- a/config/src/tests/failover/CMakeLists.txt +++ b/config/src/tests/failover/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_failover_test_app TEST SOURCES failover.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_failover_test_app COMMAND config_failover_test_app) vespa_generate_config(config_failover_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/failover/failover.cpp b/config/src/tests/failover/failover.cpp index 80d89f41c16..a679b2a0ecd 100644 --- a/config/src/tests/failover/failover.cpp +++ b/config/src/tests/failover/failover.cpp @@ -10,6 +10,7 @@ #include "config-my.h" #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/util/barrier.h> #include <vespa/log/log.h> LOG_SETUP("failover"); diff --git a/config/src/tests/file_acquirer/CMakeLists.txt b/config/src/tests/file_acquirer/CMakeLists.txt index dd7fb77e203..fc647a69e81 100644 --- a/config/src/tests/file_acquirer/CMakeLists.txt +++ b/config/src/tests/file_acquirer/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(config_file_acquirer_test_app TEST SOURCES file_acquirer_test.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_file_acquirer_test_app COMMAND config_file_acquirer_test_app) diff --git a/config/src/tests/file_subscription/CMakeLists.txt b/config/src/tests/file_subscription/CMakeLists.txt index ae321186e1c..2f463ef2358 100644 --- a/config/src/tests/file_subscription/CMakeLists.txt +++ b/config/src/tests/file_subscription/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_file_subscription_test_app TEST SOURCES file_subscription.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_file_subscription_test_app COMMAND config_file_subscription_test_app) vespa_generate_config(config_file_subscription_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/frt/CMakeLists.txt b/config/src/tests/frt/CMakeLists.txt index 770630d0258..152f3aa54ae 100644 --- a/config/src/tests/frt/CMakeLists.txt +++ b/config/src/tests/frt/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_frt_test_app TEST SOURCES frt.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_frt_test_app COMMAND config_frt_test_app) vespa_generate_config(config_frt_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/frtconnectionpool/CMakeLists.txt b/config/src/tests/frtconnectionpool/CMakeLists.txt index 003d3151fc7..2dd25b2b656 100644 --- a/config/src/tests/frtconnectionpool/CMakeLists.txt +++ b/config/src/tests/frtconnectionpool/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_frtconnectionpool_test_app TEST SOURCES frtconnectionpool.cpp DEPENDS - config_cloudconfig + vespa_config GTest::gtest ) vespa_add_test(NAME config_frtconnectionpool_test_app COMMAND config_frtconnectionpool_test_app) diff --git a/config/src/tests/functiontest/CMakeLists.txt b/config/src/tests/functiontest/CMakeLists.txt index cdd3d560c16..1c5d267fecc 100644 --- a/config/src/tests/functiontest/CMakeLists.txt +++ b/config/src/tests/functiontest/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_functiontest_test_app TEST SOURCES functiontest.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_functiontest_test_app COMMAND config_functiontest_test_app) vespa_generate_config(config_functiontest_test_app ../../test/resources/configdefinitions/function-test.def) diff --git a/config/src/tests/getconfig/CMakeLists.txt b/config/src/tests/getconfig/CMakeLists.txt index 5bf20c2a24c..ca98c47a805 100644 --- a/config/src/tests/getconfig/CMakeLists.txt +++ b/config/src/tests/getconfig/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_getconfig_test_app TEST SOURCES getconfig.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_getconfig_test_app COMMAND config_getconfig_test_app) vespa_generate_config(config_getconfig_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/legacysubscriber/CMakeLists.txt b/config/src/tests/legacysubscriber/CMakeLists.txt index 10aa835852d..1a25d6dae2c 100644 --- a/config/src/tests/legacysubscriber/CMakeLists.txt +++ b/config/src/tests/legacysubscriber/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_legacysubscriber_test_app TEST SOURCES legacysubscriber.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_legacysubscriber_test_app COMMAND config_legacysubscriber_test_app) vespa_generate_config(config_legacysubscriber_test_app ../../test/resources/configdefinitions/foo.def) diff --git a/config/src/tests/misc/CMakeLists.txt b/config/src/tests/misc/CMakeLists.txt index adf994ea7e1..7bcc35cfe8d 100644 --- a/config/src/tests/misc/CMakeLists.txt +++ b/config/src/tests/misc/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(config_misc_test_app TEST SOURCES misc.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_executable(config_configsystem_test_app TEST SOURCES configsystem.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_misc_test_app COMMAND config_misc_test_app) vespa_add_test(NAME config_configsystem_test_app COMMAND config_configsystem_test_app) diff --git a/config/src/tests/payload_converter/CMakeLists.txt b/config/src/tests/payload_converter/CMakeLists.txt index 1ee42a1bd48..c006069ffbc 100644 --- a/config/src/tests/payload_converter/CMakeLists.txt +++ b/config/src/tests/payload_converter/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(config_payload_converter_test_app TEST SOURCES payload_converter.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_payload_converter_test_app COMMAND config_payload_converter_test_app) diff --git a/config/src/tests/print/CMakeLists.txt b/config/src/tests/print/CMakeLists.txt index 3dc156e5dc3..852435484ee 100644 --- a/config/src/tests/print/CMakeLists.txt +++ b/config/src/tests/print/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_print_test_app TEST SOURCES print.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_print_test_app COMMAND config_print_test_app) vespa_generate_config(config_print_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/raw_subscription/CMakeLists.txt b/config/src/tests/raw_subscription/CMakeLists.txt index dfe3617aa9f..0473230d3f5 100644 --- a/config/src/tests/raw_subscription/CMakeLists.txt +++ b/config/src/tests/raw_subscription/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_raw_subscription_test_app TEST SOURCES raw_subscription.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_raw_subscription_test_app COMMAND config_raw_subscription_test_app) vespa_generate_config(config_raw_subscription_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/subscriber/CMakeLists.txt b/config/src/tests/subscriber/CMakeLists.txt index 485e70cd8cd..b67e2c8b01d 100644 --- a/config/src/tests/subscriber/CMakeLists.txt +++ b/config/src/tests/subscriber/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_subscriber_test_app TEST SOURCES subscriber.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_subscriber_test_app COMMAND config_subscriber_test_app) vespa_generate_config(config_subscriber_test_app ../../test/resources/configdefinitions/foo.def) diff --git a/config/src/tests/subscription/CMakeLists.txt b/config/src/tests/subscription/CMakeLists.txt index ef0bd54dd4d..f64577a5e5e 100644 --- a/config/src/tests/subscription/CMakeLists.txt +++ b/config/src/tests/subscription/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_subscription_test_app TEST SOURCES subscription.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_subscription_test_app COMMAND config_subscription_test_app) vespa_generate_config(config_subscription_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/trace/CMakeLists.txt b/config/src/tests/trace/CMakeLists.txt index cbebbb7ca08..34e836681cb 100644 --- a/config/src/tests/trace/CMakeLists.txt +++ b/config/src/tests/trace/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(config_trace_test_app TEST SOURCES trace.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_trace_test_app COMMAND config_trace_test_app) diff --git a/config/src/tests/unittest/CMakeLists.txt b/config/src/tests/unittest/CMakeLists.txt index 633486ed110..eba677219c9 100644 --- a/config/src/tests/unittest/CMakeLists.txt +++ b/config/src/tests/unittest/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(config_unittest_test_app TEST SOURCES unittest.cpp DEPENDS - config_cloudconfig + vespa_config ) vespa_add_test(NAME config_unittest_test_app COMMAND config_unittest_test_app) vespa_generate_config(config_unittest_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/vespa/config/CMakeLists.txt b/config/src/vespa/config/CMakeLists.txt index 4d4faebfb7b..eebf89a2dcd 100644 --- a/config/src/vespa/config/CMakeLists.txt +++ b/config/src/vespa/config/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(config_cloudconfig +vespa_add_library(vespa_config SOURCES $<TARGET_OBJECTS:config_common> $<TARGET_OBJECTS:config_subscription> diff --git a/config/src/vespa/config/print/asciiconfigreader.h b/config/src/vespa/config/print/asciiconfigreader.h index b79a500e8a0..bcd2ec88a38 100644 --- a/config/src/vespa/config/print/asciiconfigreader.h +++ b/config/src/vespa/config/print/asciiconfigreader.h @@ -3,6 +3,8 @@ #include "configreader.h" +namespace vespalib { class asciistream; } + namespace config { class ConfigFormatter; diff --git a/config/src/vespa/config/print/asciiconfigreader.hpp b/config/src/vespa/config/print/asciiconfigreader.hpp index 8d95e1af970..213765ac1dc 100644 --- a/config/src/vespa/config/print/asciiconfigreader.hpp +++ b/config/src/vespa/config/print/asciiconfigreader.hpp @@ -5,6 +5,7 @@ #include "asciiconfigreader.h" #include <vespa/config/common/types.h> #include <vespa/config/common/configvalue.h> +#include <vespa/vespalib/stllike/asciistream.h> namespace config { diff --git a/configd/src/apps/cmd/CMakeLists.txt b/configd/src/apps/cmd/CMakeLists.txt index 478de2e4298..ff590ffac5c 100644 --- a/configd/src/apps/cmd/CMakeLists.txt +++ b/configd/src/apps/cmd/CMakeLists.txt @@ -5,6 +5,6 @@ vespa_add_executable(configd_vespa-sentinel-cmd_app OUTPUT_NAME vespa-sentinel-cmd-bin INSTALL bin DEPENDS - fnet + vespa_fnet vespalib ) diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt index d3cfcff4135..d232091ea8a 100644 --- a/configd/src/apps/sentinel/CMakeLists.txt +++ b/configd/src/apps/sentinel/CMakeLists.txt @@ -25,7 +25,7 @@ vespa_add_executable(configd_config-sentinel_app OUTPUT_NAME vespa-config-sentinel INSTALL sbin DEPENDS - fnet - configdefinitions + vespa_fnet + vespa_configdefinitions vespalib ) diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index cff3678f050..e94e7b8f093 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_define_module( DEPENDS vespalib - config_cloudconfig + vespa_config LIBS src/vespa diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 0ab12932880..f3c0dff023a 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -1,89 +1,89 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(configdefinitions +vespa_add_library(vespa_configdefinitions SOURCES INSTALL lib64 DEPENDS ) -vespa_generate_config(configdefinitions application-id.def) +vespa_generate_config(vespa_configdefinitions application-id.def) install_config_definition(application-id.def cloud.config.application-id.def) -vespa_generate_config(configdefinitions attributes.def) +vespa_generate_config(vespa_configdefinitions attributes.def) install_config_definition(attributes.def vespa.config.search.attributes.def) -vespa_generate_config(configdefinitions cluster-info.def) +vespa_generate_config(vespa_configdefinitions cluster-info.def) install_config_definition(cluster-info.def cloud.config.cluster-info.def) -vespa_generate_config(configdefinitions cluster-list.def) +vespa_generate_config(vespa_configdefinitions cluster-list.def) install_config_definition(cluster-list.def cloud.config.cluster-list.def) -vespa_generate_config(configdefinitions configserver.def) +vespa_generate_config(vespa_configdefinitions configserver.def) install_config_definition(configserver.def cloud.config.configserver.def) -vespa_generate_config(configdefinitions curator.def) +vespa_generate_config(vespa_configdefinitions curator.def) install_config_definition(curator.def cloud.config.curator.def) -vespa_generate_config(configdefinitions dispatch.def) +vespa_generate_config(vespa_configdefinitions dispatch.def) install_config_definition(dispatch.def vespa.config.search.dispatch.def) -vespa_generate_config(configdefinitions dispatch-nodes.def) +vespa_generate_config(vespa_configdefinitions dispatch-nodes.def) install_config_definition(dispatch-nodes.def vespa.config.search.dispatch-nodes.def) -vespa_generate_config(configdefinitions fleetcontroller.def) +vespa_generate_config(vespa_configdefinitions fleetcontroller.def) install_config_definition(fleetcontroller.def vespa.config.content.fleetcontroller.def) -vespa_generate_config(configdefinitions ilscripts.def) +vespa_generate_config(vespa_configdefinitions ilscripts.def) install_config_definition(ilscripts.def vespa.configdefinition.ilscripts.def) -vespa_generate_config(configdefinitions imported-fields.def) +vespa_generate_config(vespa_configdefinitions imported-fields.def) install_config_definition(imported-fields.def vespa.config.search.imported-fields.def) -vespa_generate_config(configdefinitions indexschema.def) +vespa_generate_config(vespa_configdefinitions indexschema.def) install_config_definition(indexschema.def vespa.config.search.indexschema.def) -vespa_generate_config(configdefinitions lb-services.def) +vespa_generate_config(vespa_configdefinitions lb-services.def) install_config_definition(lb-services.def cloud.config.lb-services.def) -vespa_generate_config(configdefinitions load-type.def) +vespa_generate_config(vespa_configdefinitions load-type.def) install_config_definition(load-type.def vespa.config.content.load-type.def) -vespa_generate_config(configdefinitions logforwarder.def) +vespa_generate_config(vespa_configdefinitions logforwarder.def) install_config_definition(logforwarder.def cloud.config.logforwarder.def) -vespa_generate_config(configdefinitions open-telemetry.def) +vespa_generate_config(vespa_configdefinitions open-telemetry.def) install_config_definition(open-telemetry.def cloud.config.open-telemetry.def) -vespa_generate_config(configdefinitions messagetyperouteselectorpolicy.def) +vespa_generate_config(vespa_configdefinitions messagetyperouteselectorpolicy.def) install_config_definition(messagetyperouteselectorpolicy.def vespa.config.content.messagetyperouteselectorpolicy.def) -vespa_generate_config(configdefinitions model.def) +vespa_generate_config(vespa_configdefinitions model.def) install_config_definition(model.def cloud.config.model.def) -vespa_generate_config(configdefinitions orchestrator.def) +vespa_generate_config(vespa_configdefinitions orchestrator.def) install_config_definition(orchestrator.def vespa.orchestrator.config.orchestrator.def) -vespa_generate_config(configdefinitions persistence.def) +vespa_generate_config(vespa_configdefinitions persistence.def) install_config_definition(persistence.def vespa.config.content.persistence.def) -vespa_generate_config(configdefinitions rank-profiles.def) +vespa_generate_config(vespa_configdefinitions rank-profiles.def) install_config_definition(rank-profiles.def vespa.config.search.rank-profiles.def) -vespa_generate_config(configdefinitions reindexing.def) +vespa_generate_config(vespa_configdefinitions reindexing.def) install_config_definition(reindexing.def vespa.config.content.reindexing.reindexing.def) -vespa_generate_config(configdefinitions sentinel.def) +vespa_generate_config(vespa_configdefinitions sentinel.def) install_config_definition(sentinel.def cloud.config.sentinel.def) -vespa_generate_config(configdefinitions slobroks.def) +vespa_generate_config(vespa_configdefinitions slobroks.def) install_config_definition(slobroks.def cloud.config.slobroks.def) -vespa_generate_config(configdefinitions specialtokens.def) +vespa_generate_config(vespa_configdefinitions specialtokens.def) install_config_definition(specialtokens.def vespa.configdefinition.specialtokens.def) -vespa_generate_config(configdefinitions distribution.def) +vespa_generate_config(vespa_configdefinitions distribution.def) install_config_definition(distribution.def vespa.config.content.distribution.def) -vespa_generate_config(configdefinitions stor-distribution.def) +vespa_generate_config(vespa_configdefinitions stor-distribution.def) install_config_definition(stor-distribution.def vespa.config.content.stor-distribution.def) -vespa_generate_config(configdefinitions stor-filestor.def) +vespa_generate_config(vespa_configdefinitions stor-filestor.def) install_config_definition(stor-filestor.def vespa.config.content.stor-filestor.def) -vespa_generate_config(configdefinitions summary.def) +vespa_generate_config(vespa_configdefinitions summary.def) install_config_definition(summary.def vespa.config.search.summary.def) -vespa_generate_config(configdefinitions upgrading.def) +vespa_generate_config(vespa_configdefinitions upgrading.def) install_config_definition(upgrading.def vespa.config.content.upgrading.def) -vespa_generate_config(configdefinitions zookeeper-server.def) +vespa_generate_config(vespa_configdefinitions zookeeper-server.def) install_config_definition(zookeeper-server.def cloud.config.zookeeper-server.def) -vespa_generate_config(configdefinitions zookeepers.def) +vespa_generate_config(vespa_configdefinitions zookeepers.def) install_config_definition(zookeepers.def cloud.config.zookeepers.def) -vespa_generate_config(configdefinitions bucketspaces.def) +vespa_generate_config(vespa_configdefinitions bucketspaces.def) install_config_definition(bucketspaces.def vespa.config.content.core.bucketspaces.def) -vespa_generate_config(configdefinitions all-clusters-bucket-spaces.def) +vespa_generate_config(vespa_configdefinitions all-clusters-bucket-spaces.def) install_config_definition(all-clusters-bucket-spaces.def vespa.config.content.all-clusters-bucket-spaces.def) -vespa_generate_config(configdefinitions stateserver.def) +vespa_generate_config(vespa_configdefinitions stateserver.def) install_config_definition(stateserver.def vespa.config.core.stateserver.def) -vespa_generate_config(configdefinitions ranking-constants.def) +vespa_generate_config(vespa_configdefinitions ranking-constants.def) install_config_definition(ranking-constants.def vespa.config.search.core.ranking-constants.def) -vespa_generate_config(configdefinitions ranking-expressions.def) +vespa_generate_config(vespa_configdefinitions ranking-expressions.def) install_config_definition(ranking-expressions.def vespa.config.search.core.ranking-expressions.def) -vespa_generate_config(configdefinitions onnx-models.def) +vespa_generate_config(vespa_configdefinitions onnx-models.def) install_config_definition(onnx-models.def vespa.config.search.core.onnx-models.def) -vespa_generate_config(configdefinitions proton.def) +vespa_generate_config(vespa_configdefinitions proton.def) install_config_definition(proton.def vespa.config.search.core.proton.def) -vespa_generate_config(configdefinitions hwinfo.def) -vespa_generate_config(configdefinitions dataplane-proxy.def) +vespa_generate_config(vespa_configdefinitions hwinfo.def) +vespa_generate_config(vespa_configdefinitions dataplane-proxy.def) install_config_definition(dataplane-proxy.def cloud.config.dataplane-proxy.def) install_config_definition(hugging-face-embedder.def embedding.huggingface.hugging-face-embedder.def) install_config_definition(hugging-face-tokenizer.def language.huggingface.config.hugging-face-tokenizer.def) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index c2c42054109..78a4d71158f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -6,13 +6,13 @@ import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; import ai.vespa.http.HttpURL.Scheme; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; @@ -70,6 +70,7 @@ import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.deploy.Deployment; import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SecretStoreValidator; @@ -101,6 +102,7 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -111,7 +113,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -774,10 +775,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public List<Version> getAllVersions(ApplicationId applicationId) { - Optional<ApplicationVersions> applicationSet = getActiveApplicationSet(applicationId); - return applicationSet.isEmpty() - ? List.of() - : applicationSet.get().versions(applicationId); + return getActiveApplicationVersions(applicationId).map(ApplicationVersions::versions).orElse(List.of()); } public HttpResponse validateSecretStore(ApplicationId applicationId, SystemName systemName, Slime slime) { @@ -812,8 +810,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Logs ---------------------------------------------------------------- public HttpResponse getLogs(ApplicationId applicationId, Optional<DomainName> hostname, Query apiParams) { - HttpURL logServerURI = getLogServerURI(applicationId, hostname).withQuery(apiParams); - return logRetriever.getLogs(logServerURI, activationTime(applicationId)); + Exception exception = null; + for (var uri : getLogServerUris(applicationId, hostname)) { + try { + return logRetriever.getLogs(uri.withQuery(apiParams), activationTime(applicationId)); + } catch (RuntimeException e) { + exception = e; + log.log(Level.INFO, e.getMessage()); + } + } + return HttpErrorResponse.internalServerError(Exceptions.toMessageString(exception)); } // ---------------- Methods to do call against tester containers in hosted ------------------------------ @@ -899,15 +905,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @return the active session, or null if there is no active session for the given application id. */ public Optional<Session> getActiveSession(ApplicationId applicationId) { - return getActiveRemoteSession(applicationId); - } - - /** - * Gets the active Session for the given application id. - * - * @return the active session, or null if there is no active session for the given application id. - */ - public Optional<Session> getActiveRemoteSession(ApplicationId applicationId) { Tenant tenant = getTenant(applicationId); if (tenant == null) throw new IllegalArgumentException("Could not find any tenant for '" + applicationId + "'"); return getActiveSession(tenant, applicationId); @@ -915,15 +912,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public long getSessionIdForApplication(ApplicationId applicationId) { Tenant tenant = getTenant(applicationId); - if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); - return getSessionIdForApplication(tenant, applicationId); - } - - private long getSessionIdForApplication(Tenant tenant, ApplicationId applicationId) { - TenantApplications applicationRepo = tenant.getApplicationRepo(); - if (! applicationRepo.exists(applicationId)) + if (tenant == null) + throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); + if (! tenant.getApplicationRepo().exists(applicationId)) throw new NotFoundException("Unknown application id '" + applicationId + "'"); - return applicationRepo.requireActiveSessionOf(applicationId); + + return requireActiveSession(tenant, applicationId).getSessionId(); } public void validateThatSessionIsNotActive(Tenant tenant, long sessionId) { @@ -944,7 +938,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye DeployLogger deployLogger) { Tenant tenant = getTenant(applicationId); SessionRepository sessionRepository = tenant.getSessionRepository(); - Session fromSession = getExistingSession(tenant, applicationId); + Session fromSession = requireActiveSession(tenant, applicationId); return sessionRepository.createSessionFromExisting(fromSession, internalRedeploy, timeoutBudget, deployLogger).getSessionId(); } @@ -972,10 +966,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - public int deleteExpiredRemoteSessions(Clock clock) { + public int deleteExpiredRemoteSessions() { return tenantRepository.getAllTenants() .stream() - .map(tenant -> tenant.getSessionRepository().deleteExpiredRemoteSessions(clock, session -> sessionIsActiveForItsApplication(tenant, session))) + .map(tenant -> tenant.getSessionRepository().deleteExpiredRemoteSessions(session -> sessionIsActiveForItsApplication(tenant, session))) .mapToInt(i -> i) .sum(); } @@ -1103,7 +1097,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return session; } - public Optional<ApplicationVersions> getActiveApplicationSet(ApplicationId appId) { + public Optional<ApplicationVersions> getActiveApplicationVersions(ApplicationId appId) { return getTenant(appId).getSessionRepository().activeApplicationVersions(appId); } @@ -1130,10 +1124,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - // TODO: Merge this and getActiveSession(), they are almost identical - private Session getExistingSession(Tenant tenant, ApplicationId applicationId) { - TenantApplications applicationRepo = tenant.getApplicationRepo(); - return getRemoteSession(tenant, applicationRepo.requireActiveSessionOf(applicationId)); + private Session requireActiveSession(Tenant tenant, ApplicationId applicationId) { + return getActiveSession(tenant, applicationId) + .orElseThrow(() -> new IllegalArgumentException("Application '" + applicationId + "' has no active session.")); } public Optional<Session> getActiveSession(Tenant tenant, ApplicationId applicationId) { @@ -1185,33 +1178,41 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - private HttpURL getLogServerURI(ApplicationId applicationId, Optional<DomainName> hostname) { + private List<HttpURL> getLogServerUris(ApplicationId applicationId, Optional<DomainName> hostname) { // Allow to get logs from a given hostname if the application is under the hosted-vespa tenant. // We make no validation that the hostname is actually allocated to the given application since // most applications under hosted-vespa are not known to the model, and it's OK for a user to get // logs for any host if they are authorized for the hosted-vespa tenant. if (hostname.isPresent() && HOSTED_VESPA_TENANT.equals(applicationId.tenant())) { int port = List.of(InfrastructureApplication.CONFIG_SERVER.id(), InfrastructureApplication.CONTROLLER.id()).contains(applicationId) ? 19071 : 8080; - return HttpURL.create(Scheme.http, hostname.get(), port).withPath(HttpURL.Path.empty().append("logs")); + return List.of(HttpURL.create(Scheme.http, hostname.get(), port).withPath(HttpURL.Path.parse("logs"))); } - Application application = getApplication(applicationId); - Collection<HostInfo> hostInfos = application.getModel().getHosts(); + ApplicationVersions applicationVersions = getActiveApplicationVersions(applicationId) + .orElseThrow(() -> new NotFoundException("Unable to get logs for for " + applicationId + " (application not found)")); + List<Pair<String, Integer>> hostInfo = logserverHostInfo(applicationVersions); + return hostInfo.stream() + .map(h -> HttpURL.create(Scheme.http, DomainName.of(h.getFirst()), h.getSecond(), HttpURL.Path.parse("logs"))) + .toList(); + } - HostInfo logServerHostInfo = hostInfos.stream() - .filter(host -> host.getServices().stream() + // Returns a list with hostname and port pairs for logserver container for all models/versions + private List<Pair<String, Integer>> logserverHostInfo(ApplicationVersions applicationVersions) { + return applicationVersions.applications().stream() + .peek(app -> log.log(Level.FINE, "Finding logserver host and port for version " + app.getVespaVersion())) + .map(Application::getModel) + .flatMap(model -> model.getHosts().stream()) + .filter(hostInfo -> hostInfo.getServices().stream() .anyMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver"))) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find host info for logserver")); - - ServiceInfo logService = logServerHostInfo.getServices().stream() - .filter(service -> LOGSERVER_CONTAINER.serviceName.equals(service.getServiceType())) - .findFirst() - .or(() -> logServerHostInfo.getServices().stream() - .filter(service -> CONTAINER.serviceName.equals(service.getServiceType())) - .findFirst()) - .orElseThrow(() -> new IllegalArgumentException("No container running on logserver host")); - int port = servicePort(logService); - return HttpURL.create(Scheme.http, DomainName.of(logServerHostInfo.getHostname()), port, HttpURL.Path.empty().append("logs")); + .map(hostInfo -> hostInfo.getServices().stream() + .filter(service -> LOGSERVER_CONTAINER.serviceName.equals(service.getServiceType())) + .findFirst() + .or(() -> hostInfo.getServices().stream() + .filter(service -> CONTAINER.serviceName.equals(service.getServiceType())) + .findFirst()) + .orElseThrow(() -> new IllegalArgumentException("No container running on logserver host"))) + .map(s -> new Pair<>(s.getHostName(), servicePort(s))) + .toList(); } private int servicePort(ServiceInfo serviceInfo) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index c5204f5690f..6d0f0e3af68 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.config.server; import com.yahoo.component.annotation.Inject; @@ -13,7 +12,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.GenerationCounter; import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; -import com.yahoo.vespa.flags.FlagSource; import java.time.Instant; import java.util.ArrayList; @@ -33,7 +31,6 @@ public class SuperModelManager implements SuperModelProvider { private final Zone zone; private final Object monitor = new Object(); - private final FlagSource flagSource; private SuperModelConfigProvider superModelConfigProvider; // Guarded by 'this' monitor private final List<SuperModelListener> listeners = new ArrayList<>(); // Guarded by 'this' monitor @@ -46,11 +43,7 @@ public class SuperModelManager implements SuperModelProvider { private final Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty(); @Inject - public SuperModelManager(ConfigserverConfig configserverConfig, - Zone zone, - GenerationCounter generationCounter, - FlagSource flagSource) { - this.flagSource = flagSource; + public SuperModelManager(ConfigserverConfig configserverConfig, Zone zone, GenerationCounter generationCounter) { this.zone = zone; this.generationCounter = generationCounter; this.masterGeneration = configserverConfig.masterGeneration(); @@ -123,12 +116,12 @@ public class SuperModelManager implements SuperModelProvider { // there is no need to bump generation counter. logger.log(Level.FINE, "Super model is complete"); SuperModel newSuperModel = getSuperModel().cloneAsComplete(); - superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); + superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone); listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel)); } private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) { generation = masterGeneration + generationCounter.get(); - superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); + superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java index ccf8f7d50ea..b39377eb30d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java @@ -30,10 +30,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Phaser; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; -import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static com.yahoo.slime.SlimeUtils.entriesStream; import static com.yahoo.slime.SlimeUtils.jsonToSlime; -import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; /** @@ -57,8 +55,7 @@ public class ActiveTokenFingerprintsClient implements ActiveTokenFingerprints, A return getFingerprints(application.getModel().getHosts().stream() .filter(host -> containersWithTokenFilter.contains(host.getHostname())) .flatMap(host -> host.getServices().stream()) - .filter(service -> service.getServiceType().equals(CONTAINER.serviceName) - || service.getServiceType().equals(QRSERVER.serviceName)) + .filter(service -> service.getServiceType().equals(CONTAINER.serviceName)) .toList()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java index 458815c500b..553a82f11e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java @@ -12,7 +12,8 @@ import java.util.List; import java.util.Optional; /** - * Immutable set of {@link Application}s with the same {@link ApplicationId}, applications have difference vespa versions. + * Immutable set of {@link Application}s with the same {@link ApplicationId}, + * applications have different vespa versions. There will always be at least one application version. * * @author vegard */ @@ -24,23 +25,17 @@ public final class ApplicationVersions { private final HashMap<Version, Application> applications = new HashMap<>(); private ApplicationVersions(List<Application> applications) { - if (applications.isEmpty()) throw new IllegalArgumentException("application list cannot be empty"); + if (applications.isEmpty()) + throw new IllegalArgumentException("application list cannot be empty"); + if (applications.stream().map(Application::getId).distinct().count() > 1) + throw new IllegalArgumentException("All application ids must be equal"); + if (applications.stream().map(Application::getApplicationGeneration).distinct().count() > 1) + throw new IllegalArgumentException("All config generations must be equal"); Application firstApp = applications.get(0); applicationId = firstApp.getId(); generation = firstApp.getApplicationGeneration(); - for (Application application : applications) { - this.applications.put(application.getVespaVersion(), application); - ApplicationId applicationId = application.getId(); - if ( ! applicationId.equals(this.applicationId)) { - throw new IllegalArgumentException("Trying to create set with different application ids (" + - application + " and " + this.applicationId + ")"); - } - if ( ! application.getApplicationGeneration().equals(generation)) { - throw new IllegalArgumentException("Trying to create set with different generations (" + - generation + " and " + this.generation + ")"); - } - } + applications.forEach(application -> this.applications.put(application.getVespaVersion(), application)); latestVersion = this.applications.keySet().stream().max(Version::compareTo).get(); } @@ -71,7 +66,7 @@ public final class ApplicationVersions { if (application != null) return Optional.of(application); - // Does the latest version specify we can use it regardless? + // Does the latest version specify that we can use it regardless? Application latest = applications.get(latestVersion); if (latest.getModel().allowModelVersionMismatch(now)) return Optional.of(latest); @@ -90,6 +85,7 @@ public final class ApplicationVersions { return applications.values().stream() .flatMap(app -> app.getModel().getHosts().stream() .map(HostInfo::getHostname)) + .distinct() .toList(); } @@ -101,13 +97,12 @@ public final class ApplicationVersions { return generation; } - List<Application> applications() { + public List<Application> applications() { return new ArrayList<>(applications.values()); } - public List<Version> versions(ApplicationId applicationId) { + public List<Version> versions() { return applications.values().stream() - .filter(application -> application.getId().equals(applicationId)) .map(Application::getVespaVersion) .sorted() .toList(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index c03cb7ade8c..cfc1fea0fc5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -52,7 +52,6 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERC import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; /** * Checks for convergence of config generation for a given application. @@ -67,7 +66,6 @@ public class ConfigConvergenceChecker extends AbstractComponent { private final static Set<String> serviceTypesToCheck = Set.of( CONTAINER.serviceName, - QRSERVER.serviceName, LOGSERVER_CONTAINER.serviceName, CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName, @@ -151,7 +149,7 @@ public class ConfigConvergenceChecker extends AbstractComponent { } private boolean isNotContainer(ServiceInfo serviceInfo) { - return ! List.of(CONTAINER.serviceName, QRSERVER.serviceName, METRICS_PROXY_CONTAINER).contains(serviceInfo.getServiceType()); + return ! List.of(CONTAINER.serviceName, METRICS_PROXY_CONTAINER).contains(serviceInfo.getServiceType()); } // Don't check service in a cluster which uses restartOnDeploy (new config will not be used until service is restarted) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index b83a8290cac..7b31b77d43c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -242,10 +242,7 @@ public class TenantApplications implements RequestHandler, HostValidator { } private void notifyConfigActivationListeners(ApplicationVersions applicationVersions) { - List<Application> applications = applicationVersions.applications(); - if (applications.isEmpty()) throw new IllegalArgumentException("application set cannot be empty"); - - hostRegistry.update(applications.get(0).getId(), applicationVersions.allHosts()); + hostRegistry.update(applicationVersions.getId(), applicationVersions.allHosts()); configActivationListener.configActivated(applicationVersions); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1c285270cb1..4b86d8834b9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -180,7 +180,6 @@ public class ModelContextImpl implements ModelContext { private final double feedNiceness; private final List<String> allowedAthenzProxyIdentities; private final int maxActivationInhibitedOutOfSyncGroups; - private final Predicate<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; private final double resourceLimitDisk; private final double resourceLimitMemory; private final double minNodeRatioPerGroup; @@ -213,6 +212,9 @@ public class ModelContextImpl implements ModelContext { private final boolean logserverOtelCol; private final SharedHosts sharedHosts; private final Architecture adminClusterArchitecture; + private final boolean symmetricPutAndActivateReplicaSelection; + private final boolean enforceStrictlyIncreasingClusterStateVersions; + private final boolean launchApplicationAthenzService; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(source).with(appId).with(version).value(); @@ -225,7 +227,6 @@ public class ModelContextImpl implements ModelContext { this.mbus_network_threads = Flags.MBUS_NUM_NETWORK_THREADS.bindTo(source).with(appId).with(version).value(); this.allowedAthenzProxyIdentities = Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES.bindTo(source).with(appId).with(version).value(); this.maxActivationInhibitedOutOfSyncGroups = Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS.bindTo(source).with(appId).with(version).value(); - this.jvmOmitStackTraceInFastThrow = type -> PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW.bindTo(source).with(appId).with(version).with(type).value(); this.resourceLimitDisk = PermanentFlags.RESOURCE_LIMIT_DISK.bindTo(source).with(appId).with(version).value(); this.resourceLimitMemory = PermanentFlags.RESOURCE_LIMIT_MEMORY.bindTo(source).with(appId).with(version).value(); this.minNodeRatioPerGroup = Flags.MIN_NODE_RATIO_PER_GROUP.bindTo(source).with(appId).with(version).value(); @@ -259,6 +260,9 @@ public class ModelContextImpl implements ModelContext { this.logserverOtelCol = Flags.LOGSERVER_OTELCOL_AGENT.bindTo(source).with(appId).with(version).value(); this.sharedHosts = PermanentFlags.SHARED_HOST.bindTo(source).with( appId).with(version).value(); this.adminClusterArchitecture = Architecture.valueOf(PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE.bindTo(source).with(appId).with(version).value()); + this.symmetricPutAndActivateReplicaSelection = Flags.SYMMETRIC_PUT_AND_ACTIVATE_REPLICA_SELECTION.bindTo(source).with(appId).with(version).value(); + this.enforceStrictlyIncreasingClusterStateVersions = Flags.ENFORCE_STRICTLY_INCREASING_CLUSTER_STATE_VERSIONS.bindTo(source).with(appId).with(version).value(); + this.launchApplicationAthenzService = Flags.LAUNCH_APPLICATION_ATHENZ_SERVICE.bindTo(source).with(appId).with(version).value(); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -275,9 +279,6 @@ public class ModelContextImpl implements ModelContext { @Override public int mbusNetworkThreads() { return mbus_network_threads; } @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } - @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { - return translateJvmOmitStackTraceInFastThrowToString(jvmOmitStackTraceInFastThrow, type); - } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @@ -313,12 +314,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean logserverOtelCol() { return logserverOtelCol; } @Override public SharedHosts sharedHosts() { return sharedHosts; } @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } - - private String translateJvmOmitStackTraceInFastThrowToString(Predicate<ClusterSpec.Type> function, - ClusterSpec.Type clusterType) { - return function.test(clusterType) ? "" : "-XX:-OmitStackTraceInFastThrow"; - } - + @Override public boolean symmetricPutAndActivateReplicaSelection() { return symmetricPutAndActivateReplicaSelection; } + @Override public boolean enforceStrictlyIncreasingClusterStateVersions() { return enforceStrictlyIncreasingClusterStateVersions; } } public static class Properties implements ModelContext.Properties { @@ -350,6 +347,8 @@ public class ModelContextImpl implements ModelContext { private final List<DataplaneToken> dataplaneTokens; private final boolean allowUserFilters; private final Duration endpointConnectionTtl; + private final List<String> requestPrefixForLoggingContent; + private final boolean launchApplicationAthenzService; public Properties(ApplicationId applicationId, Version modelVersion, @@ -396,6 +395,8 @@ public class ModelContextImpl implements ModelContext { this.allowUserFilters = PermanentFlags.ALLOW_USER_FILTERS.bindTo(flagSource).with(applicationId).value(); this.endpointConnectionTtl = Duration.ofSeconds(PermanentFlags.ENDPOINT_CONNECTION_TTL.bindTo(flagSource).with(applicationId).value()); this.dataplaneTokens = dataplaneTokens; + this.requestPrefixForLoggingContent = PermanentFlags.LOG_REQUEST_CONTENT.bindTo(flagSource).with(applicationId).value(); + this.launchApplicationAthenzService = Flags.LAUNCH_APPLICATION_ATHENZ_SERVICE.bindTo(flagSource).with(applicationId).value(); } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -493,5 +494,10 @@ public class ModelContextImpl implements ModelContext { @Override public boolean allowUserFilters() { return allowUserFilters; } @Override public Duration endpointConnectionTtl() { return endpointConnectionTtl; } + + @Override public List<String> requestPrefixForLoggingContent() { return requestPrefixForLoggingContent; } + + @Override public boolean launchApplicationAthenzService() { return launchApplicationAthenzService; } } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java index a634519dd0a..adfc02e3966 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java @@ -109,9 +109,9 @@ public class ZooKeeperDeployer { curator.create(sessionPath); for (String subPath : List.of(DEFCONFIGS_ZK_SUBPATH, - USER_DEFCONFIGS_ZK_SUBPATH, - USERAPP_ZK_SUBPATH, - ZKApplicationPackage.fileRegistryNode)) { + USER_DEFCONFIGS_ZK_SUBPATH, + USERAPP_ZK_SUBPATH, + ZKApplicationPackage.fileRegistryNode)) { // TODO: The replaceFirst below is hackish. curator.create(getZooKeeperAppPath().append(subPath.replaceFirst("/", ""))); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index acfa8e455c0..89724f9853d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -38,7 +38,7 @@ public class LogRetriever { if (deployTime.isPresent() && Instant.now().isBefore(deployTime.get().plus(Duration.ofMinutes(2)))) return new EmptyResponse(); - return HttpErrorResponse.internalServerError("Failed to get logs: " + Exceptions.toMessageString(e)); + throw new RuntimeException("Failed to get logs from " + logServerUri, e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index a18f96e83bb..e25b0a13f42 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -72,7 +72,7 @@ public class TesterClient { } private HttpURL testerUrl(String testerHostname, int port, String... path) { - return HttpURL.create(Scheme.https, DomainName.of(testerHostname), port, Path.empty().append(List.of(path))); + return HttpURL.create(Scheme.https, DomainName.of(testerHostname), port, Path.empty().append(List.of(path)).withoutTrailingSlash()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 4dcb2e44f36..6c4a3d22659 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -267,7 +267,7 @@ public class ApplicationHandler extends HttpHandler { private Model getActiveModelOrThrow(ApplicationId id) { - return applicationRepository.getActiveApplicationSet(id) + return applicationRepository.getActiveApplicationVersions(id) .orElseThrow(() -> new NotFoundException("Application '" + id + "' not found")) .getForVersionOrLatest(Optional.empty(), applicationRepository.clock().instant()) .getModel(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 76879ccf8ae..63667598063 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -3,12 +3,12 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -21,6 +21,7 @@ import com.yahoo.vespa.filedistribution.FileReferenceDownload; import java.io.File; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.Future; @@ -28,6 +29,8 @@ import java.util.logging.Logger; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; +import static com.yahoo.vespa.config.server.session.Session.Status.ACTIVATE; +import static com.yahoo.vespa.config.server.session.Session.Status.PREPARE; /** * Verifies that all active sessions has an application package on local disk. @@ -57,62 +60,66 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { int[] failures = new int[1]; List<Runnable> futureDownloads = new ArrayList<>(); - for (TenantName tenantName : applicationRepository.tenantRepository().getAllTenantNames()) { - for (Session session : applicationRepository.tenantRepository().getTenant(tenantName).getSessionRepository().getRemoteSessions()) { - if (shuttingDown()) - break; - - switch (session.getStatus()) { - case PREPARE, ACTIVATE: - break; - default: - continue; - } - - ApplicationId applicationId = session.getOptionalApplicationId().orElse(null); - if (applicationId == null) // dry-run sessions have no application id - continue; - - log.finest(() -> "Verifying application package for " + applicationId); - - Optional<FileReference> appFileReference = session.getApplicationPackageReference(); - if (appFileReference.isPresent()) { - long sessionId = session.getSessionId(); - attempts++; - if (!fileReferenceExistsOnDisk(downloadDirectory, appFileReference.get())) { - log.fine(() -> "Downloading application package with file reference " + appFileReference + - " for " + applicationId + " (session " + sessionId + ")"); - - FileReferenceDownload download = new FileReferenceDownload(appFileReference.get(), - this.getClass().getSimpleName(), - false); - Future<Optional<File>> futureDownload = fileDownloader.getFutureFileOrTimeout(download); - futureDownloads.add(() -> { - try { - if (futureDownload.get().isPresent()) { - createLocalSessionIfMissing(applicationId, sessionId); - return; - } + for (Session session : preparedAndActivatedSessions()) { + if (shuttingDown()) + return asSuccessFactorDeviation(attempts, failures[0]); + + ApplicationId applicationId = session.getOptionalApplicationId().orElse(null); + if (applicationId == null) // dry-run sessions have no application id + continue; + + Optional<FileReference> appFileReference = session.getApplicationPackageReference(); + if (appFileReference.isPresent()) { + long sessionId = session.getSessionId(); + FileReference fileReference = appFileReference.get(); + + attempts++; + if (! fileReferenceExistsOnDisk(downloadDirectory, fileReference)) { + Future<Optional<File>> futureDownload = startDownload(fileReference, sessionId, applicationId); + futureDownloads.add(() -> { + try { + if (futureDownload.get().isPresent()) { + createLocalSessionIfMissing(applicationId, sessionId); + return; } - catch (Exception ignored) { } - failures[0]++; - log.info("Downloading application package (" + appFileReference + ")" + - " for " + applicationId + " (session " + sessionId + ") unsuccessful. " + - "Can be ignored unless it happens many times over a long period of time, retries is expected"); - }); - } - else { - createLocalSessionIfMissing(applicationId, sessionId); - } + } + catch (Exception e) { + log.warning("Exception when downloading application package (" + fileReference + ")" + + " for " + applicationId + " (session " + sessionId + "): " + e.getMessage()); + } + failures[0]++; + log.info("Downloading application package (" + fileReference + ")" + + " for " + applicationId + " (session " + sessionId + ") unsuccessful. " + + "Can be ignored unless it happens many times over a long period of time, retries is expected"); + }); + } + else { + createLocalSessionIfMissing(applicationId, sessionId); } } } - futureDownloads.forEach(Runnable::run); - return asSuccessFactorDeviation(attempts, failures[0]); } + private Future<Optional<File>> startDownload(FileReference fileReference, long sessionId, ApplicationId applicationId) { + log.fine(() -> "Downloading application package with file reference " + fileReference + + " for " + applicationId + " (session " + sessionId + ")"); + return fileDownloader.getFutureFileOrTimeout(new FileReferenceDownload(fileReference, + this.getClass().getSimpleName(), + false)); + } + + private Collection<RemoteSession> preparedAndActivatedSessions() { + var tenantRepository = applicationRepository.tenantRepository(); + return tenantRepository.getAllTenantNames().stream() + .map(tenantRepository::getTenant) + .map(t -> t.getSessionRepository().getRemoteSessions()) + .flatMap(Collection::stream) + .filter(s -> s.getStatus() == PREPARE || s.getStatus() == ACTIVATE) + .toList(); + } + private static FileDownloader createFileDownloader(ApplicationRepository applicationRepository, File downloadDirectory, Supervisor supervisor) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java index 71e6d9e013d..b44fdfb3a9c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java @@ -32,11 +32,18 @@ public abstract class ConfigServerMaintainer extends Maintainer { /** Creates a maintainer where maintainers on different nodes in this cluster run with even delay. */ ConfigServerMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource, Clock clock, Duration interval, boolean useLock) { + this(applicationRepository, curator, flagSource, clock, interval, useLock, false); + } + + /** Creates a maintainer where maintainers on different nodes in this cluster run with even delay. */ + ConfigServerMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource, + Clock clock, Duration interval, boolean useLock, boolean ignoreCollision) { super(null, interval, clock, new JobControl(new JobControlFlags(curator, flagSource, useLock)), - new ConfigServerJobMetrics(applicationRepository.metric()), cluster(curator), false); + new ConfigServerJobMetrics(applicationRepository.metric()), cluster(curator), ignoreCollision); this.applicationRepository = applicationRepository; } + private static class ConfigServerJobMetrics extends JobMetrics { private final Metric metric; @@ -47,7 +54,7 @@ public abstract class ConfigServerMaintainer extends Maintainer { @Override public void completed(String job, double successFactorDeviation, long durationMs) { - var context = metric.createContext(Map.of("job", job)); + var context = metric.createContext(Map.of("maintainer", job)); metric.set("maintenance.successFactorDeviation", successFactorDeviation, context); metric.set("maintenance.duration", durationMs, context); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/PendingRestartsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/PendingRestartsMaintainer.java index e228e0edcb6..11a15308768 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/PendingRestartsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/PendingRestartsMaintainer.java @@ -41,7 +41,7 @@ public class PendingRestartsMaintainer extends ConfigServerMaintainer { for (Tenant tenant : applicationRepository.tenantRepository().getAllTenants()) { ApplicationCuratorDatabase database = tenant.getApplicationRepo().database(); for (ApplicationId id : database.activeApplications()) - applicationRepository.getActiveApplicationSet(id) + applicationRepository.getActiveApplicationVersions(id) .map(application -> application.getForVersionOrLatest(Optional.empty(), clock.instant())) .ifPresent(application -> { try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java index 0dc6cc004be..6688a0aafdd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java @@ -16,7 +16,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collection; -import java.util.Comparator; import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; @@ -61,7 +60,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { for (Tenant tenant : applicationRepository.tenantRepository().getAllTenants()) { ApplicationCuratorDatabase database = tenant.getApplicationRepo().database(); for (ApplicationId id : database.activeApplications()) - applicationRepository.getActiveApplicationSet(id) + applicationRepository.getActiveApplicationVersions(id) .map(application -> application.getForVersionOrLatest(Optional.empty(), clock.instant())) .ifPresent(application -> { try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index 844b667fd85..d088f42b54d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -17,14 +17,15 @@ import java.util.logging.Level; public class SessionsMaintainer extends ConfigServerMaintainer { SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { - super(applicationRepository, curator, applicationRepository.flagSource(), applicationRepository.clock(), interval, true); + super(applicationRepository, curator, applicationRepository.flagSource(), applicationRepository.clock(), + interval, true, true); } @Override protected double maintain() { applicationRepository.deleteExpiredLocalSessions(); - int deleted = applicationRepository.deleteExpiredRemoteSessions(applicationRepository.clock()); + int deleted = applicationRepository.deleteExpiredRemoteSessions(); log.log(Level.FINE, () -> "Deleted " + deleted + " expired remote sessions"); return 1.0; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java index 907928b1d40..7974159e531 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java @@ -47,10 +47,9 @@ public class ClusterDeploymentMetricsRetriever { private static final Logger log = Logger.getLogger(ClusterDeploymentMetricsRetriever.class.getName()); private static final String VESPA_CONTAINER = "vespa.container"; - private static final String VESPA_QRSERVER = "vespa.qrserver"; private static final String VESPA_DISTRIBUTOR = "vespa.distributor"; private static final String VESPA_CONTAINER_CLUSTERCONTROLLER = "vespa.container-clustercontroller"; - private static final List<String> WANTED_METRIC_SERVICES = List.of(VESPA_CONTAINER, VESPA_QRSERVER, VESPA_DISTRIBUTOR, VESPA_CONTAINER_CLUSTERCONTROLLER); + private static final List<String> WANTED_METRIC_SERVICES = List.of(VESPA_CONTAINER, VESPA_DISTRIBUTOR, VESPA_CONTAINER_CLUSTERCONTROLLER); private static final ExecutorService executor = Executors.newFixedThreadPool(10, new DaemonThreadFactory("cluster-deployment-metrics-retriever-")); @@ -132,8 +131,6 @@ public class ClusterDeploymentMetricsRetriever { optionalDouble(values.field("feed.latency.sum")).ifPresent(flSum -> aggregator.get().addFeedLatency(flSum, values.field("feed.latency.count").asDouble())); } - case VESPA_QRSERVER -> optionalDouble(values.field("query_latency.sum")).ifPresent(qlSum -> - aggregator.get().addQrLatency(qlSum, values.field("query_latency.count").asDouble())); case VESPA_DISTRIBUTOR -> { optionalDouble(values.field("vds.distributor.docsstored.average")) .ifPresent(docCount -> aggregator.get().addDocumentCount(docCount)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java index 2e2525fa459..9a68f1d6c37 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsAggregator.java @@ -11,7 +11,6 @@ public class DeploymentMetricsAggregator { private LatencyMetrics feed; private LatencyMetrics read; - private LatencyMetrics qr; private LatencyMetrics container; private Double documentCount; private ResourceUsage memoryUsage; @@ -27,11 +26,6 @@ public class DeploymentMetricsAggregator { return this; } - public synchronized DeploymentMetricsAggregator addQrLatency(double sum, double count) { - this.qr = combineLatency(this.qr, sum, count); - return this; - } - public synchronized DeploymentMetricsAggregator addContainerLatency(double sum, double count) { this.container = combineLatency(this.container, sum, count); return this; @@ -69,17 +63,15 @@ public class DeploymentMetricsAggregator { } public Optional<Double> aggregateQueryLatency() { - if (container == null && qr == null) return Optional.empty(); - var c = Optional.ofNullable(container).orElseGet(LatencyMetrics::new); - var q = Optional.ofNullable(qr).orElseGet(LatencyMetrics::new); - return Optional.of((c.sum + q.sum) / (c.count + q.count)).filter(num -> !num.isNaN()); + if (container == null) return Optional.empty(); + var c = Optional.of(container).orElseGet(LatencyMetrics::new); + return Optional.of(c.sum / c.count).filter(num -> !num.isNaN()); } public Optional<Double> aggregateQueryRate() { - if (container == null && qr == null) return Optional.empty(); - var c = Optional.ofNullable(container).orElseGet(LatencyMetrics::new); - var q = Optional.ofNullable(qr).orElseGet(LatencyMetrics::new); - return Optional.of((c.count + q.count) / 60); + if (container == null) return Optional.empty(); + var c = Optional.of(container).orElseGet(LatencyMetrics::new); + return Optional.of(c.count / 60); } public Optional<Double> aggregateDocumentCount() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 04b42f0411b..3a2286fdd4d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -17,10 +17,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; -import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; /** * Produces lb-services cfg @@ -32,7 +30,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; - public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { + public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone) { this.models = models; this.zone = zone; } @@ -99,10 +97,9 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private boolean getActiveRotation(ApplicationInfo app) { boolean activeRotation = false; for (HostInfo hostInfo : app.getModel().getHosts()) { - Optional<ServiceInfo> container = hostInfo.getServices().stream().filter( - serviceInfo -> serviceInfo.getServiceType().equals(CONTAINER.serviceName) || - serviceInfo.getServiceType().equals(QRSERVER.serviceName)). - findAny(); + Optional<ServiceInfo> container = hostInfo.getServices().stream() + .filter(serviceInfo -> serviceInfo.getServiceType().equals(CONTAINER.serviceName)) + .findAny(); if (container.isPresent()) { activeRotation |= Boolean.parseBoolean(container.get().getProperty("activeRotation").orElse("false")); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java index 236172dc5fd..3db67d78fe6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java @@ -24,9 +24,9 @@ public class SuperModelConfigProvider implements LbServicesConfig.Producer { private final SuperModel superModel; private final LbServicesProducer lbProd; - public SuperModelConfigProvider(SuperModel superModel, Zone zone, FlagSource flagSource) { + public SuperModelConfigProvider(SuperModel superModel, Zone zone) { this.superModel = superModel; - this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(superModel.getModelsPerTenant()), zone, flagSource); + this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(superModel.getModelsPerTenant()), zone); } public SuperModel getSuperModel() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java index e588da8f1f9..b64b3fb05ab 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java @@ -33,7 +33,7 @@ public class ModelFactoryRegistry { } } - public Set<Version> allVersions() { return factories.keySet(); } + public Set<Version> allVersions() { return Set.copyOf(factories.keySet()); } /** * Returns the factory for the given version diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 5105bcbdc28..1bb974a863a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -34,11 +34,11 @@ public class RemoteSession extends Session { * @param tenant The name of the tenant creating session * @param sessionId The session id for this session. * @param zooKeeperClient a SessionZooKeeperClient instance - * @param applicationSet current application set for this session + * @param applicationVersions current application versions for this session */ - RemoteSession(TenantName tenant, long sessionId, SessionZooKeeperClient zooKeeperClient, Optional<ApplicationVersions> applicationSet) { + RemoteSession(TenantName tenant, long sessionId, SessionZooKeeperClient zooKeeperClient, Optional<ApplicationVersions> applicationVersions) { super(tenant, sessionId, zooKeeperClient); - this.applicationVersions = applicationSet; + this.applicationVersions = applicationVersions; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index ac483ea20c9..e9fc68ae76e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -373,7 +373,7 @@ public class SessionRepository { return session; } - public int deleteExpiredRemoteSessions(Clock clock, Predicate<Session> sessionIsActiveForApplication) { + public int deleteExpiredRemoteSessions(Predicate<Session> sessionIsActiveForApplication) { Duration expiryTime = Duration.ofSeconds(expiryTimeFlag.value()); List<Long> remoteSessionsFromZooKeeper = getRemoteSessionsFromZooKeeper(); log.log(Level.FINE, () -> "Remote sessions for tenant " + tenantName + ": " + remoteSessionsFromZooKeeper); @@ -386,7 +386,7 @@ public class SessionRepository { if (session == null) session = new RemoteSession(tenantName, sessionId, createSessionZooKeeperClient(sessionId)); if (session.getStatus() == Session.Status.ACTIVATE && sessionIsActiveForApplication.test(session)) continue; - if (sessionHasExpired(session.getCreateTime(), expiryTime, clock)) { + if (sessionHasExpired(session.getCreateTime(), expiryTime)) { log.log(Level.FINE, () -> "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); deleteRemoteSessionFromZooKeeper(session); deleted++; @@ -411,7 +411,7 @@ public class SessionRepository { transaction.close(); } - private boolean sessionHasExpired(Instant created, Duration expiryTime, Clock clock) { + private boolean sessionHasExpired(Instant created, Duration expiryTime) { return created.plus(expiryTime).isBefore(clock.instant()); } @@ -547,13 +547,13 @@ public class SessionRepository { } } - private ApplicationVersions loadApplication(Session session, Optional<ApplicationVersions> previousApplicationSet) { + private ApplicationVersions loadApplication(Session session, Optional<ApplicationVersions> previousApplicationVersions) { log.log(Level.FINE, () -> "Loading application for " + session); SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(session.getSessionId()); ActivatedModelsBuilder builder = new ActivatedModelsBuilder(session.getTenantName(), session.getSessionId(), sessionZooKeeperClient, - previousApplicationSet, + previousApplicationVersions, sessionPreparer.getExecutor(), curator, metrics, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index bcffecf28ea..1f000bc5856 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.UserConfigDefinitionRepo; import com.yahoo.vespa.config.server.filedistribution.AddFileInterface; import com.yahoo.vespa.config.server.filedistribution.MockFileManager; +import com.yahoo.vespa.config.server.session.Session.Status; import com.yahoo.vespa.config.server.tenant.CloudAccountSerializer; import com.yahoo.vespa.config.server.tenant.DataplaneTokenSerializer; import com.yahoo.vespa.config.server.tenant.OperatorCertificateSerializer; @@ -235,7 +236,11 @@ public class SessionZooKeeperClient { public Version readVespaVersion() { Optional<byte[]> data = curator.getData(versionPath()); // TODO: Empty version should not be possible any more - verify and remove - return data.map(d -> new Version(Utf8.toString(d))).orElse(Vtag.currentVersion); + return data.map(d -> new Version(Utf8.toString(d))) + .orElseGet(() -> { + log.log(Level.WARNING, "No Vespa version found for session at " + versionPath().getAbsolute() + "," + "returning current Vtag version"); + return Vtag.currentVersion; + }); } public Optional<DockerImage> readDockerImageRepository() { @@ -248,8 +253,18 @@ public class SessionZooKeeperClient { } public Instant readCreateTime() { + // TODO jonmv: clean up Optional<byte[]> data = curator.getData(getCreateTimePath()); - return data.map(d -> Instant.ofEpochSecond(Long.parseLong(Utf8.toString(d)))).orElse(Instant.EPOCH); + return data.map(d -> Instant.ofEpochSecond(Long.parseLong(Utf8.toString(d)))) + .or(() -> { + RuntimeException stack = Math.random() < 1e-4 ? new RuntimeException("Trace log") : null; + log.log(Level.FINE, stack, () -> "No creation time found for session at " + getCreateTimePath().getAbsolute() + ", returning session path ctime"); + return curator.getStat(sessionPath).map(s -> Instant.ofEpochMilli(s.getCtime())); + }) + .orElseGet(() -> { + log.log(Level.FINE, () -> "No ZK ctime found for session at " + sessionPath.getAbsolute() + ", returning epoch"); + return Instant.EPOCH; + }); } public Instant readActivatedTime() { @@ -305,7 +320,6 @@ public class SessionZooKeeperClient { var bytes = uncheck(() -> SlimeUtils.toJsonBytes(TenantSecretStoreSerializer.toSlime(tenantSecretStores))); curator.set(tenantSecretStorePath(), bytes); } - } public List<TenantSecretStore> readTenantSecretStores() { @@ -361,7 +375,10 @@ public class SessionZooKeeperClient { public ActivationTriggers readActivationTriggers() { return curator.getData(sessionPath.append(ACTIVATION_TRIGGERS_PATH)) .map(ActivationTriggersSerializer::fromJson) - .orElse(ActivationTriggers.empty()); + .orElseGet(() -> { + log.log(Level.WARNING, "No activation triggers found for session at " + sessionPath.append(ACTIVATION_TRIGGERS_PATH).getAbsolute() + ", returning empty"); + return ActivationTriggers.empty(); + }); } /** @@ -370,12 +387,14 @@ public class SessionZooKeeperClient { * @param createTime Time of session creation. */ public void createNewSession(Instant createTime) { + log.log(Level.FINE, () -> "Creating new session at " + sessionPath.getAbsolute()); CuratorTransaction transaction = new CuratorTransaction(curator); transaction.add(CuratorOperations.create(sessionPath.getAbsolute())); transaction.add(CuratorOperations.create(sessionPath.append(UPLOAD_BARRIER).getAbsolute())); - transaction.add(createWriteStatusTransaction(Session.Status.NEW).operations()); + transaction.add(CuratorOperations.create(sessionStatusPath.getAbsolute(), Utf8.toBytes(Status.NEW.name()))); transaction.add(CuratorOperations.create(getCreateTimePath().getAbsolute(), Utf8.toBytes(String.valueOf(createTime.getEpochSecond())))); transaction.commit(); + log.log(Level.FINE, () -> "Done creating new session at " + sessionPath.getAbsolute()); } public static Path getSessionPath(TenantName tenantName, long sessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index f9f477c9693..150375dd1c0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -194,7 +194,7 @@ public class TenantRepository { this.configserverConfig = configserverConfig; this.curator = curator; this.metrics = metrics; - metricUpdater = metrics.getOrCreateMetricUpdater(Map.of()); + this.metricUpdater = metrics.getOrCreateMetricUpdater(Map.of()); this.zkCacheExecutor = zkCacheExecutor; this.zkApplicationWatcherExecutor = zkApplicationWatcherExecutor; this.zkSessionWatcherExecutor = zkSessionWatcherExecutor; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 9a24dc293ce..b5c1d9779c9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -52,7 +52,6 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.model.VespaModelFactory; @@ -69,13 +68,16 @@ import java.nio.file.Files; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.IntStream; +import static com.yahoo.vespa.config.server.session.Session.Status.ACTIVATE; +import static com.yahoo.vespa.config.server.session.Session.Status.DEACTIVATE; +import static com.yahoo.vespa.config.server.session.Session.Status.PREPARE; +import static com.yahoo.vespa.config.server.session.Session.Status.UNKNOWN; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -97,7 +99,6 @@ public class ApplicationRepositoryTest { private final static File app2 = new File("src/test/apps/cs2"); private final static TenantName tenant1 = TenantName.from("test1"); - private final static TenantName tenant2 = TenantName.from("test2"); private final static ManualClock clock = new ManualClock(Instant.now()); private ApplicationRepository applicationRepository; @@ -138,7 +139,6 @@ public class ApplicationRepositoryTest { .build(); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); - tenantRepository.addTenant(tenant2); orchestrator = new OrchestratorMock(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) @@ -235,7 +235,7 @@ public class ApplicationRepositoryTest { assertApplicationData(firstSessionId, firstSessionId); // Prepare, last deployed session id should be the new one - prepare(testApp, secondSessionId); + prepare(secondSessionId); assertApplicationData(firstSessionId, secondSessionId); // Activate, active session id should be the new one @@ -365,7 +365,7 @@ public class ApplicationRepositoryTest { deployApp(testApp); // Deploy another app (with id fooId) - ApplicationId fooId = applicationId(tenant2); + ApplicationId fooId = applicationId("fooId"); PrepareParams prepareParams2 = new PrepareParams.Builder().applicationId(fooId).build(); deployApp(testAppJdiscOnly, prepareParams2); assertNotNull(applicationRepository.getActiveSession(fooId)); @@ -395,44 +395,47 @@ public class ApplicationRepositoryTest { .flagSource(flagSource) .build(); tester.deployApp("src/test/apps/app"); // session 2 (numbering starts at 2) + var applicationRepo = tester.tenant().getApplicationRepo(); + var applicationRepository = tester.applicationRepository(); + var sessionRepository = tester.tenant().getSessionRepository(); clock.advance(Duration.ofSeconds(10)); Optional<Deployment> deployment2 = tester.redeployFromLocalActive(); - assertTrue(deployment2.isPresent()); deployment2.get().activate(); // session 3 - long activeSessionId = tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId()); + + long activeSessionId = applicationRepo.requireActiveSessionOf(tester.applicationId()); clock.advance(Duration.ofSeconds(10)); - Optional<com.yahoo.config.provision.Deployment> deployment3 = tester.redeployFromLocalActive(); + var deployment3 = tester.redeployFromLocalActive(); assertTrue(deployment3.isPresent()); deployment3.get().prepare(); // session 4 (not activated) - Session deployment3session = ((com.yahoo.vespa.config.server.deploy.Deployment) deployment3.get()).session(); + var deployment3session = ((com.yahoo.vespa.config.server.deploy.Deployment) deployment3.get()).session(); assertNotEquals(activeSessionId, deployment3session.getSessionId()); // No change to active session id - assertEquals(activeSessionId, tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId())); - SessionRepository sessionRepository = tester.tenant().getSessionRepository(); + assertEquals(activeSessionId, applicationRepo.requireActiveSessionOf(tester.applicationId())); + assertEquals(3, sessionRepository.getLocalSessions().size()); clock.advance(Duration.ofHours(1)); // longer than session lifetime // All sessions except 3 should be removed after the call to deleteExpiredLocalSessions - tester.applicationRepository().deleteExpiredLocalSessions(); - Collection<LocalSession> sessions = sessionRepository.getLocalSessions(); - assertEquals(1, sessions.size()); - ArrayList<LocalSession> localSessions = new ArrayList<>(sessions); - LocalSession localSession = localSessions.get(0); + applicationRepository.deleteExpiredLocalSessions(); + var localSessions = new ArrayList<>(sessionRepository.getLocalSessions()); + assertEquals(1, localSessions.size()); + var localSession = localSessions.get(0); assertEquals(3, localSession.getSessionId()); // All sessions except 3 should be removed after the call to deleteExpiredRemoteSessions - assertEquals(2, tester.applicationRepository().deleteExpiredRemoteSessions(clock)); - ArrayList<Long> remoteSessions = new ArrayList<>(sessionRepository.getRemoteSessionsFromZooKeeper()); - Session remoteSession = sessionRepository.getRemoteSession(remoteSessions.get(0)); + assertEquals(2, applicationRepository.deleteExpiredRemoteSessions()); + var remoteSessions = new ArrayList<>(sessionRepository.getRemoteSessionsFromZooKeeper()); + assertEquals(1, remoteSessions.size()); + var remoteSession = sessionRepository.getRemoteSession(remoteSessions.get(0)); assertEquals(3, remoteSession.getSessionId()); // Deploy, but do not activate - Optional<com.yahoo.config.provision.Deployment> deployment4 = tester.redeployFromLocalActive(); + var deployment4 = tester.redeployFromLocalActive(); assertTrue(deployment4.isPresent()); deployment4.get().prepare(); // session 5 (not activated) @@ -443,23 +446,25 @@ public class ApplicationRepositoryTest { // Create a local session without any data in zookeeper (corner case seen in production occasionally) // and check that expiring local sessions still works int sessionId = 6; - TenantName tenantName = tester.tenant().getName(); - Instant session6CreateTime = clock.instant(); - TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(serverdb, tenantName); + var tenantName = tester.tenant().getName(); + var session6CreateTime = clock.instant(); + var tenantFileSystemDirs = new TenantFileSystemDirs(serverdb, tenantName); Files.createDirectory(tenantFileSystemDirs.getUserApplicationDir(sessionId).toPath()); - LocalSession localSession2 = new LocalSession(tenantName, - sessionId, - FilesApplicationPackage.fromFile(testApp), - new SessionZooKeeperClient(curator, tenantName, sessionId, configserverConfig)); + var localSession2 = new LocalSession(tenantName, + sessionId, + FilesApplicationPackage.fromFile(testApp), + new SessionZooKeeperClient(curator, tenantName, sessionId, configserverConfig)); sessionRepository.addLocalSession(localSession2); assertEquals(2, sessionRepository.getLocalSessions().size()); // Create a session, set status to UNKNOWN, we don't want to expire those (creation time is then EPOCH, // so will be candidate for expiry) sessionId = 7; - Session session = sessionRepository.createRemoteSession(sessionId); + var session = sessionRepository.createRemoteSession(sessionId); sessionRepository.createSessionZooKeeperClient(sessionId).createNewSession(clock.instant()); - sessionRepository.createSetStatusTransaction(session, Session.Status.UNKNOWN).commit(); + try (var t = sessionRepository.createSetStatusTransaction(session, UNKNOWN)) { + t.commit(); + } assertEquals(2, sessionRepository.getLocalSessions().size()); // Still 2, no new local session // Check that trying to expire local session when there exists a local session without any data in zookeeper @@ -479,10 +484,12 @@ public class ApplicationRepositoryTest { // Create a local session with invalid application package and check that expiring local sessions still works sessionId = 8; - java.nio.file.Path applicationPath = tenantFileSystemDirs.getUserApplicationDir(sessionId).toPath(); + var applicationPath = tenantFileSystemDirs.getUserApplicationDir(sessionId).toPath(); session = sessionRepository.createRemoteSession(sessionId); sessionRepository.createSessionZooKeeperClient(sessionId).createNewSession(clock.instant()); - sessionRepository.createSetStatusTransaction(session, Session.Status.PREPARE).commit(); + try (var t = sessionRepository.createSetStatusTransaction(session, PREPARE)){ + t.commit(); + } Files.createDirectory(applicationPath); Files.writeString(Files.createFile(applicationPath.resolve("services.xml")), "non-legal xml"); assertEquals(0, sessionRepository.getLocalSessions().size()); // Will not show up in local sessions @@ -540,7 +547,10 @@ public class ApplicationRepositoryTest { list.add(new NetworkPorts.Allocation(19100, "container", "container/container.0", "http/1")); list.add(new NetworkPorts.Allocation(19101, "container", "container/container.0", "messaging")); list.add(new NetworkPorts.Allocation(19102, "container", "container/container.0", "rpc/admin")); - list.add(new NetworkPorts.Allocation(19103, "slobrok", "admin/slobrok.0", "http")); + list.add(new NetworkPorts.Allocation(19103, "logserver-container", "admin/logs/0", "http")); + list.add(new NetworkPorts.Allocation(19104, "logserver-container", "admin/logs/0", "http/1")); + list.add(new NetworkPorts.Allocation(19105, "logserver-container", "admin/logs/0", "rpc/admin")); + list.add(new NetworkPorts.Allocation(19106, "slobrok", "admin/slobrok.0", "http")); AllocatedHosts info = session.getAllocatedHosts(); assertNotNull(info); @@ -565,7 +575,7 @@ public class ApplicationRepositoryTest { Session activeSession = applicationRepository.getActiveSession(applicationId()).get(); assertEquals(firstSession, activeSession.getSessionId()); - assertEquals(Session.Status.ACTIVATE, activeSession.getStatus()); + assertEquals(ACTIVATE, activeSession.getStatus()); } @Test @@ -581,7 +591,7 @@ public class ApplicationRepositoryTest { Session activeSession = applicationRepository.getActiveSession(applicationId()).get(); assertEquals(firstSession, activeSession.getSessionId()); - assertEquals(Session.Status.ACTIVATE, activeSession.getStatus()); + assertEquals(ACTIVATE, activeSession.getStatus()); } @Test @@ -621,7 +631,7 @@ public class ApplicationRepositoryTest { deployApp(testAppJdiscOnly); - assertEquals(Session.Status.DEACTIVATE, firstSession.getStatus()); + assertEquals(DEACTIVATE, firstSession.getStatus()); } @Test @@ -705,9 +715,8 @@ public class ApplicationRepositoryTest { return applicationRepository.deploy(application, prepareParams()); } - private long prepare(File application, long sessionId) { + private void prepare(long sessionId) { applicationRepository.prepare(sessionId, prepareParams()); - return sessionId; } private PrepareResult deployApp(File applicationPackage) { @@ -722,10 +731,10 @@ public class ApplicationRepositoryTest { return new PrepareParams.Builder().applicationId(applicationId()).build(); } - private ApplicationId applicationId() { return applicationId(tenant1); } + private ApplicationId applicationId() { return applicationId("testapp"); } - private ApplicationId applicationId(TenantName tenantName) { - return ApplicationId.from(tenantName, ApplicationName.from("testapp"), InstanceName.defaultName()); + private ApplicationId applicationId(String appName) { + return ApplicationId.from(tenant1, ApplicationName.from(appName), InstanceName.defaultName()); } private Tenant tenant() { return applicationRepository.getTenant(applicationId()); } @@ -756,14 +765,10 @@ public class ApplicationRepositoryTest { return new Context(properties); } - private static class Context implements Metric.Context { - - private final Map<String, ?> point; - - public Context(Map<String, ?> point) { + private record Context(Map<String, ?> point) implements Metric.Context { + private Context(Map<String, ?> point) { this.point = Map.copyOf(point); } - } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index 39f1cb70112..8add7dac489 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Before; import org.junit.Test; @@ -55,7 +54,8 @@ public class SuperModelControllerTest { ApplicationName.from("foo"), InstanceName.defaultName()); models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp)))); SuperModel superModel = new SuperModel(models, true); - handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone()), + new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); } @Test @@ -98,7 +98,8 @@ public class SuperModelControllerTest { models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L)); SuperModel superModel = new SuperModel(models, true); - SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone()), + new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); LbServicesConfig lbc = new LbServicesConfig(lb); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 74b08b83791..889388dd56d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Before; import org.junit.Rule; @@ -46,7 +45,7 @@ public class SuperModelRequestHandlerTest { public void setup() { counter = new SuperModelGenerationCounter(new MockCurator()); ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); - manager = new SuperModelManager(configserverConfig, Zone.defaultZone(), counter, new InMemoryFlagSource()); + manager = new SuperModelManager(configserverConfig, Zone.defaultZone(), counter); controller = new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, manager); } @@ -96,7 +95,7 @@ public class SuperModelRequestHandlerTest { ApplicationId foo = applicationId("a", "foo"); long masterGen = 10; ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen)); - manager = new SuperModelManager(configserverConfig, Zone.defaultZone(), counter, new InMemoryFlagSource()); + manager = new SuperModelManager(configserverConfig, Zone.defaultZone(), counter); controller = new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, manager); long gen = counter.get(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java index 23a425b99e2..81e32c6a7ba 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java @@ -53,7 +53,7 @@ public class ApplicationVersionsTest { public void testGetAllVersions() { applicationVersions = ApplicationVersions.fromList(applications); assertEquals(List.of(Version.fromString("1.2.3"), Version.fromString("1.2.4"), Version.fromString("1.2.5")), - applicationVersions.versions(ApplicationId.defaultId())); + applicationVersions.versions()); } private Application createApplication(Version version) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 68add64ddd9..ec000951684 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -139,6 +139,13 @@ public class DeployTester { * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ public PrepareResult deployApp(String applicationPath, String vespaVersion) { + return deployApp(applicationPath, new PrepareParams.Builder().vespaVersion(vespaVersion)); + } + + /** + * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. + */ + public PrepareResult deployApp(String applicationPath, PrepareParams.Builder paramsBuilder) { String endpoints = """ [ { @@ -151,15 +158,9 @@ public class DeployTester { } ] """; - return deployApp(applicationPath, new PrepareParams.Builder().containerEndpoints(endpoints).vespaVersion(vespaVersion)); - } - - /** - * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. - */ - public PrepareResult deployApp(String applicationPath, PrepareParams.Builder paramsBuilder) { - paramsBuilder.applicationId(applicationId) - .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); + paramsBuilder.applicationId(applicationId) + .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))) + .containerEndpoints(endpoints); return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 512f4dff6b7..a82052b6356 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -22,7 +22,6 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.container.ComponentsConfig; import com.yahoo.slime.SlimeUtils; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.MockConfigConvergenceChecker; @@ -35,7 +34,6 @@ import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.maintenance.PendingRestartsMaintainer; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.session.PrepareParams; -import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.VespaReindexAction; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import org.junit.Rule; @@ -77,6 +75,7 @@ public class HostedDeployTest { @Test public void testRedeployWithVersion() { DeployTester tester = new DeployTester.Builder(temporaryFolder) + .hostedConfigserverConfig(Zone.defaultZone()) .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) .build(); tester.deployApp("src/test/apps/hosted/", "4.5.6"); @@ -90,6 +89,7 @@ public class HostedDeployTest { @Test public void testRedeploy() { DeployTester tester = new DeployTester.Builder(temporaryFolder) + .hostedConfigserverConfig(Zone.defaultZone()) .modelFactory(createHostedModelFactory()) .build(); ApplicationId appId = tester.applicationId(); @@ -105,6 +105,7 @@ public class HostedDeployTest { @Test public void testReDeployWithWantedDockerImageRepositoryAndAthenzDomain() { DeployTester tester = new DeployTester.Builder(temporaryFolder) + .hostedConfigserverConfig(Zone.defaultZone()) .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) .build(); String dockerImageRepository = "docker.foo.com:4443/bar/baz"; @@ -125,6 +126,7 @@ public class HostedDeployTest { public void testRedeployWithTenantSecretStores() { List<TenantSecretStore> tenantSecretStores = List.of(new TenantSecretStore("foo", "123", "role")); DeployTester tester = new DeployTester.Builder(temporaryFolder) + .hostedConfigserverConfig(Zone.defaultZone()) .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) .build(); tester.deployApp("src/test/apps/hosted/", new PrepareParams.Builder() @@ -141,6 +143,7 @@ public class HostedDeployTest { public void testDeployOnUnknownVersion() { List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("1.0.0"))); DeployTester tester = new DeployTester.Builder(temporaryFolder) + .hostedConfigserverConfig(Zone.defaultZone()) .modelFactories(modelFactories) .build(); @@ -561,6 +564,7 @@ public class HostedDeployTest { public void testRedeployWithCloudAccount() { CloudAccount cloudAccount = CloudAccount.from("012345678912"); DeployTester tester = new DeployTester.Builder(temporaryFolder) + .hostedConfigserverConfig(Zone.defaultZone()) .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) .build(); tester.deployApp("src/test/apps/hosted/", new PrepareParams.Builder() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandlerTest.java index d4aa0676c4f..25e4e004a61 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandlerTest.java @@ -86,7 +86,7 @@ class ApplicationApiHandlerTest { public Path dbDir, defsDir, refsDir; @BeforeEach - public void setupRepo() throws IOException { + public void setupRepo() { configserverConfig = new ConfigserverConfig.Builder() .hostedVespa(true) .configServerDBDir(dbDir.toString()) @@ -113,7 +113,7 @@ class ApplicationApiHandlerTest { Zone.defaultZone()); } - private HttpResponse put(long sessionId, Map<String, String> parameters) throws IOException { + private HttpResponse put(long sessionId, Map<String, String> parameters) { var request = com.yahoo.container.jdisc.HttpRequest.createTestRequest("http://host:123/application/v2/tenant/" + tenant + "/prepareandactivate/" + sessionId, Method.PUT, InputStream.nullInputStream(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index a2739196b78..0d674b7c1c2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -66,7 +66,7 @@ public class HostHandlerTest { public void require_correct_tenant_and_application_for_hostname() throws Exception { ApplicationId applicationId = applicationId(); applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); - String hostname = applicationRepository.getActiveApplicationSet(applicationId).get().allHosts().iterator().next(); + String hostname = applicationRepository.getActiveApplicationVersions(applicationId).get().allHosts().iterator().next(); assertApplicationForHost(hostname, applicationId); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java index 96456a898a8..b761e5b8165 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetrieverTest.java @@ -79,7 +79,6 @@ public class ClusterDeploymentMetricsRetrieverTest { new DeploymentMetricsAggregator() .addContainerLatency(3000, 43) .addContainerLatency(2000, 0) - .addQrLatency(3000, 43) .addFeedLatency(3000, 43), aggregatorMap.get(expectedContainerCluster) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index e1ad6bf51f0..57f1e22262e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -19,7 +19,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; import org.xml.sax.SAXException; @@ -56,8 +55,6 @@ public class LbServicesProducerTest { new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.application, List.of("app-endpoint")) ); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - @Test public void testDeterministicGetConfig() { Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder().endpoints(endpoints)); @@ -86,7 +83,7 @@ public class LbServicesProducerTest { } private LbServicesConfig getLbServicesConfig(Zone zone, Map<TenantName, Set<ApplicationInfo>> testModel) { - LbServicesProducer producer = new LbServicesProducer(testModel, zone, flagSource); + LbServicesProducer producer = new LbServicesProducer(testModel, zone); LbServicesConfig.Builder builder = new LbServicesConfig.Builder(); producer.getConfig(builder); return new LbServicesConfig(builder); @@ -134,7 +131,9 @@ public class LbServicesProducerTest { .applications("baz:prod:default:custom-t")); } - private void assertContainsEndpoint(List<Endpoints> endpoints, String dnsName, String clusterId, Endpoints.Scope.Enum scope, Endpoints.RoutingMethod.Enum routingMethod, int weight, List<String> hosts) { + private void assertContainsEndpoint(List<Endpoints> endpoints, String dnsName, String clusterId, + Endpoints.Scope.Enum scope, Endpoints.RoutingMethod.Enum routingMethod, + int weight, List<String> hosts) { assertTrue(endpoints.contains(new Endpoints.Builder() .dnsName(dnsName) .clusterId(clusterId) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index 48380d32b94..44d7687d37a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -24,8 +24,6 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import org.junit.After; import org.junit.rules.TemporaryFolder; import java.io.IOException; @@ -122,15 +120,13 @@ public class RpcTester implements AutoCloseable { } RpcServer createRpcServer(ConfigserverConfig config) throws IOException { - InMemoryFlagSource flagSource = new InMemoryFlagSource(); RpcServer rpcServer = new RpcServer(config, new SuperModelRequestHandler(new TestConfigDefinitionRepo(), config, new SuperModelManager( config, Zone.defaultZone(), - new MemoryGenerationCounter(), - flagSource)), + new MemoryGenerationCounter())), Metrics.createTestMetrics(), hostRegistry, new FileServer(config, new FileDirectory(temporaryFolder.newFolder())), diff --git a/configserver/src/test/resources/metrics/container_metrics.json b/configserver/src/test/resources/metrics/container_metrics.json index f8ea5591c24..4d6fbf55d29 100644 --- a/configserver/src/test/resources/metrics/container_metrics.json +++ b/configserver/src/test/resources/metrics/container_metrics.json @@ -30,22 +30,6 @@ } } ] - }, - { - "name": "vespa.qrserver", - "timestamp": 1557306075, - "metrics": [ - { - "values": { - "query_latency.count": 43.0, - "query_latency.sum": 3000 - }, - "dimensions": { - "clustertype": "container", - "clusterid": "container_cluster_id" - } - } - ] } ] }
\ No newline at end of file diff --git a/configutil/CMakeLists.txt b/configutil/CMakeLists.txt index 5684e2275d4..06ed329e257 100644 --- a/configutil/CMakeLists.txt +++ b/configutil/CMakeLists.txt @@ -2,8 +2,8 @@ vespa_define_module( DEPENDS vespadefaults - config_cloudconfig - vbench + vespa_config + vespa_vbench vespalib LIBS diff --git a/configutil/src/lib/CMakeLists.txt b/configutil/src/lib/CMakeLists.txt index 204e23c89bb..d5581e7baa8 100644 --- a/configutil/src/lib/CMakeLists.txt +++ b/configutil/src/lib/CMakeLists.txt @@ -5,5 +5,5 @@ vespa_add_library(configutil_util STATIC modelinspect.cpp configstatus.cpp DEPENDS - configdefinitions + vespa_configdefinitions ) diff --git a/configutil/src/tests/host_filter/host_filter_test.cpp b/configutil/src/tests/host_filter/host_filter_test.cpp index eb44a5453d1..ae38119aeba 100644 --- a/configutil/src/tests/host_filter/host_filter_test.cpp +++ b/configutil/src/tests/host_filter/host_filter_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <lib/hostfilter.h> TEST("empty hostfilter includes any and all hosts") { diff --git a/configutil/src/tests/tags/tags_test.cpp b/configutil/src/tests/tags/tags_test.cpp index 6d9cc125298..b311f8dbd8f 100644 --- a/configutil/src/tests/tags/tags_test.cpp +++ b/configutil/src/tests/tags/tags_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <lib/tags.h> using namespace configdefinitions; diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index e1ffda5649c..ef00a1e3087 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -1047,13 +1047,51 @@ "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder remoteAddressHeaders(java.util.Collection)", "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder remotePortHeaders(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder remotePortHeaders(java.util.Collection)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder content(com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content$Builder)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder content(java.util.function.Consumer)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Builder content(java.util.List)", "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog build()" ], "fields" : [ "public java.util.List remoteAddressHeaders", - "public java.util.List remotePortHeaders" + "public java.util.List remotePortHeaders", + "public java.util.List content" ] }, + "com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes" : [ + "public", + "final" + ], + "methods" : [ + "public void <init>()", + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content$Builder pathPrefix(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content$Builder maxSize(long)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content$Builder sampleRate(double)", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content build()" + ], + "fields" : [ ] + }, + "com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content" : { + "superClass" : "com.yahoo.config.InnerNode", + "interfaces" : [ ], + "attributes" : [ + "public", + "final" + ], + "methods" : [ + "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content$Builder)", + "public java.lang.String pathPrefix()", + "public long maxSize()", + "public double sampleRate()" + ], + "fields" : [ ] + }, "com.yahoo.jdisc.http.ConnectorConfig$AccessLog" : { "superClass" : "com.yahoo.config.InnerNode", "interfaces" : [ ], @@ -1066,7 +1104,9 @@ "public java.util.List remoteAddressHeaders()", "public java.lang.String remoteAddressHeaders(int)", "public java.util.List remotePortHeaders()", - "public java.lang.String remotePortHeaders(int)" + "public java.lang.String remotePortHeaders(int)", + "public java.util.List content()", + "public com.yahoo.jdisc.http.ConnectorConfig$AccessLog$Content content(int)" ], "fields" : [ ] }, diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java index dcc024ef2b1..31dc5e7cb77 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java @@ -156,7 +156,7 @@ public class ComponentNode extends Node { Duration duration = Duration.between(start, Instant.now()); log.log(duration.compareTo(Duration.ofMinutes(1)) > 0 ? INFO : FINE, () -> "Finished constructing " + idAndType() + " in " + duration); - } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + } catch (InvocationTargetException | InstantiationException | IllegalAccessException | IllegalArgumentException | ClassCastException e) { StackTraceElement dependencyInjectorMarker = new StackTraceElement("============= Dependency Injection =============", "newInstance", null, -1); throw removeStackTrace(new ComponentConstructorException("Error constructing " + idAndType() + ": " + e.getMessage(), cutStackTraceAtConstructor(e.getCause(), dependencyInjectorMarker))); } diff --git a/container-core/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-core/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index e4af680726a..0eb4485936d 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-core/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -31,11 +32,23 @@ import static java.util.stream.Collectors.toMap; */ public class AccessLogEntry { + public record Content(String type, long length, byte[] body) {} + private final Object monitor = new Object(); private HitCounts hitCounts; private TraceNode traceNode; private ListMap<String,String> keyValues=null; + private Content content; + + public void setContent(Content entity) { + synchronized (monitor) { + requireNull(this.content); + this.content = entity; + } + } + + public Optional<Content> getContent() { synchronized (monitor) { return Optional.ofNullable(content); } } public void setHitCounts(final HitCounts hitCounts) { synchronized (monitor) { diff --git a/container-core/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/container-core/src/main/java/com/yahoo/container/logging/JSONFormatter.java index f1e865cd596..1f42f8ded97 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/container-core/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -1,15 +1,16 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; -import com.yahoo.json.Jackson; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.json.Jackson; import com.yahoo.yolean.trace.TraceNode; import java.io.IOException; import java.io.OutputStream; import java.security.Principal; +import java.util.Base64; import java.util.Collection; import java.util.Objects; import java.util.logging.Level; @@ -102,6 +103,15 @@ public class JSONFormatter implements LogWriter<RequestLogEntry> { trace.accept(new TraceRenderer(generator, timestamp)); } + var content = entry.content().orElse(null); + if (content != null) { + generator.writeObjectFieldStart("content"); + generator.writeStringField("type", content.type()); + generator.writeNumberField("length", content.length()); + generator.writeStringField("body", Base64.getEncoder().encodeToString(content.body())); + generator.writeEndObject(); + } + // Only add search sub block of this is a search request if (isSearchRequest(entry)) { HitCounts hitCounts = entry.hitCounts().get(); diff --git a/container-core/src/main/java/com/yahoo/container/logging/RequestLogEntry.java b/container-core/src/main/java/com/yahoo/container/logging/RequestLogEntry.java index 0092d67b0fa..a5f82a8b637 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/RequestLogEntry.java +++ b/container-core/src/main/java/com/yahoo/container/logging/RequestLogEntry.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; +import com.yahoo.container.logging.AccessLogEntry.Content; import com.yahoo.yolean.trace.TraceNode; import java.security.Principal; @@ -50,6 +51,7 @@ public class RequestLogEntry { private final Principal sslPrincipal; private final HitCounts hitCounts; private final TraceNode traceNode; + private final Content content; private final SortedMap<String, Collection<String>> extraAttributes; private RequestLogEntry(Builder builder) { @@ -76,6 +78,7 @@ public class RequestLogEntry { this.sslPrincipal = builder.sslPrincipal; this.hitCounts = builder.hitCounts; this.traceNode = builder.traceNode; + this.content = builder.content; this.extraAttributes = copyExtraAttributes(builder.extraAttributes); } @@ -102,6 +105,7 @@ public class RequestLogEntry { public Optional<Principal> sslPrincipal() { return Optional.ofNullable(sslPrincipal); } public Optional<HitCounts> hitCounts() { return Optional.ofNullable(hitCounts); } public Optional<TraceNode> traceNode() { return Optional.ofNullable(traceNode); } + public Optional<Content> content() { return Optional.ofNullable(content); } public SortedSet<String> extraAttributeKeys() { return Collections.unmodifiableSortedSet((SortedSet<String>)extraAttributes.keySet()); } public Collection<String> extraAttributeValues(String key) { return Collections.unmodifiableCollection(extraAttributes.get(key)); } @@ -145,6 +149,7 @@ public class RequestLogEntry { private Principal userPrincipal; private HitCounts hitCounts; private TraceNode traceNode; + private Content content; private Principal sslPrincipal; private final Map<String, Collection<String>> extraAttributes = new HashMap<>(); @@ -171,6 +176,7 @@ public class RequestLogEntry { public Builder sslPrincipal(Principal sslPrincipal) { this.sslPrincipal = requireNonNull(sslPrincipal); return this; } public Builder hitCounts(HitCounts hitCounts) { this.hitCounts = requireNonNull(hitCounts); return this; } public Builder traceNode(TraceNode traceNode) { this.traceNode = requireNonNull(traceNode); return this; } + public Builder content(Content content) { this.content = requireNonNull(content); return this; } public Builder addExtraAttribute(String key, String value) { this.extraAttributes.computeIfAbsent(requireNonNull(key), __ -> new ArrayList<>()).add(requireNonNull(value)); return this; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index af1179fadba..6c071a162d1 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -117,6 +117,7 @@ class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty } addNonNullValue(builder, accessLogEntry.getHitCounts(), RequestLogEntry.Builder::hitCounts); addNonNullValue(builder, accessLogEntry.getTrace(), RequestLogEntry.Builder::traceNode); + accessLogEntry.getContent().ifPresent(builder::content); } http2StreamId(request).ifPresent(streamId -> builder.addExtraAttribute("http2-stream-id", Integer.toString(streamId))); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java index 2c9cffa6786..31fd55bb987 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java @@ -1,18 +1,36 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import com.google.common.base.Preconditions; +import ai.vespa.utils.BytesQuantity; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.DelegatedRequestHandler; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.jdisc.http.HttpRequest; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Random; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnector; /** * A wrapper RequestHandler that enables access logging. By wrapping the request handler, we are able to wrap the @@ -23,6 +41,7 @@ import java.util.Optional; * Does not otherwise interfere with the request processing of the delegate request handler. * * @author bakksjo + * @author bjorncs */ public class AccessLoggingRequestHandler extends AbstractRequestHandler implements DelegatedRequestHandler { public static final String CONTEXT_KEY_ACCESS_LOG_ENTRY @@ -38,27 +57,87 @@ public class AccessLoggingRequestHandler extends AbstractRequestHandler implemen (AccessLogEntry) requestContextMap.get(CONTEXT_KEY_ACCESS_LOG_ENTRY)); } - private final RequestHandler delegate; + private final org.eclipse.jetty.server.Request jettyRequest; + private final RequestHandler delegateRequestHandler; private final AccessLogEntry accessLogEntry; + private final List<String> pathPrefixes; + private final List<Double> samplingRate; + private final List<Long> maxSize; + private final Random rng = new Random(); public AccessLoggingRequestHandler( - final RequestHandler delegateRequestHandler, - final AccessLogEntry accessLogEntry) { - this.delegate = delegateRequestHandler; + org.eclipse.jetty.server.Request jettyRequest, + RequestHandler delegateRequestHandler, + AccessLogEntry accessLogEntry) { + this.jettyRequest = jettyRequest; + this.delegateRequestHandler = delegateRequestHandler; this.accessLogEntry = accessLogEntry; + var cfg = getConnector(jettyRequest).connectorConfig().accessLog().content(); + this.pathPrefixes = cfg.stream().map(e -> e.pathPrefix()).toList(); + this.samplingRate = cfg.stream().map(e -> e.sampleRate()).toList(); + this.maxSize = cfg.stream().map(e -> e.maxSize()).toList(); } @Override public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { - Preconditions.checkArgument(request instanceof HttpRequest, "Expected HttpRequest, got " + request); final HttpRequest httpRequest = (HttpRequest) request; httpRequest.context().put(CONTEXT_KEY_ACCESS_LOG_ENTRY, accessLogEntry); - return delegate.handleRequest(request, handler); + var methodsWithEntity = List.of(HttpRequest.Method.POST, HttpRequest.Method.PUT, HttpRequest.Method.PATCH); + var originalContentChannel = delegateRequestHandler.handleRequest(request, handler); + var uriPath = request.getUri().getPath(); + if (methodsWithEntity.contains(httpRequest.getMethod())) { + for (int i = 0; i < pathPrefixes.size(); i++) { + if (uriPath.startsWith(pathPrefixes.get(i))) { + if (samplingRate.get(i) > rng.nextDouble()) { + return new ContentLoggingContentChannel(originalContentChannel, maxSize.get(i)); + } + } + } + } + return originalContentChannel; } - @Override public RequestHandler getDelegate() { - return delegate; + return delegateRequestHandler; + } + + private class ContentLoggingContentChannel implements ContentChannel { + final AtomicLong length = new AtomicLong(); + final ByteArrayOutputStream accumulatedRequestContent; + final ContentChannel originalContentChannel; + final long contentLoggingMaxSize; + + public ContentLoggingContentChannel(ContentChannel originalContentChannel, long contentLoggingMaxSize) { + this.originalContentChannel = originalContentChannel; + this.contentLoggingMaxSize = contentLoggingMaxSize; + var contentLength = jettyRequest.getContentLength(); + this.accumulatedRequestContent = new ByteArrayOutputStream(contentLength == -1 ? 128 : contentLength); + } + + @Override + public void write(ByteBuffer buf, CompletionHandler handler) { + length.addAndGet(buf.remaining()); + var bytesToLog = Math.min(buf.remaining(), contentLoggingMaxSize - accumulatedRequestContent.size()); + if (bytesToLog > 0) accumulatedRequestContent.write(buf.array(), buf.arrayOffset() + buf.position(), (int)bytesToLog); + if (originalContentChannel != null) originalContentChannel.write(buf, handler); + } + + @Override + public void close(CompletionHandler handler) { + var bytes = accumulatedRequestContent.toByteArray(); + accessLogEntry.setContent(new AccessLogEntry.Content( + Objects.requireNonNullElse(jettyRequest.getHeader(HttpHeaders.Names.CONTENT_TYPE), ""), + length.get(), + bytes)); + accumulatedRequestContent.reset(); + length.set(0); + if (originalContentChannel != null) originalContentChannel.close(handler); + } + + @Override + public void onError(Throwable error) { + if (originalContentChannel != null) originalContentChannel.onError(error); + } } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 0b021ec8bcd..9d873f75516 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -214,7 +214,8 @@ class HttpRequestDispatch { new FilteringRequestHandler(context.filterResolver(), (Request)servletRequest), servletRequest, context.removeRawPostBodyForWwwUrlEncodedPost()); - return new AccessLoggingRequestHandler(requestHandler, accessLogEntry); + return new AccessLoggingRequestHandler( + (Request) servletRequest, requestHandler, accessLogEntry); } private static RequestHandler wrapHandlerIfFormPost(RequestHandler requestHandler, diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index 2906f75a1f5..44750b15324 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -154,3 +154,8 @@ accessLog.remoteAddressHeaders[] string # HTTP request headers that contain remote port accessLog.remotePortHeaders[] string + +# Path prefixes for which content should be logged +accessLog.content[].pathPrefix string +accessLog.content[].maxSize long +accessLog.content[].sampleRate double diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 73dfb85519c..792d3a6e436 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; +import ai.vespa.utils.BytesQuantity; import com.google.inject.AbstractModule; import com.google.inject.Module; import com.yahoo.container.logging.ConnectionLog; @@ -82,6 +83,7 @@ import static com.yahoo.jdisc.http.server.jetty.SimpleHttpClient.ResponseValidat import static com.yahoo.jdisc.http.server.jetty.Utils.createHttp2Client; import static com.yahoo.jdisc.http.server.jetty.Utils.createSslTestDriver; import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.startsWith; @@ -742,12 +744,26 @@ public class HttpServerTest { JettyTestDriver driver = JettyTestDriver.newConfiguredInstance( new EchoRequestHandler(), new ServerConfig.Builder(), - new ConnectorConfig.Builder(), + new ConnectorConfig.Builder().accessLog( + new ConnectorConfig.AccessLog.Builder() + .content(List.of( + new ConnectorConfig.AccessLog.Content.Builder() + .pathPrefix("/search") + .maxSize(BytesQuantity.ofKB(1).toBytes()) + .sampleRate(1), + new ConnectorConfig.AccessLog.Content.Builder() + .pathPrefix("/document/v1") + .maxSize(BytesQuantity.ofMB(1).toBytes()) + .sampleRate(0.001) + ))), binder -> binder.bind(RequestLog.class).toInstance(requestLogMock)); - driver.client().newPost("/status.html").setContent("abcdef").execute().expectStatusCode(is(OK)); + driver.client().newPost("/search/").setContent("abcdef").execute().expectStatusCode(is(OK)); RequestLogEntry entry = requestLogMock.poll(Duration.ofSeconds(5)); assertEquals(200, entry.statusCode().getAsInt()); assertEquals(6, entry.requestSize().getAsLong()); + assertEquals("text/plain; charset=UTF-8", entry.content().get().type()); + assertEquals(6, entry.content().get().length()); + assertEquals("abcdef", new String(entry.content().get().body(), UTF_8)); assertTrue(driver.close()); } @@ -894,7 +910,7 @@ public class HttpServerTest { final HttpRequest httpRequest = (HttpRequest)request; final String connectedAt = String.valueOf(httpRequest.getConnectedAt(TimeUnit.MILLISECONDS)); final ContentChannel ch = handler.handleResponse(new Response(OK)); - ch.write(ByteBuffer.wrap(connectedAt.getBytes(StandardCharsets.UTF_8)), null); + ch.write(ByteBuffer.wrap(connectedAt.getBytes(UTF_8)), null); ch.close(null); return null; } @@ -924,7 +940,7 @@ public class HttpServerTest { List<Cookie> cookies = new ArrayList<>(((HttpRequest)request).decodeCookieHeader()); cookies.sort(new CookieComparator()); final ContentChannel out = ResponseDispatch.newInstance(Response.Status.OK).connect(handler); - out.write(StandardCharsets.UTF_8.encode(cookies.toString()), null); + out.write(UTF_8.encode(cookies.toString()), null); out.close(null); return null; } @@ -938,7 +954,7 @@ public class HttpServerTest { public ContentChannel handleRequest(Request request, ResponseHandler handler) { Map<String, List<String>> parameters = new TreeMap<>(((HttpRequest)request).parameters()); ContentChannel responseContentChannel = ResponseDispatch.newInstance(Response.Status.OK).connect(handler); - responseContentChannel.write(ByteBuffer.wrap(parameters.toString().getBytes(StandardCharsets.UTF_8)), + responseContentChannel.write(ByteBuffer.wrap(parameters.toString().getBytes(UTF_8)), NULL_COMPLETION_HANDLER); // Have the request content written back to the response. @@ -1012,7 +1028,7 @@ public class HttpServerTest { @Override public ContentChannel handleRequest(Request req, ResponseHandler handler) { final ContentChannel ch = handler.handleResponse(new Response(OK)); - ch.write(ByteBuffer.wrap(req.getUri().toString().getBytes(StandardCharsets.UTF_8)), null); + ch.write(ByteBuffer.wrap(req.getUri().toString().getBytes(UTF_8)), null); ch.close(null); return null; } diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index c4807bb134f..a13485fb6bb 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -151,8 +151,8 @@ configure_memory() { available_cgroup=$((available_cgroup_bytes >> 20)) available=$((available > available_cgroup ? available_cgroup : available)) fi - #Subtract 1G as fixed overhead for an application container. - reserved_mem=1024 + # Subtract 700MB as fixed overhead for an application container -- keep in sync with com.yahoo.vespa.model.Host.memoryOverheadGb + reserved_mem=700 available=$((available > reserved_mem ? available - reserved_mem : available)) jvm_heapsize=$((available * jvm_heapSizeAsPercentageOfPhysicalMemory / 100)) diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 1c6c773afd9..c6e3bea90a8 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -712,6 +712,7 @@ "public void <init>(int, java.lang.String)", "public void <init>(long, java.lang.String)", "public void <init>(com.yahoo.prelude.query.Limit, com.yahoo.prelude.query.Limit, java.lang.String)", + "public void <init>(java.lang.Long)", "public void <init>(java.lang.String)", "public void <init>(java.lang.String, boolean)", "public void <init>(java.lang.String, java.lang.String)", @@ -1100,6 +1101,7 @@ ], "methods" : [ "public void <init>(java.lang.String)", + "public void <init>(java.lang.String, java.util.Set)", "public com.yahoo.prelude.query.Item$ItemType getItemType()", "public int encode(java.nio.ByteBuffer)", "protected void encodeThis(java.nio.ByteBuffer)", @@ -1107,8 +1109,12 @@ "protected void appendBodyString(java.lang.StringBuilder)", "public void addToken(long)", "public java.util.Collection getTokens()", + "public void disclose(com.yahoo.prelude.query.textualrepresentation.Discloser)", "public boolean equals(java.lang.Object)", - "public int hashCode()" + "public int hashCode()", + "public com.yahoo.prelude.query.NumericInItem clone()", + "public bridge synthetic com.yahoo.prelude.query.Item clone()", + "public bridge synthetic java.lang.Object clone()" ], "fields" : [ ] }, @@ -1582,6 +1588,7 @@ ], "methods" : [ "public void <init>(java.lang.String)", + "public void <init>(java.lang.String, java.util.Set)", "public com.yahoo.prelude.query.Item$ItemType getItemType()", "public int encode(java.nio.ByteBuffer)", "protected void encodeThis(java.nio.ByteBuffer)", @@ -1590,8 +1597,12 @@ "public void addToken(java.lang.String)", "public void removeToken(java.lang.String)", "public java.util.Collection getTokens()", + "public void disclose(com.yahoo.prelude.query.textualrepresentation.Discloser)", "public boolean equals(java.lang.Object)", - "public int hashCode()" + "public int hashCode()", + "public com.yahoo.prelude.query.StringInItem clone()", + "public bridge synthetic com.yahoo.prelude.query.Item clone()", + "public bridge synthetic java.lang.Object clone()" ], "fields" : [ ] }, @@ -1864,6 +1875,7 @@ "public void setIndexName(java.lang.String)", "public java.lang.String getIndexName()", "public int getN()", + "public boolean nIsExplicit()", "public void setN(int)", "public void disclose(com.yahoo.prelude.query.textualrepresentation.Discloser)", "public int hashCode()", @@ -2108,6 +2120,7 @@ "public com.yahoo.search.Query clone()", "public com.yahoo.search.query.Presentation getPresentation()", "public com.yahoo.search.query.Select getSelect()", + "public void setSelect(java.lang.String)", "public com.yahoo.search.query.Ranking getRanking()", "public com.yahoo.search.query.Model getModel()", "public com.yahoo.search.query.Trace getTrace()", @@ -5499,12 +5512,12 @@ "public com.yahoo.search.query.ranking.RankProperties getProperties()", "public void setListFeatures(boolean)", "public boolean getListFeatures()", - "public void setUseSignificance(boolean)", - "public boolean getUseSignificance()", "public com.yahoo.search.query.ranking.MatchPhase getMatchPhase()", + "public com.yahoo.search.query.ranking.SecondPhase getSecondPhase()", "public com.yahoo.search.query.ranking.GlobalPhase getGlobalPhase()", "public com.yahoo.search.query.ranking.Matching getMatching()", "public com.yahoo.search.query.ranking.SoftTimeout getSoftTimeout()", + "public com.yahoo.search.query.ranking.Significance getSignificance()", "public com.yahoo.search.query.Sorting getSorting()", "public void setSorting(com.yahoo.search.query.Sorting)", "public void setSorting(java.lang.String)", @@ -5530,8 +5543,10 @@ "public static final java.lang.String KEEPRANKCOUNT", "public static final java.lang.String RANKSCOREDROPLIMIT", "public static final java.lang.String MATCH_PHASE", + "public static final java.lang.String SECOND_PHASE", "public static final java.lang.String GLOBAL_PHASE", "public static final java.lang.String DIVERSITY", + "public static final java.lang.String SIGNIFICANCE", "public static final java.lang.String SOFTTIMEOUT", "public static final java.lang.String MATCHING", "public static final java.lang.String FEATURES", @@ -7157,6 +7172,49 @@ ], "fields" : [ ] }, + "com.yahoo.search.query.ranking.SecondPhase" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "java.lang.Cloneable" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public static com.yahoo.search.query.profile.types.QueryProfileType getArgumentType()", + "public void setRankScoreDropLimit(double)", + "public java.lang.Double getRankScoreDropLimit()", + "public void prepare(com.yahoo.search.query.ranking.RankProperties)", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public com.yahoo.search.query.ranking.SecondPhase clone()", + "public bridge synthetic java.lang.Object clone()" + ], + "fields" : [ ] + }, + "com.yahoo.search.query.ranking.Significance" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "java.lang.Cloneable" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public static com.yahoo.search.query.profile.types.QueryProfileType getArgumentType()", + "public void setUseModel(boolean)", + "public java.util.Optional getUseModel()", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public com.yahoo.search.query.ranking.Significance clone()", + "public bridge synthetic java.lang.Object clone()" + ], + "fields" : [ + "public static final java.lang.String USE_MODEL" + ] + }, "com.yahoo.search.query.ranking.SoftTimeout" : { "superClass" : "java.lang.Object", "interfaces" : [ @@ -8717,7 +8775,6 @@ "public void <init>(com.yahoo.search.Searcher, com.yahoo.search.searchchain.Execution$Context)", "public final com.yahoo.processing.Response process(com.yahoo.processing.Request)", "public com.yahoo.search.Result search(com.yahoo.search.Query)", - "protected void onInvoking(com.yahoo.processing.Request, com.yahoo.processing.Processor)", "protected com.yahoo.processing.Response defaultResponse(com.yahoo.processing.Request)", "public void fillAttributes(com.yahoo.search.Result)", "public void fill(com.yahoo.search.Result)", @@ -8917,6 +8974,18 @@ ], "fields" : [ ] }, + "com.yahoo.search.searchers.OpportunisticWeakAndSearcher" : { + "superClass" : "com.yahoo.search.Searcher", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)" + ], + "fields" : [ ] + }, "com.yahoo.search.searchers.QueryValidator" : { "superClass" : "com.yahoo.search.Searcher", "interfaces" : [ ], @@ -9182,7 +9251,7 @@ "methods" : [ "public void <init>(ai.vespa.search.llm.LlmSearcherConfig, com.yahoo.component.provider.ComponentRegistry)", "public com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)", - "protected com.yahoo.search.Result complete(com.yahoo.search.Query, ai.vespa.llm.completion.Prompt)", + "protected com.yahoo.search.Result complete(com.yahoo.search.Query, ai.vespa.llm.completion.Prompt, com.yahoo.search.Result, com.yahoo.search.searchchain.Execution)", "public java.lang.String getPrompt(com.yahoo.search.Query)", "public java.lang.String getPropertyPrefix()", "public java.lang.String lookupProperty(java.lang.String, com.yahoo.search.Query)", @@ -9207,6 +9276,8 @@ "public ai.vespa.search.llm.LlmSearcherConfig$Builder propertyPrefix(java.lang.String)", "public ai.vespa.search.llm.LlmSearcherConfig$Builder stream(boolean)", "public ai.vespa.search.llm.LlmSearcherConfig$Builder providerId(java.lang.String)", + "public ai.vespa.search.llm.LlmSearcherConfig$Builder prompt(java.lang.String)", + "public ai.vespa.search.llm.LlmSearcherConfig$Builder promptTemplate(java.util.Optional)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -9246,7 +9317,9 @@ "public void <init>(ai.vespa.search.llm.LlmSearcherConfig$Builder)", "public java.lang.String propertyPrefix()", "public boolean stream()", - "public java.lang.String providerId()" + "public java.lang.String providerId()", + "public java.lang.String prompt()", + "public java.util.Optional promptTemplate()" ], "fields" : [ "public static final java.lang.String CONFIG_DEF_MD5", diff --git a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java index 4c39506ed96..f50cbea0bfc 100755 --- a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java +++ b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java @@ -14,11 +14,17 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.JsonRenderer; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.EventStream; import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; +import com.yahoo.text.Utf8; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.concurrent.RejectedExecutionException; import java.util.function.Function; @@ -38,8 +44,13 @@ public class LLMSearcher extends Searcher { private static final String API_KEY_HEADER = "X-LLM-API-KEY"; private static final String STREAM_PROPERTY = "stream"; private static final String PROMPT_PROPERTY = "prompt"; + private static final String INCLUDE_PROMPT_IN_RESULT = "includePrompt"; + private static final String INCLUDE_HITS_IN_RESULT = "includeHits"; + + private final JsonRenderer jsonRenderer; private final String propertyPrefix; + private final String prompt; private final boolean stream; private final LanguageModel languageModel; private final String languageModelId; @@ -50,11 +61,28 @@ public class LLMSearcher extends Searcher { this.languageModelId = config.providerId(); this.languageModel = findLanguageModel(languageModelId, languageModels); this.propertyPrefix = config.propertyPrefix(); + this.prompt = loadDefaultPrompt(config); + + this.jsonRenderer = new JsonRenderer(); } @Override public Result search(Query query, Execution execution) { - return complete(query, StringPrompt.from(getPrompt(query))); + return complete(query, StringPrompt.from(getPrompt(query)), null, execution); + } + + private String loadDefaultPrompt(LlmSearcherConfig config) { + if (config.prompt() != null && ! config.prompt().isEmpty()) { + return config.prompt(); + } else if (config.promptTemplate().isPresent()) { + Path path = config.promptTemplate().get(); + try { + return new String(Files.readAllBytes(path)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read prompt template file: " + path, e); + } + } + return null; } private LanguageModel findLanguageModel(String providerId, ComponentRegistry<LanguageModel> languageModels) @@ -81,30 +109,37 @@ public class LLMSearcher extends Searcher { return languageModel; } - protected Result complete(Query query, Prompt prompt) { + protected Result complete(Query query, Prompt prompt, Result result, Execution execution) { var options = new InferenceParameters(getApiKeyHeader(query), s -> lookupProperty(s, query)); var stream = lookupPropertyBool(STREAM_PROPERTY, query, this.stream); // query value overwrites config try { - return stream ? completeAsync(query, prompt, options) : completeSync(query, prompt, options); + if (stream) { + return completeAsync(query, prompt, options, result, execution); + } + return completeSync(query, prompt, options, result, execution); } catch (RejectedExecutionException e) { return new Result(query, new ErrorMessage(429, e.getMessage())); } } private boolean shouldAddPrompt(Query query) { - return query.getTrace().getLevel() >= 1; + var includePrompt = lookupPropertyBool(INCLUDE_PROMPT_IN_RESULT, query, false); + return query.getTrace().getLevel() >= 1 || includePrompt; } private boolean shouldAddTokenStats(Query query) { return query.getTrace().getLevel() >= 1; } - private Result completeAsync(Query query, Prompt prompt, InferenceParameters options) { + private Result completeAsync(Query query, Prompt prompt, InferenceParameters options, Result result, Execution execution) { final EventStream eventStream = new EventStream(); if (shouldAddPrompt(query)) { eventStream.add(prompt.asString(), "prompt"); } + if (shouldAddHits(query) && result != null) { + eventStream.add(renderHits(result, execution), "hits"); + } final TokenStats tokenStats = new TokenStats(); languageModel.completeAsync(prompt, options, completion -> { @@ -143,12 +178,15 @@ public class LLMSearcher extends Searcher { eventStream.error(languageModelId, new ErrorMessage(errorCode, exception.getMessage())); } - private Result completeSync(Query query, Prompt prompt, InferenceParameters options) { + private Result completeSync(Query query, Prompt prompt, InferenceParameters options, Result result, Execution execution) { EventStream eventStream = new EventStream(); if (shouldAddPrompt(query)) { eventStream.add(prompt.asString(), "prompt"); } + if (shouldAddHits(query) && result != null) { + eventStream.add(renderHits(result, execution), "hits"); + } List<Completion> completions = languageModel.complete(prompt, options); eventStream.add(completions.get(0).text(), "completion"); @@ -160,11 +198,15 @@ public class LLMSearcher extends Searcher { } public String getPrompt(Query query) { - // Look for prompt with or without prefix + // Look for prompt with or without prefix given in query String prompt = lookupPropertyWithOrWithoutPrefix(PROMPT_PROPERTY, p -> query.properties().getString(p)); if (prompt != null) return prompt; + // Look for prompt defined in config + if (this.prompt != null && ! this.prompt.isEmpty()) + return this.prompt; + // If not found, use query directly prompt = query.getModel().getQueryString(); if (prompt != null) @@ -200,6 +242,18 @@ public class LLMSearcher extends Searcher { return lookupPropertyWithOrWithoutPrefix(API_KEY_HEADER, p -> query.getHttpRequest().getHeader(p)); } + private boolean shouldAddHits(Query query) { + return lookupPropertyBool(INCLUDE_HITS_IN_RESULT, query, false); + } + + private String renderHits(Result results, Execution execution) { + var bs = new ByteArrayOutputStream(); + var renderer = jsonRenderer.clone(); + renderer.init(); + renderer.renderResponse(bs, results, execution, null).join(); // wait for renderer to complete + return Utf8.toString(bs.toByteArray()); + } + private static class TokenStats { private final long start; diff --git a/container-search/src/main/java/ai/vespa/search/llm/RAGSearcher.java b/container-search/src/main/java/ai/vespa/search/llm/RAGSearcher.java index cba153d881d..ff271162458 100755 --- a/container-search/src/main/java/ai/vespa/search/llm/RAGSearcher.java +++ b/container-search/src/main/java/ai/vespa/search/llm/RAGSearcher.java @@ -11,7 +11,11 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.searchchain.Execution; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * An LLM searcher that uses the RAG (Retrieval-Augmented Generation) model to generate completions. @@ -26,6 +30,7 @@ public class RAGSearcher extends LLMSearcher { private static Logger log = Logger.getLogger(RAGSearcher.class.getName()); private static final String CONTEXT_PROPERTY = "context"; + private static final String FIELDS_TO_INCLUDE_PROPERTY = "fields"; @Inject public RAGSearcher(LlmSearcherConfig config, ComponentRegistry<LanguageModel> languageModels) { @@ -37,7 +42,7 @@ public class RAGSearcher extends LLMSearcher { public Result search(Query query, Execution execution) { Result result = execution.search(query); execution.fill(result); - return complete(query, buildPrompt(query, result)); + return complete(query, buildPrompt(query, result), result, execution); } protected Prompt buildPrompt(Query query, Result result) { @@ -59,16 +64,29 @@ public class RAGSearcher extends LLMSearcher { } private String buildContext(Result result) { + Set<String> fieldsToInclude = getFieldsToInclude(result.getQuery()); + StringBuilder sb = new StringBuilder(); var hits = result.hits(); - hits.forEach(hit -> { + int counter = 1; + for (var hit: hits) { + sb.append("document [").append(counter++).append("]:\n"); hit.fields().forEach((key, value) -> { - sb.append(key).append(": ").append(value).append("\n"); + if (fieldsToInclude.isEmpty() || fieldsToInclude.contains(key)) { + sb.append(key).append(": ").append(value).append("\n"); + } }); sb.append("\n"); - }); - var context = sb.toString(); - return context; + } + return sb.toString(); + } + + private Set<String> getFieldsToInclude(Query query) { + String includedFields = lookupProperty(FIELDS_TO_INCLUDE_PROPERTY, query); + if (includedFields != null) { + return Arrays.stream(includedFields.split(",")).map(String::trim).collect(Collectors.toSet()); + } + return new HashSet<>(); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java index 9836934acc1..a8ae21fbea4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IndexedBackend.java @@ -103,7 +103,7 @@ public class IndexedBackend extends VespaBackend { if (result.isFilled(summaryClass)) return; Query query = result.getQuery(); - traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass)); + traceQuery(getName(), DispatchPhase.FILL, query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass)); try (FillInvoker invoker = getFillInvoker(result)) { invoker.fill(result, summaryClass); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java index a463fb9d0e6..c6c9d3d18d5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java @@ -52,6 +52,7 @@ public abstract class VespaBackend { /** The name of this source */ private final String name; + protected enum DispatchPhase{SEARCH, FILL} protected VespaBackend(ClusterParams clusterParams) { this.serverId = clusterParams.getServerId(); @@ -166,7 +167,7 @@ public abstract class VespaBackend { resolveDocumentDatabase(query); transformQuery(query); - traceQuery(name, "search", query, query.getOffset(), query.getHits(), 1, Optional.empty()); + traceQuery(name, DispatchPhase.SEARCH, query, query.getOffset(), query.getHits(), 1, Optional.empty()); root = query.getModel().getQueryTree().getRoot(); if (root == null || root instanceof NullItem) // root can become null after resolving and transformation? @@ -238,11 +239,11 @@ public abstract class VespaBackend { destination.hits().addErrorsFrom(source.hits()); } - void traceQuery(String sourceName, String type, Query query, int offset, int hits, int level, Optional<String> quotedSummaryClass) { + void traceQuery(String sourceName, DispatchPhase phase, Query query, int offset, int hits, int level, Optional<String> quotedSummaryClass) { if ((query.getTrace().getLevel()<level) || !query.getTrace().getQuery()) return; StringBuilder s = new StringBuilder(); - s.append(sourceName).append(" ").append(type).append(" to dispatch: ") + s.append(sourceName).append(" ").append(phase.name().toLowerCase()).append(" to dispatch: ") .append("query=[") .append(query.getModel().getQueryTree().getRoot().toString()) .append("]"); @@ -285,10 +286,12 @@ public abstract class VespaBackend { s.append(" sessionId=").append(query.getSessionId(getServerId())); } - List<Grouping> grouping = GroupingExecutor.getGroupingList(query); - s.append(" grouping=").append(grouping.size()).append(" : "); - for(Grouping g : grouping) { - s.append(g.toString()); + if (query.getTrace().getLevel() >= (level+3)) { + List<Grouping> grouping = GroupingExecutor.getGroupingList(query); + s.append(" grouping=").append(grouping.size()).append(" : "); + for (Grouping g : grouping) { + s.append(g.toString()); + } } if ( ! query.getRanking().getProperties().isEmpty()) { @@ -311,10 +314,10 @@ public abstract class VespaBackend { if (query.getTrace().isTraceable(level + 1) && query.getTrace().getQuery()) { query.trace("Current state of query tree: " + new TextualQueryRepresentation(query.getModel().getQueryTree().getRoot()), - false, level+1); + false, level + 1); } if (query.getTrace().isTraceable(level + 2) && query.getTrace().getQuery()) { - query.trace("YQL+ representation: " + query.yqlRepresentation(), level+2); + query.trace("YQL+ representation: " + query.yqlRepresentation(), level + 2); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java index de1d444434a..8325101b4ad 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java @@ -210,7 +210,7 @@ public abstract class CompositeItem extends Item { public CompositeItem clone() { CompositeItem copy = (CompositeItem) super.clone(); - copy.subitems = new java.util.ArrayList<>(subitems.size()); + copy.subitems = new ArrayList<>(subitems.size()); for (Item subItem : subitems) { Item subItemCopy = subItem.clone(); subItemCopy.setParent(copy); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/InItem.java b/container-search/src/main/java/com/yahoo/prelude/query/InItem.java index 27213000e3a..25abf465199 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/InItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/InItem.java @@ -5,13 +5,15 @@ import java.util.Objects; import static java.util.Objects.requireNonNullElse; -/* +/** * Abstract class representing an IN operator. * * @author toregge */ public abstract class InItem extends Item { + private String indexName; + public InItem(String indexName) { this.indexName = requireNonNullElse(indexName, ""); } @@ -20,6 +22,7 @@ public abstract class InItem extends Item { public void setIndexName(String index) { this.indexName = requireNonNullElse(index, ""); } + public String getIndexName() { return indexName; } @@ -43,4 +46,4 @@ public abstract class InItem extends Item { return Objects.hash(super.hashCode(), indexName); } -}; +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java index 2c6d69d3fac..9763e64b83c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java @@ -57,6 +57,10 @@ public class IntItem extends TermItem { expression = toExpression(from, to, 0); } + public IntItem(Long expression) { + this(expression, ""); + } + public IntItem(String expression) { this(expression, ""); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java index 9333173d898..e981e251064 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.query; import com.yahoo.compress.IntegerCompressor; +import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; import java.util.Collection; @@ -9,13 +10,14 @@ import java.util.HashSet; import java.util.Objects; import java.util.Set; -/* +/** * Class representing an IN operator with a set of 64-bit * integer values. * * @author toregge */ public class NumericInItem extends InItem { + private Set<Long> tokens; public NumericInItem(String indexName) { @@ -23,6 +25,11 @@ public class NumericInItem extends InItem { tokens = new HashSet<>(1000); } + public NumericInItem(String indexName, Set<Long> tokens) { + super(indexName); + this.tokens = new HashSet<>(tokens); + } + @Override public Item.ItemType getItemType() { return Item.ItemType.NUMERIC_IN; @@ -73,6 +80,13 @@ public class NumericInItem extends InItem { public Collection<Long> getTokens() { return Set.copyOf(tokens); } @Override + public void disclose(Discloser discloser) { + super.disclose(discloser); + for (Long token : tokens) + discloser.addChild(new IntItem(token)); + } + + @Override public boolean equals(Object o) { if (o == this) return true; if ( ! super.equals(o)) return false; @@ -86,4 +100,11 @@ public class NumericInItem extends InItem { return Objects.hash(super.hashCode(), tokens); } + @Override + public NumericInItem clone() { + NumericInItem clone = (NumericInItem)super.clone(); + clone.tokens = new HashSet<>(tokens); + return clone; + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java b/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java index 4473010082e..caa066eddd3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.query; import com.yahoo.compress.IntegerCompressor; +import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; import java.util.Collection; @@ -9,12 +10,13 @@ import java.util.HashSet; import java.util.Objects; import java.util.Set; -/* +/** * Class representing an IN operator with a set of string values. * * @author toregge */ public class StringInItem extends InItem { + private Set<String> tokens; public StringInItem(String indexName) { @@ -22,6 +24,11 @@ public class StringInItem extends InItem { tokens = new HashSet<>(1000); } + public StringInItem(String indexName, Set<String> tokens) { + super(indexName); + this.tokens = new HashSet<>(tokens); + } + @Override public ItemType getItemType() { return ItemType.STRING_IN; @@ -72,14 +79,21 @@ public class StringInItem extends InItem { tokens.remove(token); } - public Collection<String> getTokens() { return Set.copyOf(tokens); } + public Collection<String> getTokens() {return Set.copyOf(tokens);} + + @Override + public void disclose(Discloser discloser) { + super.disclose(discloser); + for (String token : tokens) + discloser.addChild(new WordItem(token)); + } @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! super.equals(o)) return false; - var other = (StringInItem)o; - if ( ! Objects.equals(this.tokens, other.tokens)) return false; + if (!super.equals(o)) return false; + var other = (StringInItem) o; + if (!Objects.equals(this.tokens, other.tokens)) return false; return true; } @@ -88,4 +102,11 @@ public class StringInItem extends InItem { return Objects.hash(super.hashCode(), tokens); } + @Override + public StringInItem clone() { + StringInItem clone = (StringInItem) super.clone(); + clone.tokens = new HashSet<>(tokens); + return clone; + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 931f9a1f1d9..612a6ca5618 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -20,14 +20,14 @@ import java.util.Objects; public final class WeakAndItem extends NonReducibleCompositeItem { /** The default N used if none is specified: 100 */ - public static final int defaultN = 100; + public static final int defaultN = 100; //TODO Make private private int n; private String index; /** Creates a WAND item with default N */ public WeakAndItem() { - this(defaultN); + this(-1); } public WeakAndItem(int N) { @@ -72,25 +72,25 @@ public final class WeakAndItem extends NonReducibleCompositeItem { protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); - buffer.append(n); + buffer.append(getN()); buffer.append(") "); } - public int getN() { return n; } - + public int getN() { return nIsExplicit() ? n : defaultN; } + public boolean nIsExplicit() { return n > 0; } public void setN(int N) { this.n = N; } @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); - IntegerCompressor.putCompressedPositiveNumber(n, buffer); + IntegerCompressor.putCompressedPositiveNumber(getN(), buffer); putString(index, buffer); } @Override public void disclose(Discloser discloser) { super.disclose(discloser); - discloser.addProperty("N", n); + discloser.addProperty("N", getN()); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index 7dc94194f37..05a10efc12a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -118,7 +118,7 @@ public class AdvancedParser extends StructuredParser { } /** Returns whether the item is a specific word item */ - private boolean isTheWord(String word, Item item) { + private static boolean isTheWord(String word, Item item) { if (!(item instanceof WordItem)) { return false; } @@ -126,6 +126,11 @@ public class AdvancedParser extends StructuredParser { } + private static boolean needWeakAnd(Item topLevelItem, int n) { + return !(topLevelItem instanceof WeakAndItem topLevelWeakAnd) || + ((n != 0 || topLevelWeakAnd.nIsExplicit()) && (n != topLevelWeakAnd.getN())); + + } /** Returns the new top level, or null if the current item is not an operator */ private Item handleAdvancedOperator(Item topLevelItem, Item item, boolean topLevelIsClosed) { @@ -155,11 +160,11 @@ public class AdvancedParser extends StructuredParser { return topLevelItem; } else if (isTheWord("wand", item) || isTheWord("weakand", item)) { int n = consumeNumericArgument(); - if (n == 0) - n = WeakAndItem.defaultN; - if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n != ((WeakAndItem)topLevelItem).getN()) { + if (topLevelIsClosed || needWeakAnd(topLevelItem, n)) { WeakAndItem wand = new WeakAndItem(); - wand.setN(n); + if (n != 0) { + wand.setN(n); + } wand.addItem(topLevelItem); return wand; } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index 3c4e8107df5..2403ae0d6c8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -72,35 +72,22 @@ public class JuniperSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { Result result = execution.search(query); - highlight(query.getPresentation().getBolding(), result.hits().deepIterator(), null, + highlight(query.getPresentation().getBolding(), result.hits().unorderedDeepIterator(), execution.context().getIndexFacts().newSession(query)); return result; } @Override public void fill(Result result, String summaryClass, Execution execution) { - int worstCase = result.getHitCount(); - List<Hit> hits = new ArrayList<>(worstCase); - for (Iterator<Hit> i = result.hits().deepIterator(); i.hasNext();) { - Hit hit = i.next(); - if ( ! (hit instanceof FastHit fastHit)) continue; - if (fastHit.isFilled(summaryClass)) continue; - - hits.add(fastHit); - } execution.fill(result, summaryClass); - highlight(result.getQuery().getPresentation().getBolding(), hits.iterator(), summaryClass, + highlight(result.getQuery().getPresentation().getBolding(), result.hits().unorderedDeepIterator(), execution.context().getIndexFacts().newSession(result.getQuery())); } - private void highlight(boolean bolding, Iterator<Hit> hitsToHighlight, - String summaryClass, IndexFacts.Session indexFacts) { + private void highlight(boolean bolding, Iterator<Hit> hitsToHighlight, IndexFacts.Session indexFacts) { while (hitsToHighlight.hasNext()) { Hit hit = hitsToHighlight.next(); if ( ! (hit instanceof FastHit fastHit)) continue; - - if (summaryClass != null && ! fastHit.isFilled(summaryClass)) continue; - Object searchDefinitionField = fastHit.getField(Hit.SDDOCNAME_FIELD); if (searchDefinitionField == null) continue; diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 4ec3fa358d2..8e0897b866f 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -384,9 +384,11 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // We need special handling for "select" because it can be both the prefix of the nested JSON select // parameters, and a plain select expression. The latter will be disallowed by query profile types // since they contain the former. - String select = requestMap.get(Select.SELECT); + Object select = requestMap.get(Select.SELECT); + if (select == null) + select = queryProfile.get(Select.SELECT, requestMap); if (select != null) - properties().set(Select.SELECT, select); + properties().set(Select.SELECT, select.toString()); } else { // bypass these complications if there is no query profile to get values from and validate against properties(). @@ -605,7 +607,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public void setHits(int hits) { if (hits < 0) - throw new IllegalArgumentException("Must be a positive number"); + throw new IllegalArgumentException("'hits' must be a positive number, not " + hits); this.hits = hits; } @@ -614,12 +616,12 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public void setOffset(int offset) { if (offset < 0) - throw new IllegalArgumentException("Must be a positive number"); + throw new IllegalArgumentException("'offset' must be a positive number, not " + offset); this.offset = offset; } /** Convenience method to set both the offset and the number of hits to return */ - public void setWindow(int offset,int hits) { + public void setWindow(int offset, int hits) { setOffset(offset); setHits(hits); } @@ -906,6 +908,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns the select to be used for this query, never null */ public Select getSelect() { return select; } + /** Sets the select (grouping) parameter from a string. */ + public void setSelect(String groupingString) { select.setGroupingExpressionString(groupingString); } + /** Returns the ranking to be used for this query, never null */ public Ranking getRanking() { return ranking; } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index fba9064298c..312a25d356a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -15,6 +15,7 @@ import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.GroupingValidator; import com.yahoo.search.grouping.result.Group; import com.yahoo.search.grouping.result.RootGroup; +import com.yahoo.search.query.Trace; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; @@ -98,30 +99,34 @@ public class GroupingExecutor extends Searcher { return result; } + private String extractSummaryClass(Hit hit, String summaryClass) { + Object metaData = hit.getSearcherSpecificMetaData(this); + if (metaData instanceof String metaDataString) { + // Use the summary class specified by grouping, set in HitConverter, for the first fill request + // after grouping. This assumes the first fill request is using the default summary class, + // which may be a fragile assumption. But currently we cannot do better because the difference + // between explicit and implicit summary class in fill is erased by the Execution. + // + // We reset the summary class here such that following fill calls will execute with the + // summary class they specify + hit.setSearcherSpecificMetaData(this, null); + return metaDataString; + } + return summaryClass; + } + @Override public void fill(Result result, String summaryClass, Execution execution) { Map<String, Result> summaryMap = new HashMap<>(); for (Iterator<Hit> it = result.hits().unorderedDeepIterator(); it.hasNext(); ) { Hit hit = it.next(); - Object metaData = hit.getSearcherSpecificMetaData(this); - if (metaData instanceof String) { - // Use the summary class specified by grouping, set in HitConverter, for the first fill request - // after grouping. This assumes the first fill request is using the default summary class, - // which may be a fragile assumption. But currently we cannot do better because the difference - // between explicit and implicit summary class in fill is erased by the Execution. - // - // We reset the summary class here such that following fill calls will execute with the - // summary class they specify - summaryClass = (String) metaData; - hit.setSearcherSpecificMetaData(this, null); - } - Result summaryResult = summaryMap.get(summaryClass); - if (summaryResult == null) { - summaryResult = new Result(result.getQuery()); - summaryMap.put(summaryClass, summaryResult); - } + Result summaryResult = summaryMap.computeIfAbsent(extractSummaryClass(hit, summaryClass), key -> new Result(result.getQuery())); summaryResult.hits().add(hit); } + Trace trace = result.getQuery().getTrace(); + if (trace.isTraceable(2)) { + trace.trace("GroupingExecutor.fill(" + summaryClass + ") = {" + summaryMap.keySet() + "}", 2); + } for (Map.Entry<String, Result> entry : summaryMap.entrySet()) { Result res = entry.getValue(); execution.fill(res, entry.getKey()); @@ -218,7 +223,7 @@ public class GroupingExecutor extends Searcher { if (lastPass > 0) { baseRoot = origRoot.clone(); } - if (query.getTrace().isTraceable(3) && query.getGroupingSessionCache()) { + if (query.getTrace().isTraceable(3)) { query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId() + "'.", 3); } for (int pass = 0; pass <= lastPass; ++pass) { diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index 09de1a24ef9..f2505a47aa2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -14,7 +14,9 @@ import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.query.ranking.Matching; import com.yahoo.search.query.ranking.RankFeatures; import com.yahoo.search.query.ranking.RankProperties; +import com.yahoo.search.query.ranking.SecondPhase; import com.yahoo.search.query.ranking.SoftTimeout; +import com.yahoo.search.query.ranking.Significance; import com.yahoo.search.result.ErrorMessage; import java.util.Objects; @@ -45,8 +47,10 @@ public class Ranking implements Cloneable { public static final String KEEPRANKCOUNT = "keepRankCount"; public static final String RANKSCOREDROPLIMIT = "rankScoreDropLimit"; public static final String MATCH_PHASE = "matchPhase"; + public static final String SECOND_PHASE = "secondPhase"; public static final String GLOBAL_PHASE = "globalPhase"; public static final String DIVERSITY = "diversity"; + public static final String SIGNIFICANCE = "significance"; public static final String SOFTTIMEOUT = "softtimeout"; public static final String MATCHING = "matching"; public static final String FEATURES = "features"; @@ -69,9 +73,11 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(RANKSCOREDROPLIMIT, "double")); argumentType.addField(new FieldDescription(GLOBAL_PHASE, new QueryProfileFieldType(GlobalPhase.getArgumentType()))); argumentType.addField(new FieldDescription(MATCH_PHASE, new QueryProfileFieldType(MatchPhase.getArgumentType()), "matchPhase")); + argumentType.addField(new FieldDescription(SECOND_PHASE, new QueryProfileFieldType(SecondPhase.getArgumentType()))); argumentType.addField(new FieldDescription(DIVERSITY, new QueryProfileFieldType(Diversity.getArgumentType()))); argumentType.addField(new FieldDescription(SOFTTIMEOUT, new QueryProfileFieldType(SoftTimeout.getArgumentType()))); argumentType.addField(new FieldDescription(MATCHING, new QueryProfileFieldType(Matching.getArgumentType()))); + argumentType.addField(new FieldDescription(SIGNIFICANCE, new QueryProfileFieldType(Significance.getArgumentType()))); argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature input")); // Repeated at the end of RankFeatures argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty")); argumentType.freeze(); @@ -107,13 +113,15 @@ public class Ranking implements Cloneable { private MatchPhase matchPhase = new MatchPhase(); + private SecondPhase secondPhase = new SecondPhase(); + private GlobalPhase globalPhase = new GlobalPhase(); private Matching matching = new Matching(); private SoftTimeout softTimeout = new SoftTimeout(); - private boolean useSignificance = false; + private Significance significance = new Significance(); public Ranking(Query parent) { this.parent = parent; @@ -219,17 +227,12 @@ public class Ranking implements Cloneable { /** Returns whether rank features should be dumped with the result of this query, default false */ public boolean getListFeatures() { return listFeatures; } - /** Set whether to use significance in ranking */ - @com.yahoo.api.annotations.Beta - public void setUseSignificance(boolean useSignificance) { this.useSignificance = useSignificance; } - - /** Returns whether to use significance in ranking */ - @com.yahoo.api.annotations.Beta - public boolean getUseSignificance() { return useSignificance; } - /** Returns the match phase rank settings of this. This is never null. */ public MatchPhase getMatchPhase() { return matchPhase; } + /** Return the second-phase rank settings of this. This is never null. */ + public SecondPhase getSecondPhase() { return secondPhase; } + /** Returns the global-phase rank settings of this. This is never null. */ public GlobalPhase getGlobalPhase() { return globalPhase; } @@ -239,6 +242,10 @@ public class Ranking implements Cloneable { /** Returns the soft timeout settings of this. This is never null. */ public SoftTimeout getSoftTimeout() { return softTimeout; } + /** Returns the significance settings of this. This is never null. */ + @com.yahoo.api.annotations.Beta + public Significance getSignificance() { return significance; } + /** Returns the sorting spec of this query, or null if none is set */ public Sorting getSorting() { return sorting; } @@ -266,6 +273,7 @@ public class Ranking implements Cloneable { public void prepare() { rankFeatures.prepare(rankProperties); matchPhase.prepare(rankProperties); + secondPhase.prepare(rankProperties); matching.prepare(rankProperties); softTimeout.prepare(rankProperties); prepareNow(freshness); diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index 6735a6bd050..38ef7b8f190 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -115,9 +115,7 @@ public class Select implements Cloneable { public String getGroupingExpressionString() { return groupingExpressionString; } /** Returns the grouping in the query */ - public String getGroupingString(){ - return grouping; - } + public String getGroupingString() { return grouping; } /** * Returns the query's {@link GroupingRequest} as a mutable list. Changing this directly changes the grouping diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 90d5e04d2b6..df9d95892ed 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -478,10 +478,7 @@ public class SelectParser implements Parser { if (annotations != null) { annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { - if (TARGET_HITS.equals(annotation_name)){ - weakAnd.setN((int)(annotation_value.asDouble())); - } - if (TARGET_NUM_HITS.equals(annotation_name)) { + if (TARGET_HITS.equals(annotation_name) || TARGET_NUM_HITS.equals(annotation_name)){ weakAnd.setN((int)(annotation_value.asDouble())); } }); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 8806854b9ce..eed6962b14a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -22,6 +22,7 @@ import com.yahoo.search.query.ranking.Diversity; import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.query.ranking.Matching; import com.yahoo.search.query.ranking.SoftTimeout; +import com.yahoo.search.query.ranking.Significance; import com.yahoo.tensor.Tensor; import java.util.HashMap; @@ -101,12 +102,14 @@ public class QueryProperties extends Properties { map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.MINGROUPS), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getMinGroups(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setMinGroups(asLong(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.CUTOFF, Diversity.FACTOR), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getCutoffFactor(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setCutoffFactor(asDouble(value, 10.0)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.CUTOFF, Diversity.STRATEGY), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getCutoffStrategy(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setCutoffStrategy(asString(value, "loose")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SECOND_PHASE, Ranking.RANKSCOREDROPLIMIT), GetterSetter.of(query -> query.getRanking().getSecondPhase().getRankScoreDropLimit(), (query, value) -> query.getRanking().getSecondPhase().setRankScoreDropLimit(asDouble(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.GLOBAL_PHASE, Ranking.RERANKCOUNT), GetterSetter.of(query -> query.getRanking().getGlobalPhase().getRerankCount(), (query, value) -> query.getRanking().getGlobalPhase().setRerankCount(asInteger(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.ENABLE), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getEnable(), (query, value) -> query.getRanking().getSoftTimeout().setEnable(asBoolean(value, true)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.FACTOR), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getFactor(), (query, value) -> query.getRanking().getSoftTimeout().setFactor(asDouble(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.TAILCOST), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getTailcost(), (query, value) -> query.getRanking().getSoftTimeout().setTailcost(asDouble(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SIGNIFICANCE, Significance.USE_MODEL), GetterSetter.of(query -> query.getRanking().getSignificance().getUseModel().orElse(false), (query, value) -> query.getRanking().getSignificance().setUseModel(asBoolean(value, false)))); map.put(CompoundName.fromComponents(Select.SELECT), GetterSetter.of(query -> query.getSelect().getGroupingExpressionString(), (query, value) -> query.getSelect().setGroupingExpressionString(asString(value, "")))); map.put(CompoundName.fromComponents(Select.SELECT, Select.WHERE), GetterSetter.of(query -> query.getSelect().getWhereString(), (query, value) -> query.getSelect().setWhereString(asString(value, "")))); map.put(CompoundName.fromComponents(Select.SELECT, Select.GROUPING), GetterSetter.of(query -> query.getSelect().getGroupingString(), (query, value) -> query.getSelect().setGroupingString(asString(value, "")))); diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SecondPhase.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SecondPhase.java new file mode 100644 index 00000000000..0f6564d827f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SecondPhase.java @@ -0,0 +1,73 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.ranking; + +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.Objects; + +/** + * The second-phase ranking settings of this query. + * + * @author toregge + */ +public class SecondPhase implements Cloneable { + + /** The type representing the property arguments consumed by this */ + private static final QueryProfileType argumentType; + + static { + argumentType = new QueryProfileType(Ranking.SECOND_PHASE); + argumentType.setStrict(true); + argumentType.setBuiltin(true); + argumentType.addField(new FieldDescription(Ranking.RANKSCOREDROPLIMIT, FieldType.doubleType)); + argumentType.freeze(); + } + public static QueryProfileType getArgumentType() { return argumentType; } + + private Double rankScoreDropLimit = null; + + /** Sets the second phase rank-score-drop-limit that will be used, or null if not set */ + public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } + + /** Returns the second phase rank-score-drop-limit that will be used, or null if not set */ + public Double getRankScoreDropLimit() { return rankScoreDropLimit; } + + /** Internal operation - DO NOT USE */ + public void prepare(RankProperties rankProperties) { + if (rankScoreDropLimit == null) { + return; + } + rankProperties.put("vespa.hitcollector.secondphase.rankscoredroplimit", String.valueOf(rankScoreDropLimit)); + } + + @Override + public int hashCode() { + return Objects.hashCode(this.rankScoreDropLimit); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o instanceof SecondPhase other) { + if ( ! Objects.equals(this.rankScoreDropLimit, other.rankScoreDropLimit)) return false; + return true; + } + return false; + } + + @Override + public SecondPhase clone() { + try { + SecondPhase clone = (SecondPhase)super.clone(); + clone.rankScoreDropLimit = this.rankScoreDropLimit; + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Won't happen", e); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/Significance.java b/container-search/src/main/java/com/yahoo/search/query/ranking/Significance.java new file mode 100644 index 00000000000..d4762e55910 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/Significance.java @@ -0,0 +1,68 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.ranking; + +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.Objects; +import java.util.Optional; + +/** + * The significance ranking settings of this query. + * + * @author MariusArhaug + */ +public class Significance implements Cloneable { + + /** The type representing the property arguments consumed by this */ + private static final QueryProfileType argumentType; + + public static final String USE_MODEL = "useModel"; + + static { + argumentType = new QueryProfileType(Ranking.SECOND_PHASE); + argumentType.setStrict(true); + argumentType.setBuiltin(true); + argumentType.addField(new FieldDescription(USE_MODEL, FieldType.booleanType)); + argumentType.freeze(); + } + public static QueryProfileType getArgumentType() { return argumentType; } + + private Boolean useModel = null; + + public void setUseModel(boolean useModel) { + this.useModel = useModel; + } + + public Optional<Boolean> getUseModel() { + return Optional.ofNullable(useModel); + } + + @Override + public int hashCode() { + return Objects.hashCode(this.useModel); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o instanceof Significance other) { + if ( ! Objects.equals(this.useModel, other.useModel)) return false; + return true; + } + return false; + } + + @Override + public Significance clone() { + try { + return (Significance) super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Won't happen", e); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 7536e74042c..afea39ac787 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -12,6 +12,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.yql.MinimalQueryInserter; import com.yahoo.yolean.chain.After; +import com.yahoo.yolean.chain.Provides; /** * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true. @@ -19,10 +20,12 @@ import com.yahoo.yolean.chain.After; * * @author karowan */ +@Provides(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND) @After(MinimalQueryInserter.EXTERNAL_YQL) public class WeakAndReplacementSearcher extends Searcher { + public static final String REPLACE_OR_WITH_WEAKAND = "replace-or-with-weakand"; static final CompoundName WEAKAND_REPLACE = CompoundName.from("weakAnd.replace"); - static final CompoundName WAND_HITS = CompoundName.from("wand.hits"); + public static final CompoundName WAND_HITS = CompoundName.from("wand.hits"); @Override public Result search(Query query, Execution execution) { if (!query.properties().getBoolean(WEAKAND_REPLACE)) { @@ -38,7 +41,9 @@ public class WeakAndReplacementSearcher extends Searcher { */ private void replaceOrItems(Query query) { Item root = query.getModel().getQueryTree().getRoot(); - int hits = query.properties().getInteger(WAND_HITS, WeakAndItem.defaultN); + int hits = query.getHits(); + Integer wandHits = query.properties().getInteger(WAND_HITS); + if (wandHits != null) hits = wandHits; query.getModel().getQueryTree().setRoot(replaceOrItems(root, hits)); if (root != query.getModel().getQueryTree().getRoot()) query.trace("Replaced OR by WeakAnd", true, 2); diff --git a/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java index 88a1e6c1485..ffbb63514f1 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java @@ -79,13 +79,16 @@ public class EventRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeRaw("event: " + event.type() + "\n"); } generator.writeRaw("data: "); - generator.writeStartObject(); - generator.writeStringField(event.type(), event.toString()); - generator.writeEndObject(); + if (event.type().equals("hits")) { + generator.writeRaw(event.toString()); + } else { + generator.writeStartObject(); + generator.writeStringField(event.type(), event.toString()); + generator.writeEndObject(); + } generator.writeRaw("\n\n"); generator.flush(); } - // Todo: support other types of data such as search results (hits), timing and trace } @Override diff --git a/container-search/src/main/java/com/yahoo/search/result/DeepHitIterator.java b/container-search/src/main/java/com/yahoo/search/result/DeepHitIterator.java index e130ac49c27..3b72bd77e8d 100644 --- a/container-search/src/main/java/com/yahoo/search/result/DeepHitIterator.java +++ b/container-search/src/main/java/com/yahoo/search/result/DeepHitIterator.java @@ -1,7 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.result; -import java.util.*; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; /** * An iterator for the forest of hits in a result. @@ -21,7 +25,7 @@ public class DeepHitIterator implements Iterator<Hit> { * Create a deep hit iterator based on the given hit iterator. * * @param it The hits iterator to traverse. - * @param ordered Whether or not the hits should be ordered. + * @param ordered Whether the hits should be ordered. */ public DeepHitIterator(Iterator<Hit> it, boolean ordered) { this.ordered = ordered; diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index be31b91a304..553d3356e49 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -442,7 +442,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< /** * Combines two error hits to one. Any one argument may be null, in which case the other is returned. * - * @return true if this should also be added to the list of hits of this result + * @return Merged errors */ private DefaultErrorHit merge(DefaultErrorHit first, DefaultErrorHit second) { if (first == null) return second; @@ -716,7 +716,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< Set<String> filled = getFilledInternal(); if (filled == null) { if (hitFilled.isEmpty()) { - filled = null; + // Intentionally empty } else if (hitFilled.size() == 1) { //TODO Avoid needing set that allows null .... filled = Collections.singleton(hitFilled.iterator().next()); @@ -813,7 +813,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< } - for (; i.hasNext();) { + while (i.hasNext()) { Hit hit = i.next(); analyzeHit(hit); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java index 84563818007..b7aeecdb7c8 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java @@ -121,11 +121,7 @@ public class AsyncExecution { }, query); } - /** - * The future of this functions returns the original Result - * - * @see com.yahoo.search.searchchain.Execution - */ + /** Fills this result and returns the future where it is filled. */ public FutureResult fill(Result result, String summaryClass) { return getFutureResult(execution.context().executor(), () -> { execution.fill(result, summaryClass); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 6eb69c76afd..08526ecbaef 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -499,16 +499,6 @@ public class Execution extends com.yahoo.processing.execution.Execution { return (Result)super.process(query); } - @Override - protected void onInvoking(Request request, Processor processor) { - super.onInvoking(request,processor); - final int traceDependencies = 6; - Query query = (Query) request; - if (query.getTrace().getLevel() >= traceDependencies) { - query.trace(processor.getId() + " " + processor.getDependencies(), traceDependencies); - } - } - /** * The default response returned from this kind of execution when there are not further processors * - an empty Result diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java index 69a1f8ec6cb..c03a74ea2c5 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java @@ -34,7 +34,8 @@ public class VespaSearchers { com.yahoo.prelude.searcher.PosSearcher.class, com.yahoo.prelude.semantics.SemanticSearcher.class, com.yahoo.search.grouping.GroupingQueryParser.class, - com.yahoo.search.querytransform.WeakAndReplacementSearcher.class); + com.yahoo.search.querytransform.WeakAndReplacementSearcher.class, + com.yahoo.search.searchers.OpportunisticWeakAndSearcher.class); public static final Collection<ChainedComponentModel> nativeSearcherModels; diff --git a/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java new file mode 100644 index 00000000000..924951fe430 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java @@ -0,0 +1,105 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.searchers; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.querytransform.WeakAndReplacementSearcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Will opportunistically replace the WeakAND with an AND as it is faster. + * If enough hits are returned all is good and we return. If not we fall back to the original query. + * It is default off, and is enabled with weakAnd.opportunistic.and=true. + * It can be tuned with weakAnd.opportunistic.factor. Higher value than 1 might increase quality, lower value will + * improve performance. Default is 1.0. This factor is multiplied with the heap size of the wand(default 100) as target hits. + * + * @author baldersheim + */ +@Beta +@After(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND) +public class OpportunisticWeakAndSearcher extends Searcher { + private static final CompoundName OPPORTUNISTIC_AND = CompoundName.from("weakAnd.opportunistic.and"); + private static final CompoundName OPPORTUNISTIC_FACTOR = CompoundName.from("weakAnd.opportunistic.factor"); + + @Override + public Result search(Query query, Execution execution) { + if (query.getHits() > WeakAndItem.defaultN) { + adjustWeakAndHeap(query.getModel().getQueryTree().getRoot(), query.getHits()); + } + if (!query.properties().getBoolean(OPPORTUNISTIC_AND)) { + return execution.search(query); + } + + Item originalRoot = query.getModel().getQueryTree().getRoot(); + int targetHits = (int)(targetHits(originalRoot) * query.properties().getDouble(OPPORTUNISTIC_FACTOR, 1.0)); + if (targetHits >= 0) { + query.getModel().getQueryTree().setRoot(weakAnd2AndRecurse(originalRoot.clone())); + query.trace("WeakAND(" + targetHits+ ") => AND", true, 2); + Result result = execution.search(query); + if (result.getTotalHitCount() >= targetHits) { + return result; + } + query.getModel().getQueryTree().setRoot(originalRoot); + query.trace("Fallback to WeakAND(" + targetHits+ ") as AND => " + result, true, 2); + return execution.search(query); + } + return execution.search(query); + } + + static Item adjustWeakAndHeap(Item item, int hits) { + if (item instanceof WeakAndItem weakAnd && hits > weakAnd.getN() && !weakAnd.nIsExplicit()) { + weakAnd.setN(hits); + } + if (item instanceof CompositeItem compositeItem) { + for (int i = 0; i < compositeItem.getItemCount(); i++) { + adjustWeakAndHeap(compositeItem.getItem(i), hits); + } + } + return item; + } + + // returns targetHits for the first WeakAndItem found, -1 if none found. + static int targetHits(Item item) { + if (!(item instanceof CompositeItem compositeItem)) return -1; + if (item instanceof WeakAndItem weakAndItem) { + return (weakAndItem.getItemCount() >= 2) ? weakAndItem.getN() : -1; + } + for (int i = 0; i < compositeItem.getItemCount(); i++) { + int targetHits = targetHits(compositeItem.getItem(i)); + if (targetHits >= 0) return targetHits; + } + return -1; + } + + static Item weakAnd2AndRecurse(Item item) { + if (!(item instanceof CompositeItem compositeItem)) return item; + compositeItem = weakAnd2And(compositeItem); + for (int i = 0; i < compositeItem.getItemCount(); i++) { + Item subItem = compositeItem.getItem(i); + Item replacedItem = weakAnd2AndRecurse(subItem); + if (replacedItem != subItem) { + compositeItem.setItem(i, replacedItem); + } + } + return compositeItem; + } + + private static CompositeItem weakAnd2And(CompositeItem item) { + if (item instanceof WeakAndItem weakAndItem) { + AndItem andItem = new AndItem(); + andItem.setWeight(weakAndItem.getWeight()); + item.items().forEach(andItem::addItem); + return andItem; + } + return item; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java index e3a559da8f9..9e6e2f785fd 100644 --- a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java @@ -51,8 +51,17 @@ public class SignificanceSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { + var ranking = query.getRanking(); var rankProfileName = query.getRanking().getProfile(); + Optional<Boolean> useSignificanceModelOverride = ranking.getSignificance().getUseModel(); + + if (useSignificanceModelOverride.isPresent() && !useSignificanceModelOverride.get()) { + return execution.search(query); + } + if (useSignificanceModelOverride.isPresent()) { + return calculateAndSetSignificance(query, execution); + } // Determine significance setup per schema for the given rank profile var perSchemaSetup = schemaInfo.newSession(query).schemas().stream() .collect(Collectors.toMap(Schema::name, schema -> @@ -72,7 +81,7 @@ public class SignificanceSearcher extends Searcher { "(https://docs.vespa.ai/en/schemas.html#multiple-schemas). " + "Specify same 'significance' configuration for all selected schemas " + "(https://docs.vespa.ai/en/reference/schema-reference.html#significance).") - .formatted(rankProfileName, perSchemaSetup.keySet()))); + .formatted(rankProfileName, perSchemaSetup.keySet()))); return result; } @@ -80,6 +89,10 @@ public class SignificanceSearcher extends Searcher { var useSignificanceModel = uniqueSetups.iterator().next(); if (!useSignificanceModel) return execution.search(query); + return calculateAndSetSignificance(query, execution); + } + + private Result calculateAndSetSignificance(Query query, Execution execution) { Language language = query.getModel().getParsingLanguage(); Optional<SignificanceModel> model = significanceModelRegistry.getModel(language); @@ -111,7 +124,6 @@ public class SignificanceSearcher extends Searcher { public static double calculateIDF(long N, long nq_i) { return Math.log(1 + (N - nq_i + 0.5) / (nq_i + 0.5)); } - } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index a354006aa9b..850430484b2 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -233,7 +233,7 @@ public class VespaSerializer { boolean serialize(StringBuilder destination, EquivItem item) { String annotations = leafAnnotations(item); destination.append(getIndexName(item.getItem(0))).append(" contains "); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append("({").append(annotations).append("}"); } destination.append(EQUIV).append('('); @@ -251,7 +251,7 @@ public class VespaSerializer { destination.append('"'); } } - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } destination.append(')'); @@ -270,7 +270,7 @@ public class VespaSerializer { String annotations = nearAnnotations(item); destination.append(getIndexName(item.getItem(0))).append(" contains "); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append('(').append(annotations); } destination.append(NEAR).append('('); @@ -284,7 +284,7 @@ public class VespaSerializer { escape(close.getIndexedString(), destination).append('"'); } destination.append(')'); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } return false; @@ -415,15 +415,15 @@ public class VespaSerializer { } else if (rightOpen) { boundsAnnotation = BOUNDS + ": " + "\"" + BOUNDS_RIGHT_OPEN + "\""; } - if (annotations.length() > 0 || boundsAnnotation.length() > 0) { + if (!annotations.isEmpty() || !boundsAnnotation.isEmpty()) { destination.append("({"); } initLen = destination.length(); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(annotations); } comma(destination, initLen); - if (boundsAnnotation.length() > 0) { + if (!boundsAnnotation.isEmpty()) { destination.append(boundsAnnotation); } if (initLen != annotations.length()) { @@ -434,7 +434,7 @@ public class VespaSerializer { .append(", ").append(intItem.getFromLimit().number()) .append(", ").append(intItem.getToLimit().number()) .append(")"); - if (annotations.length() > 0 || boundsAnnotation.length() > 0) { + if (!annotations.isEmpty() || !boundsAnnotation.isEmpty()) { destination.append(")"); } } @@ -442,7 +442,7 @@ public class VespaSerializer { private void annotatedNumberImage(IntItem item, String rawNumber, StringBuilder image) { String annotations = leafAnnotations(item); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { image.append("({").append(annotations).append("}"); } if ('-' == rawNumber.charAt(0)) { @@ -453,7 +453,7 @@ public class VespaSerializer { if ('-' == rawNumber.charAt(0)) { image.append(')'); } - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { image.append(')'); } } @@ -533,7 +533,7 @@ public class VespaSerializer { destination.append(normalizeIndexName(fuzzy.getIndexName())).append(" contains "); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append('(').append(annotations); } @@ -542,7 +542,7 @@ public class VespaSerializer { escape(fuzzy.getIndexedString(), destination).append('"'); destination.append(')'); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } return false; @@ -591,7 +591,7 @@ public class VespaSerializer { String annotations = NearSerializer.nearAnnotations(item); destination.append(getIndexName(item.getItem(0))).append(" contains "); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append('(').append(annotations); } destination.append(ONEAR).append('('); @@ -605,7 +605,7 @@ public class VespaSerializer { escape(close.getIndexedString(), destination).append('"'); } destination.append(')'); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } return false; @@ -681,7 +681,7 @@ public class VespaSerializer { destination.append("({"); serializeOrigin(destination, image, offset, length); String annotations = leafAnnotations(phrase); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(", ").append(annotations); } if (phrase.getSegmentingRule() == SegmentingRule.BOOLEAN_AND) { @@ -710,7 +710,7 @@ public class VespaSerializer { if (includeField) destination.append(normalizeIndexName(phrase.getIndexName())).append(" contains "); - if (annotations.length() > 0) + if (!annotations.isEmpty()) destination.append("({").append(annotations).append("}"); destination.append(PHRASE).append('('); @@ -730,7 +730,7 @@ public class VespaSerializer { } } destination.append(')'); - if (annotations.length() > 0) + if (!annotations.isEmpty()) destination.append(')'); return false; } @@ -780,7 +780,7 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, GeoLocationItem item) { String annotations = leafAnnotations(item); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append("({").append(annotations).append("}"); } destination.append(GEO_LOCATION).append('('); @@ -896,7 +896,7 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, RangeItem range) { String annotations = leafAnnotations(range); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append("{").append(annotations).append("}"); } destination.append(RANGE).append('(') @@ -953,7 +953,7 @@ public class VespaSerializer { String annotations = leafAnnotations(alternatives); Substring origin = alternatives.getOrigin(); boolean isFromQuery = alternatives.isFromQuery(); - boolean needsAnnotations = annotations.length() > 0 || origin != null || !isFromQuery; + boolean needsAnnotations = !annotations.isEmpty() || origin != null || !isFromQuery; if (includeField) { destination.append(normalizeIndexName(alternatives.getIndexName())).append(" contains "); @@ -973,7 +973,7 @@ public class VespaSerializer { comma(destination, initLen); destination.append(IMPLICIT_TRANSFORMS).append(": false"); } - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { comma(destination, initLen); destination.append(annotations); } @@ -985,7 +985,7 @@ public class VespaSerializer { int initLen = destination.length(); List<WordAlternativesItem.Alternative> sortedAlternatives = new ArrayList<>(alternatives.getAlternatives()); // ensure most precise forms first - Collections.sort(sortedAlternatives, (x, y) -> Double.compare(y.exactness, x.exactness)); + sortedAlternatives.sort((x, y) -> Double.compare(y.exactness, x.exactness)); for (WordAlternativesItem.Alternative alternative : sortedAlternatives) { comma(destination, initLen); destination.append('"'); @@ -1049,7 +1049,7 @@ public class VespaSerializer { } private boolean needsAnnotationBlock(WeakAndItem item) { - return nonDefaultTargetNumHits(item); + return item.nIsExplicit(); } @Override @@ -1057,7 +1057,7 @@ public class VespaSerializer { if (needsAnnotationBlock(item)) { destination.append("({"); } - if (nonDefaultTargetNumHits(item)) { + if (item.nIsExplicit()) { destination.append(TARGET_NUM_HITS).append(": ").append(item.getN()); } if (needsAnnotationBlock(item)) { @@ -1067,9 +1067,6 @@ public class VespaSerializer { return true; } - private boolean nonDefaultTargetNumHits(WeakAndItem w) { - return w.getN() != WeakAndItem.defaultN; - } } private static class WeightedSetSerializer extends Serializer<WeightedSetItem> { @@ -1157,7 +1154,7 @@ public class VespaSerializer { StringBuilder wordAnnotations = new StringBuilder(WordSerializer.wordAnnotations(w)); String leafAnnotations = leafAnnotations(w); - if (leafAnnotations.length() > 0) { + if (!leafAnnotations.isEmpty()) { comma(wordAnnotations, 0); wordAnnotations.append(leafAnnotations(w)); } @@ -1472,7 +1469,7 @@ public class VespaSerializer { for (Iterator<Entry<Object, Integer>> i = weightedSet.getTokens(); i.hasNext();) { tokens.add(i.next()); } - Collections.sort(tokens, tokenComparator); + tokens.sort(tokenComparator); for (Entry<Object, Integer> entry : tokens) { comma(destination, initLen); destination.append('"'); @@ -1491,14 +1488,14 @@ public class VespaSerializer { int incomingLen = destination.length(); String annotations = leafAnnotations(weightedSet); - if (optionalAnnotations.length() > 0 || annotations.length() > 0) { + if (!optionalAnnotations.isEmpty() || !annotations.isEmpty()) { destination.append("({"); } preAnnotationValueLen = destination.length(); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(annotations); } - if (optionalAnnotations.length() > 0) { + if (!optionalAnnotations.isEmpty()) { comma(destination, preAnnotationValueLen); destination.append(optionalAnnotations); } @@ -1592,7 +1589,7 @@ public class VespaSerializer { } private static String normalizeIndexName(String indexName) { - if (indexName.length() == 0) { + if (indexName.isEmpty()) { return "default"; } else { return indexName; @@ -1600,12 +1597,12 @@ public class VespaSerializer { } private static void annotatedTerm(StringBuilder destination, IndexedItem w, String annotations) { - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append("({").append(annotations).append("}"); } destination.append('"'); escape(w.getIndexedString(), destination).append('"'); - if (annotations.length() > 0) { + if (!annotations.isEmpty()) { destination.append(')'); } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index fb4ec5ba872..1f377afdb5e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -434,18 +434,12 @@ public class YqlParser implements Parser { } private ParsedDegree degreesFromArg(OperatorNode<ExpressionOperator> ast, boolean first) { - Object arg = null; - switch (ast.getOperator()) { - case LITERAL: - arg = ast.getArgument(0); - break; - case READ_FIELD: - arg = ast.getArgument(1); - break; - default: - throw newUnexpectedArgumentException(ast.getOperator(), - ExpressionOperator.READ_FIELD, ExpressionOperator.PROPREF); - } + Object arg = switch (ast.getOperator()) { + case LITERAL -> ast.getArgument(0); + case READ_FIELD -> ast.getArgument(1); + default -> throw newUnexpectedArgumentException(ast.getOperator(), + ExpressionOperator.READ_FIELD, ExpressionOperator.PROPREF); + }; if (arg instanceof Number n) { return new ParsedDegree(n.doubleValue(), first, !first); } @@ -822,8 +816,10 @@ public class YqlParser implements Parser { // Set grammar-specific annotations if (WEAKAND_GRAMMARS.contains(grammar) && item instanceof WeakAndItem weakAndItem) { - weakAndItem.setN(getAnnotation(ast, TARGET_HITS, Integer.class, WeakAndItem.defaultN, - "'targetHits' (N) for weak and")); + Integer targetNumHits = getAnnotation(ast, TARGET_HITS, Integer.class, null, "'targetHits' (N) for weak and"); + if (targetNumHits != null) { + weakAndItem.setN(targetNumHits); + } } return item; } @@ -845,17 +841,16 @@ public class YqlParser implements Parser { } private String getStringContents(OperatorNode<ExpressionOperator> operator) { - switch (operator.getOperator()) { - case LITERAL: - return operator.getArgument(0, String.class); - case VARREF: + return switch (operator.getOperator()) { + case LITERAL -> operator.getArgument(0, String.class); + case VARREF -> { Preconditions.checkState(userQuery != null, - "properties must be available when trying to fetch user input"); - return userQuery.properties().getString(operator.getArgument(0, String.class)); - default: - throw newUnexpectedArgumentException(operator.getOperator(), - ExpressionOperator.LITERAL, ExpressionOperator.VARREF); - } + "properties must be available when trying to fetch user input"); + yield userQuery.properties().getString(operator.getArgument(0, String.class)); + } + default -> throw newUnexpectedArgumentException(operator.getOperator(), + ExpressionOperator.LITERAL, ExpressionOperator.VARREF); + }; } private void propagateUserInputAnnotationsRecursively(OperatorNode<ExpressionOperator> ast, Item item) { @@ -1139,16 +1134,15 @@ public class YqlParser implements Parser { negative = "-"; ast = ast.getArgument(0); } - switch (ast.getOperator()) { - case VARREF: + return switch (ast.getOperator()) { + case VARREF -> { Preconditions.checkState(userQuery != null, - "properties must be available when trying to fetch user input"); - return negative + userQuery.properties().getString(ast.getArgument(0, String.class)); - case LITERAL: - return negative + ast.getArgument(0).toString(); - default: - throw new IllegalArgumentException("Expected VARREF or LITERAL, got " + ast.getOperator()); - } + "properties must be available when trying to fetch user input"); + yield negative + userQuery.properties().getString(ast.getArgument(0, String.class)); + } + case LITERAL -> negative + ast.getArgument(0).toString(); + default -> throw new IllegalArgumentException("Expected VARREF or LITERAL, got " + ast.getOperator()); + }; } private String fetchConditionWord(OperatorNode<ExpressionOperator> ast) { @@ -1301,17 +1295,21 @@ public class YqlParser implements Parser { } else { Limit from; Limit to; - if (BOUNDS_OPEN.equals(bounds)) { - from = new Limit(lowerArg, false); - to = new Limit(upperArg, false); - } else if (BOUNDS_LEFT_OPEN.equals(bounds)) { - from = new Limit(lowerArg, false); - to = new Limit(upperArg, true); - } else if (BOUNDS_RIGHT_OPEN.equals(bounds)) { - from = new Limit(lowerArg, true); - to = new Limit(upperArg, false); - } else { - throw newUnexpectedArgumentException(bounds, BOUNDS_OPEN, BOUNDS_LEFT_OPEN, BOUNDS_RIGHT_OPEN); + switch (bounds) { + case BOUNDS_OPEN -> { + from = new Limit(lowerArg, false); + to = new Limit(upperArg, false); + } + case BOUNDS_LEFT_OPEN -> { + from = new Limit(lowerArg, false); + to = new Limit(upperArg, true); + } + case BOUNDS_RIGHT_OPEN -> { + from = new Limit(lowerArg, true); + to = new Limit(upperArg, false); + } + default -> + throw newUnexpectedArgumentException(bounds, BOUNDS_OPEN, BOUNDS_LEFT_OPEN, BOUNDS_RIGHT_OPEN); } return new IntItem(from, to, getIndex(args.get(0))); } @@ -1418,7 +1416,7 @@ public class YqlParser implements Parser { private Item instantiateWordAlternativesItem(String field, OperatorNode<ExpressionOperator> ast) { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); - Preconditions.checkArgument(args.size() >= 1, "Expected 1 or more arguments, got %s.", args.size()); + Preconditions.checkArgument(!args.isEmpty(), "Expected 1 or more arguments, got %s.", args.size()); Preconditions.checkArgument(args.get(0).getOperator() == ExpressionOperator.MAP, "Expected MAP, got %s.", args.get(0).getOperator()); @@ -1566,7 +1564,7 @@ public class YqlParser implements Parser { List<String> words = segmenter.segment(toSegment, usedLanguage); TaggableItem wordItem; - if (words.size() == 0) { + if (words.isEmpty()) { wordItem = new WordItem(wordData, fromQuery); } else if (words.size() == 1 || !phraseArgumentSupported(parent)) { wordItem = new WordItem(words.get(0), fromQuery); @@ -1909,18 +1907,7 @@ public class YqlParser implements Parser { return new IllegalArgumentException(out.toString()); } - private static final class ConnectedItem { - - final double weight; - final int toId; - final TaggableItem fromItem; - - ConnectedItem(TaggableItem fromItem, int toId, double weight) { - this.weight = weight; - this.toId = toId; - this.fromItem = fromItem; - } - } + private record ConnectedItem(TaggableItem fromItem, int toId, double weight) { } private class AnnotationPropagator extends QueryVisitor { diff --git a/container-search/src/main/resources/configdefinitions/llm-searcher.def b/container-search/src/main/resources/configdefinitions/llm-searcher.def index 608f44f04d0..04a9019bfbb 100755 --- a/container-search/src/main/resources/configdefinitions/llm-searcher.def +++ b/container-search/src/main/resources/configdefinitions/llm-searcher.def @@ -9,3 +9,9 @@ stream bool default=true # The external LLM provider - the id of a LanguageModel component providerId string default="" + +# The default prompt to use if not overridden in query +prompt string default="" + +# The default prompt template file to use if not overridden in query. Above prompt has precedence if it is set. +promptTemplate path optional
\ No newline at end of file diff --git a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java index 2cc72a43f43..a8e988057d4 100755 --- a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java +++ b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java @@ -8,6 +8,7 @@ import ai.vespa.llm.completion.Prompt; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.FileReference; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -16,11 +17,13 @@ import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.io.File; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -74,6 +77,14 @@ public class LLMSearcherTest { } @Test + public void testPromptTemplate() { + var templateFile = FileReference.mockFileReferenceForUnitTesting(new File("src/test/resources/llm.template.txt")); + var config = new LlmSearcherConfig.Builder().stream(false).promptTemplate(Optional.of(templateFile)).build(); + var searcher = createLLMSearcher(config, createLLMClient()); + assertEquals("Dogs are loyal and friendly.", getCompletion(runMockSearch(searcher, Map.of()))); + } + + @Test public void testPromptEvent() { var client = createLLMClient(); var searcher = createLLMSearcher(client); @@ -213,6 +224,8 @@ public class LLMSearcherTest { if (temperature.isPresent() && temperature.get() > 0.5) { answer = "Random text about ducks vs cats that makes no sense whatsoever."; } + } else if (prompt.asString().contains("dogs")) { + answer = "Dogs are loyal and friendly."; } var maxTokens = options.getInt("maxTokens"); if (maxTokens.isPresent()) { diff --git a/container-search/src/test/java/ai/vespa/search/llm/RAGSearcherTest.java b/container-search/src/test/java/ai/vespa/search/llm/RAGSearcherTest.java index d6b66b1a8c6..ae4a6db31b3 100755 --- a/container-search/src/test/java/ai/vespa/search/llm/RAGSearcherTest.java +++ b/container-search/src/test/java/ai/vespa/search/llm/RAGSearcherTest.java @@ -32,18 +32,21 @@ public class RAGSearcherTest { public void testRAGGeneration() { var eventStream = runRAGQuery(Map.of( "prompt", "why are ducks better than cats?", - "traceLevel", "1")); + "llm.fields", "title,content", + "llm.includePrompt", "true")); var events = eventStream.incoming().drain(); assertEquals(2, events.size()); // Generated prompt var promptEvent = (EventStream.Event) events.get(0); assertEquals("prompt", promptEvent.type()); - assertEquals("title: " + DOC1_TITLE + "\n" + - "content: " + DOC1_CONTENT + "\n\n" + - "title: " + DOC2_TITLE + "\n" + - "content: " + DOC2_CONTENT + "\n\n\n" + - "why are ducks better than cats?", promptEvent.toString()); + assertEquals("document [1]:\n" + + "title: " + DOC1_TITLE + "\n" + + "content: " + DOC1_CONTENT + "\n\n" + + "document [2]:\n" + + "title: " + DOC2_TITLE + "\n" + + "content: " + DOC2_CONTENT + "\n\n\n" + + "why are ducks better than cats?", promptEvent.toString()); // Generated completion var completionEvent = (EventStream.Event) events.get(1); @@ -55,14 +58,17 @@ public class RAGSearcherTest { public void testPromptGeneration() { var eventStream = runRAGQuery(Map.of( "query", "why are ducks better than cats?", - "prompt", "{context}\nGiven these documents, answer this query as concisely as possible: @query", + "llm.fields", "title,content", + "llm.prompt", "{context}\nGiven these documents, answer this query as concisely as possible: @query", "traceLevel", "1")); var events = eventStream.incoming().drain(); var promptEvent = (EventStream.Event) events.get(0); assertEquals("prompt", promptEvent.type()); - assertEquals("title: " + DOC1_TITLE + "\n" + + assertEquals("document [1]:\n" + + "title: " + DOC1_TITLE + "\n" + "content: " + DOC1_CONTENT + "\n\n" + + "document [2]:\n" + "title: " + DOC2_TITLE + "\n" + "content: " + DOC2_CONTENT + "\n\n\n" + "Given these documents, answer this query as concisely as possible: " + @@ -89,10 +95,13 @@ public class RAGSearcherTest { Hit hit1 = new Hit("1"); hit1.setField("title", DOC1_TITLE); hit1.setField("content", DOC1_CONTENT); + hit1.setField("matchfeatures", "..."); Hit hit2 = new Hit("2"); hit2.setField("title", DOC2_TITLE); hit2.setField("content", DOC2_CONTENT); + hit2.setField("summaryfeatures", "..."); + hit2.setField("rankfeatures", "..."); Result r = new Result(query); r.hits().add(hit1); @@ -115,7 +124,7 @@ public class RAGSearcherTest { return execution.search(query); } - private static Searcher createRAGSearcher(Map<String, LanguageModel> llms) { + static Searcher createRAGSearcher(Map<String, LanguageModel> llms) { var config = new LlmSearcherConfig.Builder().stream(false).build(); ComponentRegistry<LanguageModel> models = new ComponentRegistry<>(); llms.forEach((key, value) -> models.register(ComponentId.fromString(key), value)); diff --git a/container-search/src/test/java/ai/vespa/search/llm/RAGWithEventRendererTest.java b/container-search/src/test/java/ai/vespa/search/llm/RAGWithEventRendererTest.java new file mode 100644 index 00000000000..40aba0d5b6a --- /dev/null +++ b/container-search/src/test/java/ai/vespa/search/llm/RAGWithEventRendererTest.java @@ -0,0 +1,77 @@ +package ai.vespa.search.llm; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.search.Result; +import com.yahoo.search.rendering.EventRenderer; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.text.Utf8; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RAGWithEventRendererTest { + + @Test + public void testPromptAndHitsAreRendered() throws Exception { + var params = Map.of( + "query", "why are ducks better than cats?", + "llm.stream", "false", + "llm.includePrompt", "true", + "llm.includeHits", "true" + ); + var llm = LLMSearcherTest.createLLMClient(); + var searcher = RAGSearcherTest.createRAGSearcher(Map.of("mock", llm)); + var results = RAGSearcherTest.runMockSearch(searcher, params); + + var result = render(results); + + var promptEvent = extractEvent(result, "prompt"); + assertNotNull(promptEvent); + assertTrue(promptEvent.has("prompt")); + + var resultsEvent = extractEvent(result, "hits"); + assertNotNull(resultsEvent); + assertTrue(resultsEvent.has("root")); + assertEquals(2, resultsEvent.get("root").get("children").size()); + } + + private JsonNode extractEvent(String result, String eventName) throws JsonProcessingException { + var lines = result.split("\n"); + for (int i = 0; i < lines.length; i++) { + if (lines[i].startsWith("event: " + eventName)) { + var data = lines[i + 1].substring("data: ".length()).trim(); + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.readTree(data); + } + } + return null; + } + + private String render(Result r) throws InterruptedException, ExecutionException { + var execution = new Execution(Execution.Context.createContextStub()); + return render(execution, r); + } + + private String render(Execution execution, Result r) throws ExecutionException, InterruptedException { + var renderer = new EventRenderer(); + try { + renderer.init(); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + CompletableFuture<Boolean> f = renderer.renderResponse(bs, r, execution, null); + assertTrue(f.get()); + return Utf8.toString(bs.toByteArray()); + } finally { + renderer.deconstruct(); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java index cafb79d8542..ca7d5bf6999 100644 --- a/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java @@ -740,13 +740,6 @@ public class JSONStringTestCase { Inspector value5 = new JSONString("{\"foo\":1}").inspect(); Inspector value6 = new JSONString("[1,2,3]").inspect(); - System.out.println("1: " + value1); - System.out.println("2: " + value2); - System.out.println("3: " + value3); - System.out.println("4: " + value4); - System.out.println("5: " + value5); - System.out.println("6: " + value6); - assertEquals(Type.STRING, value1.type()); assertEquals("", value1.asString()); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/InItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/InItemTestCase.java new file mode 100644 index 00000000000..a4e225d29d3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/InItemTestCase.java @@ -0,0 +1,68 @@ +package com.yahoo.prelude.query; + +import com.yahoo.search.Query; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bratseth + */ +public class InItemTestCase { + + @Test + void testNumericInItemTracing() { + Query q = new Query(); + q.getTrace().setLevel(7); + q.getModel().getQueryTree().setRoot(new NumericInItem("default", Set.of(3L, 5L))); + q.trace("Trace 1", true, 0); + List<String> traces = new ArrayList<>(); + for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) + traces.add(trace); + + String expected = """ +Trace 1: [ +select * from sources * where default in (3, 5) +NUMERIC_IN{ + INT[index="" origin=null]{ + "3" + } + INT[index="" origin=null]{ + "5" + } +} + +]"""; + assertEquals(expected, traces.get(1)); + } + + @Test + void testStringItemTracing() { + Query q = new Query(); + q.getTrace().setLevel(7); + q.getModel().getQueryTree().setRoot(new StringInItem("default", Set.of("foo", "bar"))); + q.trace("Trace 1", true, 0); + List<String> traces = new ArrayList<>(); + for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) + traces.add(trace); + String expected = """ +Trace 1: [ +select * from sources * where default in ("bar", "foo") +STRING_IN{ + WORD[fromSegmented=false index="" origin=null segmentIndex=0 stemmed=false words=true]{ + "bar" + } + WORD[fromSegmented=false index="" origin=null segmentIndex=0 stemmed=false words=true]{ + "foo" + } +} + +]"""; + assertEquals(expected, traces.get(1)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/ItemHelperTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/ItemHelperTestCase.java index 5b20eda4344..5e0da17c186 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/ItemHelperTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/ItemHelperTestCase.java @@ -34,7 +34,6 @@ public class ItemHelperTestCase { ItemHelper helper = new ItemHelper(); Query q = new Query("/?query=" + enc("a b c \"d e\" -f")); List<IndexedItem> l = new ArrayList<>(); - System.out.println(q.getModel()); helper.getPositiveTerms(q.getModel().getQueryTree().getRoot(), l); assertEquals(4, l.size()); boolean a = false; diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java index 0516a8a227a..a662c445bc7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java @@ -22,6 +22,15 @@ public class QueryCanonicalizerTestCase { } @Test + void testNoCanonicalizationWithWhereTrue() { + CompositeItem root = new AndItem(); + + root.addItem(new TrueItem()); + root.addItem(new WordItem("word")); + assertCanonicalized("AND TRUE word", null, root); + } + + @Test void testSingleLevelSingleItemNonReducibleComposite() { CompositeItem root = new WeakAndItem(); diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java index 8ced5c81372..12110e2d2f3 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -220,7 +220,7 @@ public class QueryTestCase { void testNegativeHitValue() { assertQueryError( "?query=test&hits=-1", - "Could not set 'hits' to '-1': Must be a positive number"); + "Could not set 'hits' to '-1': 'hits' must be a positive number, not -1"); } @Test @@ -241,7 +241,7 @@ public class QueryTestCase { void testNegativeOffsetValue() { assertQueryError( "?query=test&offset=-1", - "Could not set 'offset' to '-1': Must be a positive number"); + "Could not set 'offset' to '-1': 'offset' must be a positive number, not -1"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java index 4cd453746bb..2f36d174ad4 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java @@ -88,7 +88,7 @@ public class TopKEstimatorTest { @Test void requireThatLargeKAreSane() { - System.out.println(dumpProbability(10, 0.05)); + // System.out.println(dumpProbability(10, 0.05)); TopKEstimator idealEstimator = new TopKEstimator(30, 0.9999); TopKEstimator skewedEstimator = new TopKEstimator(30, 0.9999, 0.05); int [] K = {10, 20, 40, 80, 100, 200, 400, 800, 1000, 2000, 4000, 8000, 10000, 20000, 40000, 80000, 100000}; diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java index 780066d0afe..fff99258309 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java @@ -185,7 +185,7 @@ public class GroupingExecutorTestCase { Group grp = req.getResultGroup(exec.search(query)); assertEquals(1, grp.size()); Hit hit = grp.get(0); - assertTrue(hit instanceof GroupList); + assertInstanceOf(GroupList.class, hit); GroupList lst = (GroupList) hit; assertEquals(3, lst.size()); assertNotNull(hit = lst.get("group:string:uniqueA")); @@ -218,7 +218,7 @@ public class GroupingExecutorTestCase { Group grp = req.getResultGroup(exec.search(query)); assertEquals(1, grp.size()); Hit hit = grp.get(0); - assertTrue(hit instanceof GroupList); + assertInstanceOf(GroupList.class, hit); GroupList lst = (GroupList) hit; assertEquals(1, lst.size()); assertNotNull(hit = lst.get("group:string:expected")); @@ -399,7 +399,7 @@ public class GroupingExecutorTestCase { assertNotNull(lst); assertEquals(1, lst.size()); Hit hit = lst.get(0); - assertTrue(hit instanceof FastHit); + assertInstanceOf(FastHit.class, hit); assertEquals(1, ((FastHit) hit).getPartId()); assertEquals(gid1, ((FastHit) hit).getGlobalId()); @@ -407,7 +407,7 @@ public class GroupingExecutorTestCase { assertNotNull(lst); assertEquals(1, lst.size()); hit = lst.get(0); - assertTrue(hit instanceof FastHit); + assertInstanceOf(FastHit.class, hit); assertEquals(4, ((FastHit) hit).getPartId()); assertEquals(gid2, ((FastHit) hit).getGlobalId()); } @@ -439,7 +439,7 @@ public class GroupingExecutorTestCase { exec.fill(res); Hit hit = ((HitList) ((Group) ((GroupList) req.getResultGroup(res).get(0)).get(0)).get(0)).get(0); - assertTrue(hit instanceof FastHit); + assertInstanceOf(FastHit.class, hit); assertTrue(hit.isFilled(null)); } @@ -577,7 +577,7 @@ public class GroupingExecutorTestCase { Group group = request.getResultGroup(result); assertEquals(1, group.size()); Hit hit = group.get(0); - assertTrue(hit instanceof GroupList); + assertInstanceOf(GroupList.class, hit); GroupList list = (GroupList) hit; assertEquals(4, list.size()); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java index 357bba0c7b1..7cbd5747d2d 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java @@ -25,7 +25,6 @@ public class DumpToolTestCase { @Test void testNoDimensionValues() { - System.out.println(new DumpTool().resolveAndDump("multiprofile1", profileDir)); assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir).contains("a=general-a\n")); } diff --git a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java index 84d353ec316..eb17a1560e0 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java @@ -5,8 +5,10 @@ import com.yahoo.search.Query; import com.yahoo.search.query.Sorting; import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Arne Bergene Fossaa @@ -86,4 +88,27 @@ public class RankingTestCase { assertEquals("(0,0,10,0,10,5,20,5)", query.getRanking().getProperties().get("distanceToPath(gps_position).path").get(0)); } + @Test + void testSecondPhaseRankScoreDropLimit() { + var query = new Query("?query=test&ranking.secondPhase.rankScoreDropLimit=17.5"); + var ranking = query.getRanking(); + assertEquals(17.5, ranking.getSecondPhase().getRankScoreDropLimit()); + ranking.prepare(); + assertEquals("17.5", ranking.getProperties().get("vespa.hitcollector.secondphase.rankscoredroplimit").get(0)); + } + + @Test + void testSignificanceUseModel() { + var query = new Query("?query=test"); + var ranking = query.getRanking(); + assertTrue(ranking.getSignificance().getUseModel().isEmpty()); + + var query1 = new Query("?query=test&ranking.significance.useModel=true"); + var ranking1 = query1.getRanking(); + assertEquals(true, ranking1.getSignificance().getUseModel().get()); + + var query2 = new Query("?query=test&ranking.significance.useModel=false"); + var ranking2 = query2.getRanking(); + assertEquals(false, ranking2.getSignificance().getUseModel().get()); + } } diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java index 52f5fd0cafb..7b91a5d3c25 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java @@ -23,6 +23,7 @@ import static com.yahoo.search.querytransform.WeakAndReplacementSearcher.WAND_HI import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; public class WeakAndReplacementSearcherTestCase { @@ -57,7 +58,7 @@ public class WeakAndReplacementSearcherTestCase { Result result = buildExec().search(query); Item root = TestUtils.getQueryTreeRoot(result); assertFalse(orItemsExist(root)); - assertTrue(root instanceof WeakAndItem); + assertInstanceOf(WeakAndItem.class, root); assertEquals(N, ((WeakAndItem) root).getN()); } @@ -103,24 +104,22 @@ public class WeakAndReplacementSearcherTestCase { if (item1 != item2) { return false; } - if (!(item1 instanceof CompositeItem)) { + if (!(item1 instanceof CompositeItem compositeItem1)) { return true; } - CompositeItem compositeItem1 = (CompositeItem) item1; CompositeItem compositeItem2 = (CompositeItem) item2; return IntStream.range(0, compositeItem1.getItemCount()) .allMatch(i -> deepEquals(compositeItem1.getItem(i), compositeItem2.getItem(i))); } private boolean orItemsExist(Item item) { - if (!(item instanceof CompositeItem)) { + if (!(item instanceof CompositeItem compositeItem)) { return false; } if (item instanceof OrItem) { return true; } - CompositeItem compositeItem = (CompositeItem) item; return compositeItem.items().stream().anyMatch(this::orItemsExist); } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/EventRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/EventRendererTestCase.java index c0a677b2094..f6f6f40bdae 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/EventRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/EventRendererTestCase.java @@ -141,7 +141,7 @@ public class EventRendererTestCase { }); assertFalse(future.isDone()); result = render(new Result(new Query(), newHitGroup(tokenStream, "token_stream"))); - assertTrue(future.isDone()); // Renderer waits for async completion + future.join(); // Renderer waits for async completion } finally { executor.shutdownNow(); @@ -232,7 +232,7 @@ public class EventRendererTestCase { event: end """; - assertEquals(expected, result); // Todo: support other types of data such as search results (hits), timing and trace + assertEquals(expected, result); } static HitGroup newHitGroup(EventStream eventStream, String id) { diff --git a/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java new file mode 100644 index 00000000000..cdfbdd62765 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java @@ -0,0 +1,81 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.searchers; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.TrueItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordItem; +import org.junit.jupiter.api.Test; + +import static com.yahoo.search.searchers.OpportunisticWeakAndSearcher.targetHits; +import static com.yahoo.search.searchers.OpportunisticWeakAndSearcher.weakAnd2AndRecurse; +import static com.yahoo.search.searchers.OpportunisticWeakAndSearcher.adjustWeakAndHeap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +public class OpportunisticWeakAndSearcherTestCase { + private static Item buildQueryItem(CompositeItem root, CompositeItem injectAtLevel2) { + root.addItem(new WordItem("text")); + injectAtLevel2.addItem(new WordItem("a")); + injectAtLevel2.addItem(new WordItem("b")); + root.addItem(injectAtLevel2); + return root; + } + + private static CompositeItem addItem(CompositeItem composite, Item item) { + composite.addItem(item); + return composite; + } + + @Test + public void requireThatWeakAndIsDetected() { + assertEquals(-1, targetHits(new OrItem())); + assertEquals(-1, targetHits(new WeakAndItem(33))); + assertEquals(-1, targetHits(addItem(new WeakAndItem(33), new TrueItem()))); + assertEquals(33, targetHits(addItem(addItem(new WeakAndItem(33), new TrueItem()), new TrueItem()))); + assertEquals(77, targetHits(buildQueryItem(new OrItem(), new WeakAndItem(77)))); + assertEquals(77, targetHits(buildQueryItem(new AndItem(), new WeakAndItem(77)))); + assertEquals(-1, targetHits(buildQueryItem(new OrItem(), new AndItem()))); + } + + @Test + public void requireThatWeakAndIsReplacedWithAnd() { + assertEquals(buildQueryItem(new OrItem(), new AndItem()), + weakAnd2AndRecurse(buildQueryItem(new OrItem(), new WeakAndItem()))); + assertEquals(buildQueryItem(new AndItem(), new AndItem()), + weakAnd2AndRecurse(buildQueryItem(new AndItem(), new WeakAndItem()))); + } + + private static WeakAndItem try2Adjust(WeakAndItem item, int hits) { + adjustWeakAndHeap(item, hits); + return item; + } + + private static WeakAndItem try2Adjust(WeakAndItem item, CompositeItem parent, int hits) { + parent.addItem(item); + adjustWeakAndHeap(parent, hits); + return item; + } + + @Test + public void requireThatDefaultWeakAndHeapIsAdjustedUpToHits() { + assertEquals(1000, try2Adjust(new WeakAndItem(), 1000).getN()); + assertFalse(try2Adjust(new WeakAndItem(), 10).nIsExplicit()); + + assertEquals(1000, try2Adjust(new WeakAndItem(), new OrItem(), 1000).getN()); + assertFalse(try2Adjust(new WeakAndItem(), new OrItem(), 10).nIsExplicit()); + } + @Test + public void requireThatNonDefaultWeakAndHeapIsNotAdjustedUpToHits() { + assertEquals(33, try2Adjust(new WeakAndItem(33), 1000).getN()); + assertEquals(33, try2Adjust(new WeakAndItem(33), 11).getN()); + assertEquals(WeakAndItem.defaultN, try2Adjust(new WeakAndItem(WeakAndItem.defaultN), 1000).getN()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java index eaa66755608..be68c87efb3 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -79,6 +79,40 @@ public class SignificanceSearcherTest { WordItem w0 = (WordItem) root.getItem(0); assertEquals(helloSignificanceValue, w0.getSignificance()); } + + @Test + void testSignificanceValueOnSimpleQueryWithRankingOverride() { + Query q1 = new Query("?query=hello&ranking.significance.useModel=true"); + q1.getRanking().setProfile("significance-ranking"); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("hello", true); + root.addItem(tmp); + + q1.getModel().getQueryTree().setRoot(root); + + SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get(); + var helloFrequency = model.documentFrequency("hello"); + var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency()); + Result r = createExecution(searcher).search(q1); + + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + assertEquals(helloSignificanceValue, w0.getSignificance()); + + Query q2 = new Query("?query=hello&ranking.significance.useModel=false"); + q2.getRanking().setProfile("significance-ranking"); + root = new AndItem(); + tmp = new WordItem("hello", true); + root.addItem(tmp); + + q2.getModel().getQueryTree().setRoot(root); + Result r2 = createExecution(searcher).search(q2); + root = (AndItem) r2.getQuery().getModel().getQueryTree().getRoot(); + WordItem w1 = (WordItem) root.getItem(0); + assertEquals(0.0, w1.getSignificance()); + } + @Test void testSignificanceValueOnSimpleANDQuery() { diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 6a310180eab..259a79b095b 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -306,6 +306,24 @@ public class QueryTestCase { assertEquals("Profile: myProfile", q.properties().get("myField")); } + /** Select is special handled due to the strange idea to also use it to contain subproperties with JSON. */ + @Test + void testQueryProfileWithSelect() { + String grouping = "all(group(customerid) each(output(count())))"; + + { // select in the request + QueryProfile profile = new QueryProfile("myProfile"); + Query q = new Query(QueryTestCase.httpEncode("/search?query=macbook&queryProfile=myProfile&select=" + grouping), profile.compile(null)); + assertEquals(grouping, q.getSelect().getGroupingExpressionString()); + } + { // select in the query profile + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("select", grouping, null); + Query q = new Query(QueryTestCase.httpEncode("/search?query=macbook&queryProfile=myProfile"), profile.compile(null)); + assertEquals(grouping, q.getSelect().getGroupingExpressionString()); + } + } + @Test void testQueryProfileSourceAccess() { QueryProfile profile = new QueryProfile("myProfile"); diff --git a/container-search/src/test/resources/llm.template.txt b/container-search/src/test/resources/llm.template.txt new file mode 100644 index 00000000000..380fb344d69 --- /dev/null +++ b/container-search/src/test/resources/llm.template.txt @@ -0,0 +1 @@ +Why are dogs better than cats?
\ No newline at end of file diff --git a/defaults/src/vespa/CMakeLists.txt b/defaults/src/vespa/CMakeLists.txt index e2bfbf3264b..b47936c6ac1 100644 --- a/defaults/src/vespa/CMakeLists.txt +++ b/defaults/src/vespa/CMakeLists.txt @@ -33,4 +33,5 @@ endfunction() vespa_configure_config_h() -install(FILES defaults.h config.h DESTINATION include/vespa) +install(FILES defaults.h DESTINATION include/vespa) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/config.h DESTINATION include/vespa) diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 1633ef68dd6..f172b09b7fe 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -33,8 +33,8 @@ <!-- DO NOT UPGRADE THESE TO A NEW MAJOR VERSION WITHOUT CHECKING FOR BINARY COMPATIBILITY --> <aopalliance.vespa.version>1.0</aopalliance.vespa.version> - <error-prone-annotations.vespa.version>2.27.1</error-prone-annotations.vespa.version> - <guava.vespa.version>33.2.0-jre</guava.vespa.version> + <error-prone-annotations.vespa.version>2.28.0</error-prone-annotations.vespa.version> + <guava.vespa.version>33.2.1-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> <j2objc-annotations.vespa.version>3.0.0</j2objc-annotations.vespa.version> <jackson2.vespa.version>2.17.1</jackson2.vespa.version> @@ -65,11 +65,11 @@ <apache.httpcore5.vespa.version>5.2.4</apache.httpcore5.vespa.version> <apiguardian.vespa.version>1.1.2</apiguardian.vespa.version> <asm.vespa.version>9.7</asm.vespa.version> - <assertj.vespa.version>3.25.3</assertj.vespa.version> + <assertj.vespa.version>3.26.0</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.726</aws-sdk.vespa.version> - <athenz.vespa.version>1.11.59</athenz.vespa.version> + <aws-sdk.vespa.version>1.12.748</aws-sdk.vespa.version> + <athenz.vespa.version>1.11.60</athenz.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -79,7 +79,7 @@ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> <bouncycastle.vespa.version>1.78.1</bouncycastle.vespa.version> - <byte-buddy.vespa.version>1.14.15</byte-buddy.vespa.version> + <byte-buddy.vespa.version>1.14.17</byte-buddy.vespa.version> <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version> <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version> <commons-codec.vespa.version>1.17.0</commons-codec.vespa.version> @@ -90,19 +90,19 @@ <commons-lang3.vespa.version>3.14.0</commons-lang3.vespa.version> <commons-logging.vespa.version>1.3.2</commons-logging.vespa.version> <!-- Bindings exported by jdisc through jcl-over-slf4j. --> <commons.math3.vespa.version>3.6.1</commons.math3.vespa.version> - <commons-compress.vespa.version>1.26.1</commons-compress.vespa.version> - <commons-cli.vespa.version>1.7.0</commons-cli.vespa.version> - <curator.vespa.version>5.6.0</curator.vespa.version> - <dropwizard.metrics.vespa.version>4.2.25</dropwizard.metrics.vespa.version> <!-- ZK 3.9.1 requires this --> + <commons-compress.vespa.version>1.26.2</commons-compress.vespa.version> + <commons-cli.vespa.version>1.8.0</commons-cli.vespa.version> + <curator.vespa.version>5.7.0</curator.vespa.version> + <dropwizard.metrics.vespa.version>4.2.26</dropwizard.metrics.vespa.version> <!-- ZK 3.9.1 requires this --> <eclipse-angus.vespa.version>2.0.2</eclipse-angus.vespa.version> <eclipse-collections.vespa.version>11.1.0</eclipse-collections.vespa.version> - <eclipse-sisu.vespa.version>0.9.0.M2</eclipse-sisu.vespa.version> + <eclipse-sisu.vespa.version>0.9.0.M3</eclipse-sisu.vespa.version> <failureaccess.vespa.version>1.0.2</failureaccess.vespa.version> <felix.vespa.version>7.0.5</felix.vespa.version> <felix.log.vespa.version>1.3.0</felix.log.vespa.version> <findbugs.vespa.version>3.0.2</findbugs.vespa.version> <!-- Should be kept in sync with guava --> <hamcrest.vespa.version>2.2</hamcrest.vespa.version> - <hdrhistogram.vespa.version>2.2.1</hdrhistogram.vespa.version> + <hdrhistogram.vespa.version>2.2.2</hdrhistogram.vespa.version> <huggingface.vespa.version>0.28.0</huggingface.vespa.version> <icu4j.vespa.version>75.1</icu4j.vespa.version> <java-jjwt.vespa.version>0.11.5</java-jjwt.vespa.version> @@ -117,18 +117,18 @@ <junit.vespa.version>5.10.2</junit.vespa.version> <junit.platform.vespa.version>1.10.2</junit.platform.vespa.version> <junit4.vespa.version>4.13.2</junit4.vespa.version> - <kherud.llama.vespa.version>3.0.2</kherud.llama.vespa.version> + <kherud.llama.vespa.version>3.2.1</kherud.llama.vespa.version> <luben.zstd.vespa.version>1.5.6-3</luben.zstd.vespa.version> - <lucene.vespa.version>9.10.0</lucene.vespa.version> + <lucene.vespa.version>9.11.0</lucene.vespa.version> <maven-archiver.vespa.version>3.6.2</maven-archiver.vespa.version> <maven-wagon.vespa.version>3.5.3</maven-wagon.vespa.version> - <maven-xml-impl.vespa.version>4.0.0-alpha-13</maven-xml-impl.vespa.version> + <maven-xml-impl.vespa.version>4.0.0-beta-3</maven-xml-impl.vespa.version> <mimepull.vespa.version>1.10.0</mimepull.vespa.version> <mockito.vespa.version>5.12.0</mockito.vespa.version> <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> - <netty.vespa.version>4.1.109.Final</netty.vespa.version> + <netty.vespa.version>4.1.111.Final</netty.vespa.version> <netty-tcnative.vespa.version>2.0.65.Final</netty-tcnative.vespa.version> - <onnxruntime.vespa.version>1.17.3</onnxruntime.vespa.version> + <onnxruntime.vespa.version>1.18.0</onnxruntime.vespa.version> <opennlp.vespa.version>2.3.3</opennlp.vespa.version> <opentest4j.vespa.version>1.3.0</opentest4j.vespa.version> <org.json.vespa.version>20240303</org.json.vespa.version> @@ -138,16 +138,17 @@ <plexus-interpolation.vespa.version>1.27</plexus-interpolation.vespa.version> <plexus-io.vespa.version>3.4.2</plexus-io.vespa.version> <plexus-utils.vespa.version>4.0.1</plexus-utils.vespa.version> - <plexus-xml.vespa.version>4.0.3</plexus-xml.vespa.version> + <plexus-xml.vespa.version>4.0.4</plexus-xml.vespa.version> + <plexus-classworlds.vespa.version>2.8.0</plexus-classworlds.vespa.version> <protobuf.vespa.version>3.25.3</protobuf.vespa.version> <questdb.vespa.version>7.4.2</questdb.vespa.version> <spifly.vespa.version>1.3.7</spifly.vespa.version> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> - <surefire.vespa.version>3.2.5</surefire.vespa.version> + <surefire.vespa.version>3.3.0</surefire.vespa.version> <velocity.vespa.version>2.3</velocity.vespa.version> <velocity.tools.vespa.version>3.1</velocity.tools.vespa.version> - <wiremock.vespa.version>3.5.4</wiremock.vespa.version> - <woodstox.vespa.version>6.6.2</woodstox.vespa.version> + <wiremock.vespa.version>3.7.0</wiremock.vespa.version> + <woodstox.vespa.version>7.0.0</woodstox.vespa.version> <stax2-api.vespa.version>4.2.2</stax2-api.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> <zero-allocation-hashing.vespa.version>0.16</zero-allocation-hashing.vespa.version> @@ -169,20 +170,22 @@ <maven-assembly-plugin.vespa.version>3.7.1</maven-assembly-plugin.vespa.version> <maven-bundle-plugin.vespa.version>5.1.9</maven-bundle-plugin.vespa.version> <maven-compiler-plugin.vespa.version>3.13.0</maven-compiler-plugin.vespa.version> - <maven-core.vespa.version>3.9.6</maven-core.vespa.version> - <maven-dependency-plugin.vespa.version>3.6.1</maven-dependency-plugin.vespa.version> + <maven-core.vespa.version>3.9.8</maven-core.vespa.version> + <maven-dependency-plugin.vespa.version>3.7.1</maven-dependency-plugin.vespa.version> <maven-deploy-plugin.vespa.version>3.1.2</maven-deploy-plugin.vespa.version> - <maven-enforcer-plugin.vespa.version>3.4.1</maven-enforcer-plugin.vespa.version> - <maven-failsafe-plugin.vespa.version>3.2.5</maven-failsafe-plugin.vespa.version> + <maven-enforcer-plugin.vespa.version>3.5.0</maven-enforcer-plugin.vespa.version> + <maven-failsafe-plugin.vespa.version>3.3.0</maven-failsafe-plugin.vespa.version> <maven-gpg-plugin.vespa.version>3.2.4</maven-gpg-plugin.vespa.version> <maven-install-plugin.vespa.version>3.1.2</maven-install-plugin.vespa.version> - <maven-jar-plugin.vespa.version>3.4.1</maven-jar-plugin.vespa.version> - <maven-javadoc-plugin.vespa.version>3.6.3</maven-javadoc-plugin.vespa.version> + <maven-jar-plugin.vespa.version>3.4.2</maven-jar-plugin.vespa.version> + <maven-javadoc-plugin.vespa.version>3.7.0</maven-javadoc-plugin.vespa.version> <maven-plugin-api.vespa.version>${maven-core.vespa.version}</maven-plugin-api.vespa.version> - <maven-plugin-tools.vespa.version>3.13.0</maven-plugin-tools.vespa.version> + <maven-plugin-tools.vespa.version>3.13.1</maven-plugin-tools.vespa.version> <maven-resources-plugin.vespa.version>3.3.1</maven-resources-plugin.vespa.version> <maven-resolver.vespa.version>1.9.20</maven-resolver.vespa.version> <maven-shade-plugin.vespa.version>3.5.3</maven-shade-plugin.vespa.version> + <maven-shared-utils.vespa.version>3.4.2</maven-shared-utils.vespa.version> + <maven-dependency-tree.vespa.version>3.3.0</maven-dependency-tree.vespa.version> <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.1</maven-source-plugin.vespa.version> <properties-maven-plugin.vespa.version>1.2.1</properties-maven-plugin.vespa.version> diff --git a/dist/vespa.spec b/dist/vespa.spec index 2e6e3c042d2..9e617fb8acd 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -33,7 +33,7 @@ %define _defattr_is_vespa_vespa 0 %define _command_cmake cmake3 %global _vespa_abseil_cpp_version 20240116.1 -%global _vespa_build_depencencies_version 1.3.2 +%global _vespa_build_depencencies_version 1.3.3 %global _vespa_gtest_version 1.14.0 %global _vespa_protobuf_version 5.26.1 %global _vespa_openblas_version 0.3.27 @@ -164,7 +164,6 @@ Requires: openssl-libs %endif Requires: vespa-lz4 >= 1.9.4-1 Requires: vespa-libzstd >= 1.5.6-1 -Requires: vespa-openblas >= %{_vespa_openblas_version} %if 0%{?amzn2023} Requires: vespa-re2 = 20210801 %else @@ -201,8 +200,9 @@ Requires: vespa-protobuf = %{_vespa_protobuf_version} Requires: vespa-protobuf = %{_vespa_protobuf_version} Requires: llvm-libs %endif -Requires: vespa-onnxruntime = 1.17.3 -Requires: vespa-jllama >= 3.0.1 +Requires: vespa-onnxruntime = 1.18.0 +Requires: vespa-jllama = 3.2.1 +Requires: vespa-openblas >= %{_vespa_openblas_version} %description libs @@ -570,7 +570,7 @@ fi %endif %dir %{_prefix} %dir %{_prefix}/lib64 -%{_prefix}/lib64/libfnet.so +%{_prefix}/lib64/libvespa_fnet.so %{_prefix}/lib64/libvespadefaults.so %{_prefix}/lib64/libvespalib.so %{_prefix}/lib64/libvespalog.so @@ -581,7 +581,7 @@ fi %endif %dir %{_prefix} %{_prefix}/lib64 -%exclude %{_prefix}/lib64/libfnet.so +%exclude %{_prefix}/lib64/libvespa_fnet.so %exclude %{_prefix}/lib64/libvespadefaults.so %exclude %{_prefix}/lib64/libvespalib.so %exclude %{_prefix}/lib64/libvespalog.so diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index 725eea0b5b8..7c170b85e46 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_define_module( DEPENDS vespalog vespalib - config_cloudconfig + vespa_config vespaeval LIBS diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java index c6eccdacf26..465d7da5e8e 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -6,7 +6,9 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.PositionDataType; import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.TokenBuffer; import com.yahoo.document.update.ValueUpdate; @@ -41,6 +43,11 @@ public class SingleValueReader { } public static FieldValue readSingleValue(TokenBuffer buffer, DataType expectedType, boolean ignoreUndefinedFields) { + if (expectedType instanceof TensorDataType) { + FieldValue fieldValue = expectedType.createFieldValue(); + TensorReader.fillTensor(buffer, (TensorFieldValue) fieldValue); + return fieldValue; + } if (buffer.current().isScalarValue()) { return readAtomic(buffer.currentText(), expectedType); } else { diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index 1fd4029b1a5..a8166e9fa5e 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -37,6 +37,18 @@ public class TensorReader { // MUST be kept in sync with com.yahoo.tensor.serialization.JsonFormat.decode in vespajlib static void fillTensor(TokenBuffer buffer, TensorFieldValue tensorFieldValue) { Tensor.Builder builder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); + if (buffer.current() == JsonToken.VALUE_STRING + && builder instanceof IndexedTensor.BoundBuilder indexedBuilder) + { + double[] decoded = decodeHexString(buffer.currentText(), builder.type().valueType()); + if (decoded.length == 0) + throw new IllegalArgumentException("Bad string input for tensor with type " + builder.type()); + for (int i = 0; i < decoded.length; i++) { + indexedBuilder.cellByDirectIndex(i, decoded[i]); + } + tensorFieldValue.assign(builder.build()); + return; + } expectOneOf(buffer.current(), JsonToken.START_OBJECT, JsonToken.START_ARRAY); int initNesting = buffer.nesting(); while (true) { @@ -199,7 +211,7 @@ public class TensorReader { readTensorCells(buffer, builder); else if ( ! hasMapped) readTensorValues(buffer, builder); - else if (hasMapped && hasIndexed) + else if (hasIndexed) readTensorBlocks(buffer, builder); else readTensorCells(buffer, builder); diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index e72d3720024..a61ad87ca8d 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -175,6 +175,8 @@ public class JsonReaderTestCase { new TensorDataType(new TensorType.Builder().indexed("x", 2).indexed("y", 3).build()))); x.addField(new Field("dense_int8_tensor", new TensorDataType(TensorType.fromSpec("tensor<int8>(x[2],y[3])")))); + x.addField(new Field("dense_float_tensor", + new TensorDataType(TensorType.fromSpec("tensor<float>(y[3])")))); x.addField(new Field("dense_unbound_tensor", new TensorDataType(new TensorType.Builder().indexed("x").indexed("y").build()))); x.addField(new Field("mixed_tensor", @@ -1780,7 +1782,7 @@ public class JsonReaderTestCase { "remove": "id:unittest:smoke::whee", "what is love": "baby, do not hurt me... much } - ]"""; + ]"""; // " new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); } @@ -1996,6 +1998,30 @@ public class JsonReaderTestCase { "values": "020304050607" }""", "dense_int8_tensor"), "dense_int8_tensor"); assertTrue(tensor instanceof IndexedTensor); // this matters for performance + tensor = assertTensorField(expected, + createPutWithTensor(""" + "020304050607" + """, "dense_int8_tensor"), "dense_int8_tensor"); + assertTrue(tensor instanceof IndexedTensor); // this matters for performance + builder = Tensor.Builder.of(TensorType.fromSpec("tensor<float>(y[3])")); + builder.cell().label("y", 0).value(42.0); + builder.cell().label("y", 1).value(-0.125); + builder.cell().label("y", 2).value(Double.POSITIVE_INFINITY); + expected = builder.build(); + tensor = assertTensorField(expected, + createPutWithTensor(""" + "42280000be0000007f800000" + """, "dense_float_tensor"), "dense_float_tensor"); + try { + assertTensorField(expected, + createPutWithTensor(""" + "" + """, "dense_int8_tensor"), "dense_int8_tensor"); + } + catch (IllegalArgumentException e) { + assertTrue(Exceptions.toMessageString(e).contains( + "Bad string input for tensor with type tensor<int8>(x[2],y[3])")); + } } @Test @@ -2018,6 +2044,13 @@ public class JsonReaderTestCase { """; var put = createPutWithTensor(inputJson(mixedJson), "mixed_bfloat16_tensor"); Tensor tensor = assertTensorField(expected, put, "mixed_bfloat16_tensor"); + mixedJson = """ + { + "blocks":{"foo":"400040404080", "bar":"40A040C040E0"} + } + """; + put = createPutWithTensor(inputJson(mixedJson), "mixed_bfloat16_tensor"); + tensor = assertTensorField(expected, put, "mixed_bfloat16_tensor"); } /** Tests parsing of various tensor values set at the root, i.e. no 'cells', 'blocks' or 'values' */ diff --git a/document/src/tests/CMakeLists.txt b/document/src/tests/CMakeLists.txt index 71be631319b..181bf138ecf 100644 --- a/document/src/tests/CMakeLists.txt +++ b/document/src/tests/CMakeLists.txt @@ -31,7 +31,7 @@ vespa_add_executable(document_gtest_runner_app TEST testxml.cpp weightedsetfieldvaluetest.cpp DEPENDS - document + vespa_document GTest::GTest ) diff --git a/document/src/tests/annotation/CMakeLists.txt b/document/src/tests/annotation/CMakeLists.txt index ddb23f4773d..fef08928537 100644 --- a/document/src/tests/annotation/CMakeLists.txt +++ b/document/src/tests/annotation/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(document_annotation_test_app TEST SOURCES annotation_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_annotation_test_app COMMAND document_annotation_test_app) diff --git a/document/src/tests/annotation/annotation_test.cpp b/document/src/tests/annotation/annotation_test.cpp index d95e5c7d1b9..c5635b9f4f9 100644 --- a/document/src/tests/annotation/annotation_test.cpp +++ b/document/src/tests/annotation/annotation_test.cpp @@ -17,7 +17,7 @@ #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/doublefieldvalue.h> #include <vespa/document/fieldvalue/structfieldvalue.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <memory> using std::unique_ptr; diff --git a/document/src/tests/base/CMakeLists.txt b/document/src/tests/base/CMakeLists.txt index 99c0d5ac661..ce19dc9b9ce 100644 --- a/document/src/tests/base/CMakeLists.txt +++ b/document/src/tests/base/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(document_documentid_test_app TEST SOURCES documentid_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_documentid_test_app COMMAND document_documentid_test_app) vespa_add_executable(document_documentid_benchmark_app SOURCES documentid_benchmark.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_documentid_benchmark_app COMMAND document_documentid_benchmark_app BENCHMARK) diff --git a/document/src/tests/base/documentid_test.cpp b/document/src/tests/base/documentid_test.cpp index bf61440af52..66632fc1bf6 100644 --- a/document/src/tests/base/documentid_test.cpp +++ b/document/src/tests/base/documentid_test.cpp @@ -3,7 +3,7 @@ #include <vespa/document/base/documentid.h> #include <vespa/document/base/idstringexception.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using vespalib::string; diff --git a/document/src/tests/datatype/CMakeLists.txt b/document/src/tests/datatype/CMakeLists.txt index d392a4b4b70..0822c6f28f7 100644 --- a/document/src/tests/datatype/CMakeLists.txt +++ b/document/src/tests/datatype/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(document_datatype_test_app TEST SOURCES datatype_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_datatype_test_app COMMAND document_datatype_test_app) @@ -11,6 +11,6 @@ vespa_add_executable(document_referencedatatype_test_app TEST SOURCES referencedatatype_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_referencedatatype_test_app COMMAND document_referencedatatype_test_app) diff --git a/document/src/tests/datatype/datatype_test.cpp b/document/src/tests/datatype/datatype_test.cpp index f81a6be6768..1362afe8dbc 100644 --- a/document/src/tests/datatype/datatype_test.cpp +++ b/document/src/tests/datatype/datatype_test.cpp @@ -7,7 +7,7 @@ #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/eval/eval/value_type.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> using namespace document; diff --git a/document/src/tests/datatype/referencedatatype_test.cpp b/document/src/tests/datatype/referencedatatype_test.cpp index 2442e51e7ec..54489feb2f2 100644 --- a/document/src/tests/datatype/referencedatatype_test.cpp +++ b/document/src/tests/datatype/referencedatatype_test.cpp @@ -3,7 +3,7 @@ #include <vespa/document/base/field.h> #include <vespa/document/datatype/referencedatatype.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <ostream> #include <sstream> diff --git a/document/src/tests/document_type_repo_factory/CMakeLists.txt b/document/src/tests/document_type_repo_factory/CMakeLists.txt index 5eeba299048..774ad629537 100644 --- a/document/src/tests/document_type_repo_factory/CMakeLists.txt +++ b/document/src/tests/document_type_repo_factory/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(document_document_type_repo_factory_test_app TEST SOURCES document_type_repo_factory_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_document_type_repo_factory_test_app COMMAND document_document_type_repo_factory_test_app) diff --git a/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp b/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp index c56b9ca55a6..edb7f8e4f45 100644 --- a/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp +++ b/document/src/tests/document_type_repo_factory/document_type_repo_factory_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/document_type_repo_factory.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using vespalib::string; using namespace document::config_builder; diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 9da03d6001b..7c571df257c 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -33,8 +33,11 @@ #include <vespa/eval/eval/value.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/test/test_data.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/exceptions.h> +#include <filesystem> #include <fstream> #include <unistd.h> @@ -44,6 +47,7 @@ using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; using vespalib::nbostream; +using vespalib::test::TestDataBase; namespace document { @@ -87,31 +91,38 @@ void testRoundtripSerialize(const UpdateType& update, const DataType &type) { } } -void -writeBufferToFile(const nbostream &buf, const vespalib::string &fileName) +} + +class DocumentUpdateTest : public ::testing::Test, public vespalib::test::TestData<DocumentUpdateTest> { +protected: + DocumentUpdateTest(); + ~DocumentUpdateTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +DocumentUpdateTest::DocumentUpdateTest() + : ::testing::Test(), + vespalib::test::TestData<DocumentUpdateTest>() { - auto file = std::fstream(fileName, std::ios::out | std::ios::binary); - file.write(buf.data(), buf.size()); - assert(file.good()); - file.close(); } -nbostream -readBufferFromFile(const vespalib::string &fileName) +DocumentUpdateTest::~DocumentUpdateTest() = default; + +void +DocumentUpdateTest::SetUpTestSuite() { - auto file = std::fstream(fileName, std::ios::in | std::ios::binary | std::ios::ate); - auto size = file.tellg(); - file.seekg(0); - vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(size); - file.read(static_cast<char *>(buf.get()), size); - assert(file.good()); - file.close(); - return nbostream(std::move(buf), size); + setup_test_data(TEST_PATH("data"), "documentupdate-build-data"); + std::filesystem::create_directory(build_testdata()); } +void +DocumentUpdateTest::TearDownTestSuite() +{ + tear_down_test_data(); } -TEST(DocumentUpdateTest, testSimpleUsage) +TEST_F(DocumentUpdateTest, testSimpleUsage) { DocumenttypesConfigBuilderHelper builder; builder.document(42, "test", @@ -205,7 +216,7 @@ TEST(DocumentUpdateTest, testSimpleUsage) } } -TEST(DocumentUpdateTest, testClearField) +TEST_F(DocumentUpdateTest, testClearField) { // Create a document. TestDocMan docMan; @@ -220,7 +231,7 @@ TEST(DocumentUpdateTest, testClearField) EXPECT_FALSE(doc->getValue("headerval")); } -TEST(DocumentUpdateTest, testUpdateApplySingleValue) +TEST_F(DocumentUpdateTest, testUpdateApplySingleValue) { // Create a document. TestDocMan docMan; @@ -235,7 +246,7 @@ TEST(DocumentUpdateTest, testUpdateApplySingleValue) EXPECT_EQ(9, doc->getValue("headerval")->getAsInt()); } -TEST(DocumentUpdateTest, testUpdateArray) +TEST_F(DocumentUpdateTest, testUpdateArray) { // Create a document. TestDocMan docMan; @@ -314,7 +325,7 @@ createAddUpdate(int key, int weight) { return upd; } -TEST(DocumentUpdateTest, testUpdateWeightedSet) +TEST_F(DocumentUpdateTest, testUpdateWeightedSet) { // Create a test document TestDocMan docMan; @@ -427,7 +438,7 @@ WeightedSetAutoCreateFixture::WeightedSetAutoCreateFixture() } } // anon ns -TEST(DocumentUpdateTest, testIncrementNonExistingAutoCreateWSetField) +TEST_F(DocumentUpdateTest, testIncrementNonExistingAutoCreateWSetField) { WeightedSetAutoCreateFixture fixture; @@ -440,7 +451,7 @@ TEST(DocumentUpdateTest, testIncrementNonExistingAutoCreateWSetField) EXPECT_EQ(1, ws->get(StringFieldValue("foo"), 0)); } -TEST(DocumentUpdateTest, testIncrementExistingWSetField) +TEST_F(DocumentUpdateTest, testIncrementExistingWSetField) { WeightedSetAutoCreateFixture fixture; { @@ -456,7 +467,7 @@ TEST(DocumentUpdateTest, testIncrementExistingWSetField) EXPECT_EQ(1, ws->get(StringFieldValue("foo"), 0)); } -TEST(DocumentUpdateTest, testIncrementWithZeroResultWeightIsRemoved) +TEST_F(DocumentUpdateTest, testIncrementWithZeroResultWeightIsRemoved) { WeightedSetAutoCreateFixture fixture; fixture.update.addUpdate(FieldUpdate(fixture.field) @@ -471,13 +482,13 @@ TEST(DocumentUpdateTest, testIncrementWithZeroResultWeightIsRemoved) EXPECT_FALSE(ws->contains(StringFieldValue("baz"))); } -TEST(DocumentUpdateTest, testReadSerializedFile) +TEST_F(DocumentUpdateTest, testReadSerializedFile) { // Reads a file serialized from java - const std::string file_name = "data/crossplatform-java-cpp-doctypes.cfg"; + const std::string file_name = source_testdata() + "/crossplatform-java-cpp-doctypes.cfg"; DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); - auto is = readBufferFromFile("data/serializeupdatejava.dat"); + auto is = read_buffer_from_file(source_testdata() + "/serializeupdatejava.dat"); DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, is)); DocumentUpdate& upd(*updp); @@ -533,11 +544,11 @@ TEST(DocumentUpdateTest, testReadSerializedFile) } -TEST(DocumentUpdateTest, testGenerateSerializedFile) +TEST_F(DocumentUpdateTest, testGenerateSerializedFile) { // Tests nothing, only generates a file for java test - const std::string file_name = "data/crossplatform-java-cpp-doctypes.cfg"; - DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); + const std::string cfg_file_name = source_testdata() + "/crossplatform-java-cpp-doctypes.cfg"; + DocumentTypeRepo repo(readDocumenttypesConfig(cfg_file_name)); const DocumentType *type(repo.getDocumentType("serializetest")); DocumentUpdate upd(repo, *type, DocumentId("id:ns:serializetest::update")); @@ -557,11 +568,13 @@ TEST(DocumentUpdateTest, testGenerateSerializedFile) .addUpdate(std::make_unique<MapValueUpdate>(StringFieldValue::make("foo"), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Mul, 2)))); nbostream buf(serializeHEAD(upd)); - writeBufferToFile(buf, "data/serializeupdatecpp.dat"); + std::string file_name("serializeupdatecpp.dat"); + write_buffer_to_file(buf, build_testdata() + "/" + file_name); + ASSERT_NO_FATAL_FAILURE(remove_unchanged_build_testdata_file_or_fail(buf, file_name)); } -TEST(DocumentUpdateTest, testSetBadFieldTypes) +TEST_F(DocumentUpdateTest, testSetBadFieldTypes) { // Create a test document TestDocMan docMan; @@ -582,7 +595,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) doc->getValue(doc->getField("headerval")).get()); } -TEST(DocumentUpdateTest, testUpdateApplyNoParams) +TEST_F(DocumentUpdateTest, testUpdateApplyNoParams) { TestDocMan docMan; Document::UP doc(docMan.createDocument()); @@ -597,7 +610,7 @@ TEST(DocumentUpdateTest, testUpdateApplyNoParams) EXPECT_FALSE(doc->hasValue(doc->getField("tags"))); } -TEST(DocumentUpdateTest, testUpdateApplyNoArrayValues) +TEST_F(DocumentUpdateTest, testUpdateApplyNoArrayValues) { TestDocMan docMan; Document::UP doc(docMan.createDocument()); @@ -617,7 +630,7 @@ TEST(DocumentUpdateTest, testUpdateApplyNoArrayValues) EXPECT_EQ((size_t) 0, fval->size()); } -TEST(DocumentUpdateTest, testUpdateArrayEmptyParamValue) +TEST_F(DocumentUpdateTest, testUpdateArrayEmptyParamValue) { // Create a test document. TestDocMan docMan; @@ -645,7 +658,7 @@ TEST(DocumentUpdateTest, testUpdateArrayEmptyParamValue) EXPECT_FALSE(fval2); } -TEST(DocumentUpdateTest, testUpdateWeightedSetEmptyParamValue) +TEST_F(DocumentUpdateTest, testUpdateWeightedSetEmptyParamValue) { // Create a test document TestDocMan docMan; @@ -673,7 +686,7 @@ TEST(DocumentUpdateTest, testUpdateWeightedSetEmptyParamValue) EXPECT_FALSE(fval2); } -TEST(DocumentUpdateTest, testUpdateArrayWrongSubtype) +TEST_F(DocumentUpdateTest, testUpdateArrayWrongSubtype) { // Create a test document TestDocMan docMan; @@ -697,7 +710,7 @@ TEST(DocumentUpdateTest, testUpdateArrayWrongSubtype) EXPECT_EQ((document::FieldValue*) 0, fval.get()); } -TEST(DocumentUpdateTest, testUpdateWeightedSetWrongSubtype) +TEST_F(DocumentUpdateTest, testUpdateWeightedSetWrongSubtype) { // Create a test document TestDocMan docMan; @@ -721,7 +734,7 @@ TEST(DocumentUpdateTest, testUpdateWeightedSetWrongSubtype) EXPECT_EQ((document::FieldValue*) 0, fval.get()); } -TEST(DocumentUpdateTest, testMapValueUpdate) +TEST_F(DocumentUpdateTest, testMapValueUpdate) { // Create a test document TestDocMan docMan; @@ -940,7 +953,7 @@ struct TensorUpdateFixture { }; -TEST(DocumentUpdateTest, tensor_assign_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_assign_update_can_be_applied) { TensorUpdateFixture f; f.applyUpdate(std::make_unique<AssignValueUpdate>(f.makeBaselineTensor())); @@ -948,7 +961,7 @@ TEST(DocumentUpdateTest, tensor_assign_update_can_be_applied) f.assertTensor(*f.makeBaselineTensor()); } -TEST(DocumentUpdateTest, tensor_clear_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_clear_update_can_be_applied) { TensorUpdateFixture f; f.setTensor(*f.makeBaselineTensor()); @@ -957,7 +970,7 @@ TEST(DocumentUpdateTest, tensor_clear_update_can_be_applied) EXPECT_FALSE(f.getTensor()); } -TEST(DocumentUpdateTest, tensor_add_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_add_update_can_be_applied) { TensorUpdateFixture f; f.assertApplyUpdate(f.spec().add({{"x", "a"}}, 2) @@ -971,7 +984,7 @@ TEST(DocumentUpdateTest, tensor_add_update_can_be_applied) .add({{"x", "c"}}, 7)); } -TEST(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor) +TEST_F(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor) { TensorUpdateFixture f; f.assertApplyUpdateNonExisting(std::make_unique<TensorAddUpdate>(f.makeTensor(f.spec().add({{"x", "b"}}, 5) @@ -981,7 +994,7 @@ TEST(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor) .add({{"x", "c"}}, 7)); } -TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_remove_update_can_be_applied) { TensorUpdateFixture f; f.assertApplyUpdate(f.spec().add({{"x", "a"}}, 2) @@ -992,13 +1005,13 @@ TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied) f.spec().add({{"x", "a"}}, 2)); } -TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied_to_nonexisting_tensor) +TEST_F(DocumentUpdateTest, tensor_remove_update_can_be_applied_to_nonexisting_tensor) { TensorUpdateFixture f; f.assertApplyUpdateNonExisting(std::make_unique<TensorRemoveUpdate>(f.makeTensor(f.spec().add({{"x", "b"}}, 1)))); } -TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_modify_update_can_be_applied) { TensorUpdateFixture f; auto baseLine = f.spec().add({{"x", "a"}}, 2) @@ -1024,7 +1037,7 @@ TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied) .add({{"x", "b"}}, 15)); } -TEST(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_can_be_applied) +TEST_F(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_can_be_applied) { TensorUpdateFixture f; auto baseLine = f.spec().add({{"x", "a"}}, 2) @@ -1038,14 +1051,14 @@ TEST(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_can .add({{"x", "c"}}, 6)); } -TEST(DocumentUpdateTest, tensor_modify_update_is_ignored_when_applied_to_nonexisting_tensor) +TEST_F(DocumentUpdateTest, tensor_modify_update_is_ignored_when_applied_to_nonexisting_tensor) { TensorUpdateFixture f; f.assertApplyUpdateNonExisting(std::make_unique<TensorModifyUpdate>(TensorModifyUpdate::Operation::ADD, f.makeTensor(f.spec().add({{"x", "b"}}, 5)))); } -TEST(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_is_applied_to_nonexisting_tensor) +TEST_F(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_is_applied_to_nonexisting_tensor) { TensorUpdateFixture f; f.assertApplyUpdateNonExisting(std::make_unique<TensorModifyUpdate>(TensorModifyUpdate::Operation::ADD, @@ -1055,25 +1068,25 @@ TEST(DocumentUpdateTest, tensor_modify_update_with_create_non_existing_cells_is_ .add({{"x", "c"}}, 6)); } -TEST(DocumentUpdateTest, tensor_assign_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_assign_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; f.assertRoundtripSerialize(AssignValueUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_add_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_add_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; f.assertRoundtripSerialize(TensorAddUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_remove_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_remove_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; f.assertRoundtripSerialize(TensorRemoveUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_remove_update_with_not_fully_specified_address_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_remove_update_with_not_fully_specified_address_can_be_roundtrip_serialized) { TensorUpdateFixture f("sparse_xy_tensor"); TensorDataType type(ValueType::from_spec("tensor(y{})")); @@ -1081,13 +1094,13 @@ TEST(DocumentUpdateTest, tensor_remove_update_with_not_fully_specified_address_c makeTensorFieldValue(TensorSpec("tensor(y{})").add({{"y", "a"}}, 1), type))); } -TEST(DocumentUpdateTest, tensor_remove_update_on_float_tensor_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_remove_update_on_float_tensor_can_be_roundtrip_serialized) { TensorUpdateFixture f("sparse_float_tensor"); f.assertRoundtripSerialize(TensorRemoveUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_modify_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_modify_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, f.makeBaselineTensor())); @@ -1098,7 +1111,7 @@ TEST(DocumentUpdateTest, tensor_modify_update_can_be_roundtrip_serialized) f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::MULTIPLY, f.makeBaselineTensor(), 1.0)); } -TEST(DocumentUpdateTest, tensor_modify_update_on_float_tensor_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_modify_update_on_float_tensor_can_be_roundtrip_serialized) { TensorUpdateFixture f("sparse_float_tensor"); f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, f.makeBaselineTensor())); @@ -1109,7 +1122,7 @@ TEST(DocumentUpdateTest, tensor_modify_update_on_float_tensor_can_be_roundtrip_s f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::MULTIPLY, f.makeBaselineTensor(), 1.0)); } -TEST(DocumentUpdateTest, tensor_modify_update_on_dense_tensor_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, tensor_modify_update_on_dense_tensor_can_be_roundtrip_serialized) { TensorUpdateFixture f("dense_tensor"); vespalib::string sparseType("tensor(x{})"); @@ -1118,25 +1131,25 @@ TEST(DocumentUpdateTest, tensor_modify_update_on_dense_tensor_can_be_roundtrip_s f.assertRoundtripSerialize(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, std::move(sparseTensor))); } -TEST(DocumentUpdateTest, tensor_add_update_throws_on_non_tensor_field) +TEST_F(DocumentUpdateTest, tensor_add_update_throws_on_non_tensor_field) { TensorUpdateFixture f; f.assertThrowOnNonTensorField(TensorAddUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_remove_update_throws_on_non_tensor_field) +TEST_F(DocumentUpdateTest, tensor_remove_update_throws_on_non_tensor_field) { TensorUpdateFixture f; f.assertThrowOnNonTensorField(TensorRemoveUpdate(f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_modify_update_throws_on_non_tensor_field) +TEST_F(DocumentUpdateTest, tensor_modify_update_throws_on_non_tensor_field) { TensorUpdateFixture f; f.assertThrowOnNonTensorField(TensorModifyUpdate(TensorModifyUpdate::Operation::REPLACE, f.makeBaselineTensor())); } -TEST(DocumentUpdateTest, tensor_remove_update_throws_if_address_tensor_is_not_sparse) +TEST_F(DocumentUpdateTest, tensor_remove_update_throws_if_address_tensor_is_not_sparse) { TensorUpdateFixture f("dense_tensor"); auto addressTensor = f.makeTensor(f.spec().add({{"x", 0}}, 2)); // creates a dense address tensor @@ -1145,7 +1158,7 @@ TEST(DocumentUpdateTest, tensor_remove_update_throws_if_address_tensor_is_not_sp vespalib::IllegalStateException); } -TEST(DocumentUpdateTest, tensor_modify_update_throws_if_cells_tensor_is_not_sparse) +TEST_F(DocumentUpdateTest, tensor_modify_update_throws_if_cells_tensor_is_not_sparse) { TensorUpdateFixture f("dense_tensor"); auto cellsTensor = f.makeTensor(f.spec().add({{"x", 0}}, 2)); // creates a dense cells tensor @@ -1209,31 +1222,34 @@ struct TensorUpdateSerializeFixture { void serializeUpdateToFile(const DocumentUpdate &update, const vespalib::string &fileName) { nbostream buf = serializeHEAD(update); - writeBufferToFile(buf, fileName); + TestDataBase::write_buffer_to_file(buf, fileName); } DocumentUpdate::UP deserializeUpdateFromFile(const vespalib::string &fileName) { - auto stream = readBufferFromFile(fileName); + auto stream = TestDataBase::read_buffer_from_file(fileName); return DocumentUpdate::createHEAD(*repo, stream); } }; -TEST(DocumentUpdateTest, tensor_update_file_java_can_be_deserialized) +TEST_F(DocumentUpdateTest, tensor_update_file_java_can_be_deserialized) { TensorUpdateSerializeFixture f; - auto update = f.deserializeUpdateFromFile("data/serialize-tensor-update-java.dat"); + auto update = f.deserializeUpdateFromFile(source_testdata() + "/serialize-tensor-update-java.dat"); EXPECT_EQ(*f.makeUpdate(), *update); } -TEST(DocumentUpdateTest, generate_serialized_tensor_update_file_cpp) +TEST_F(DocumentUpdateTest, generate_serialized_tensor_update_file_cpp) { TensorUpdateSerializeFixture f; auto update = f.makeUpdate(); - f.serializeUpdateToFile(*update, "data/serialize-tensor-update-cpp.dat"); + std::string file_name("serialize-tensor-update-cpp.dat"); + auto act_path = build_testdata() + "/" + file_name; + f.serializeUpdateToFile(*update, act_path); + auto buf = read_buffer_from_file(act_path); + ASSERT_NO_FATAL_FAILURE(remove_unchanged_build_testdata_file_or_fail(buf, file_name)); } - void assertDocumentUpdateFlag(bool createIfNonExistent, int value) { @@ -1249,7 +1265,7 @@ assertDocumentUpdateFlag(bool createIfNonExistent, int value) EXPECT_EQ(value, extractedValue); } -TEST(DocumentUpdateTest, testThatDocumentUpdateFlagsIsWorking) +TEST_F(DocumentUpdateTest, testThatDocumentUpdateFlagsIsWorking) { { // create-if-non-existent = true assertDocumentUpdateFlag(true, 0); @@ -1289,7 +1305,7 @@ CreateIfNonExistentFixture::CreateIfNonExistentFixture() update->setCreateIfNonExistent(true); } -TEST(DocumentUpdateTest, testThatCreateIfNonExistentFlagIsSerializedAndDeserialized) +TEST_F(DocumentUpdateTest, testThatCreateIfNonExistentFlagIsSerializedAndDeserialized) { CreateIfNonExistentFixture f; @@ -1322,7 +1338,7 @@ ArrayUpdateFixture::ArrayUpdateFixture() } ArrayUpdateFixture::~ArrayUpdateFixture() = default; -TEST(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) +TEST_F(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) { ArrayUpdateFixture f; @@ -1332,7 +1348,7 @@ TEST(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) EXPECT_EQ(*f.update, *deserialized); } -TEST(DocumentUpdateTest, array_element_update_applies_to_specified_element) +TEST_F(DocumentUpdateTest, array_element_update_applies_to_specified_element) { ArrayUpdateFixture f; @@ -1351,7 +1367,7 @@ TEST(DocumentUpdateTest, array_element_update_applies_to_specified_element) EXPECT_EQ(vespalib::string("blarg"), (*result_array)[2].getAsString()); } -TEST(DocumentUpdateTest, array_element_update_for_invalid_index_is_ignored) +TEST_F(DocumentUpdateTest, array_element_update_for_invalid_index_is_ignored) { ArrayUpdateFixture f; @@ -1414,7 +1430,7 @@ struct UpdateToEmptyDocumentFixture { } }; -TEST(DocumentUpdateTest, string_field_annotations_can_be_deserialized_after_assign_update_to_empty_document) +TEST_F(DocumentUpdateTest, string_field_annotations_can_be_deserialized_after_assign_update_to_empty_document) { UpdateToEmptyDocumentFixture f; auto doc = f.make_empty_doc(); diff --git a/document/src/tests/fieldvalue/CMakeLists.txt b/document/src/tests/fieldvalue/CMakeLists.txt index 2bfc4cd72da..9ad5e3da44d 100644 --- a/document/src/tests/fieldvalue/CMakeLists.txt +++ b/document/src/tests/fieldvalue/CMakeLists.txt @@ -3,21 +3,21 @@ vespa_add_executable(document_document_test_app TEST SOURCES document_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_document_test_app COMMAND document_document_test_app) vespa_add_executable(document_fieldvalue_test_app TEST SOURCES fieldvalue_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_fieldvalue_test_app COMMAND document_fieldvalue_test_app) vespa_add_executable(document_predicatefieldvalue_test_app TEST SOURCES predicatefieldvalue_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_predicatefieldvalue_test_app COMMAND document_predicatefieldvalue_test_app) @@ -25,6 +25,6 @@ vespa_add_executable(document_referencefieldvalue_test_app TEST SOURCES referencefieldvalue_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_referencefieldvalue_test_app COMMAND document_referencefieldvalue_test_app) diff --git a/document/src/tests/fieldvalue/document_test.cpp b/document/src/tests/fieldvalue/document_test.cpp index ef338c3e97f..0f2d6b30462 100644 --- a/document/src/tests/fieldvalue/document_test.cpp +++ b/document/src/tests/fieldvalue/document_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/base/documentid.h> #include <vespa/document/base/testdocrepo.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> using namespace document; diff --git a/document/src/tests/fieldvalue/fieldvalue_test.cpp b/document/src/tests/fieldvalue/fieldvalue_test.cpp index e78509143ad..6cb5fb2dcb8 100644 --- a/document/src/tests/fieldvalue/fieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/fieldvalue_test.cpp @@ -5,7 +5,7 @@ #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/document/fieldvalue/intfieldvalue.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("fieldvalue_test"); diff --git a/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp b/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp index 93505d71c96..67d5b163463 100644 --- a/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicatefieldvalue. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/predicatefieldvalue.h> diff --git a/document/src/tests/fieldvalue/referencefieldvalue_test.cpp b/document/src/tests/fieldvalue/referencefieldvalue_test.cpp index 8dc5778055a..57b313b2237 100644 --- a/document/src/tests/fieldvalue/referencefieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/referencefieldvalue_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/datatype/referencedatatype.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <ostream> diff --git a/document/src/tests/predicate/CMakeLists.txt b/document/src/tests/predicate/CMakeLists.txt index 41eb81581a5..58803322c22 100644 --- a/document/src/tests/predicate/CMakeLists.txt +++ b/document/src/tests/predicate/CMakeLists.txt @@ -3,20 +3,20 @@ vespa_add_executable(document_predicate_test_app TEST SOURCES predicate_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_predicate_test_app COMMAND document_predicate_test_app) vespa_add_executable(document_predicate_builder_test_app TEST SOURCES predicate_builder_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_predicate_builder_test_app COMMAND document_predicate_builder_test_app) vespa_add_executable(document_predicate_printer_test_app TEST SOURCES predicate_printer_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_predicate_printer_test_app COMMAND document_predicate_printer_test_app) diff --git a/document/src/tests/predicate/predicate_builder_test.cpp b/document/src/tests/predicate/predicate_builder_test.cpp index 81809846a9a..ea5544d5d98 100644 --- a/document/src/tests/predicate/predicate_builder_test.cpp +++ b/document/src/tests/predicate/predicate_builder_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("predicate_builder_test"); #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_builder.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using std::string; using vespalib::Slime; diff --git a/document/src/tests/predicate/predicate_printer_test.cpp b/document/src/tests/predicate/predicate_printer_test.cpp index 49a0a5abc81..06c9b025ee2 100644 --- a/document/src/tests/predicate/predicate_printer_test.cpp +++ b/document/src/tests/predicate/predicate_printer_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("predicate_printer_test"); #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_printer.h> #include <vespa/document/predicate/predicate_slime_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using vespalib::Slime; using vespalib::slime::Cursor; diff --git a/document/src/tests/predicate/predicate_test.cpp b/document/src/tests/predicate/predicate_test.cpp index 9835bbc765b..327d05743a0 100644 --- a/document/src/tests/predicate/predicate_test.cpp +++ b/document/src/tests/predicate/predicate_test.cpp @@ -3,7 +3,7 @@ #include <vespa/document/predicate/predicate.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/predicate/predicate_slime_builder.h> #include <string> diff --git a/document/src/tests/repo/CMakeLists.txt b/document/src/tests/repo/CMakeLists.txt index c3c4bbc191a..099918ac437 100644 --- a/document/src/tests/repo/CMakeLists.txt +++ b/document/src/tests/repo/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(document_documenttyperepo_test_app TEST SOURCES documenttyperepo_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_documenttyperepo_test_app COMMAND document_documenttyperepo_test_app) @@ -11,6 +11,6 @@ vespa_add_executable(document_doctype_config_test_app TEST SOURCES doctype_config_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_doctype_config_test_app COMMAND document_doctype_config_test_app) diff --git a/document/src/tests/repo/doctype_config_test.cpp b/document/src/tests/repo/doctype_config_test.cpp index 044293d90c6..0705b53c2df 100644 --- a/document/src/tests/repo/doctype_config_test.cpp +++ b/document/src/tests/repo/doctype_config_test.cpp @@ -14,7 +14,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <set> diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index e8017ae166a..b2d79209c12 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -14,7 +14,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <set> diff --git a/document/src/tests/select/CMakeLists.txt b/document/src/tests/select/CMakeLists.txt index ab29941470d..10ce38cb9aa 100644 --- a/document/src/tests/select/CMakeLists.txt +++ b/document/src/tests/select/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(document_select_test_app TEST SOURCES select_test.cpp DEPENDS - document + vespa_document GTest::GTest ) vespa_add_test(NAME document_select_test_app COMMAND document_select_test_app) diff --git a/document/src/tests/serialization/CMakeLists.txt b/document/src/tests/serialization/CMakeLists.txt index 8a997299e38..3841a81a258 100644 --- a/document/src/tests/serialization/CMakeLists.txt +++ b/document/src/tests/serialization/CMakeLists.txt @@ -3,14 +3,14 @@ vespa_add_executable(document_vespadocumentserializer_test_app TEST SOURCES vespadocumentserializer_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_vespadocumentserializer_test_app COMMAND document_vespadocumentserializer_test_app) vespa_add_executable(document_annotationserializer_test_app TEST SOURCES annotationserializer_test.cpp DEPENDS - document + vespa_document GTest::gtest ) vespa_add_test(NAME document_annotationserializer_test_app COMMAND document_annotationserializer_test_app) diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index ddc633e42f3..65a77b1f872 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -43,7 +43,7 @@ #include <vespa/eval/eval/test/value_compare.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/exceptions.h> #include <vespa/vespalib/util/compressionconfig.h> #include <filesystem> diff --git a/document/src/tests/struct_anno/CMakeLists.txt b/document/src/tests/struct_anno/CMakeLists.txt index 2d688454058..93ce6b80998 100644 --- a/document/src/tests/struct_anno/CMakeLists.txt +++ b/document/src/tests/struct_anno/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(document_struct_anno_test_app TEST SOURCES struct_anno_test.cpp DEPENDS - document + vespa_document GTest::gtest ) vespa_add_test(NAME document_struct_anno_test_app COMMAND document_struct_anno_test_app) diff --git a/document/src/tests/tensor_fieldvalue/CMakeLists.txt b/document/src/tests/tensor_fieldvalue/CMakeLists.txt index 66544cf62d2..ea286ca47a2 100644 --- a/document/src/tests/tensor_fieldvalue/CMakeLists.txt +++ b/document/src/tests/tensor_fieldvalue/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(document_tensor_fieldvalue_test_app TEST SOURCES tensor_fieldvalue_test.cpp DEPENDS - document + vespa_document ) vespa_add_test(NAME document_tensor_fieldvalue_test_app COMMAND document_tensor_fieldvalue_test_app) diff --git a/document/src/tests/tensor_fieldvalue/partial_add/CMakeLists.txt b/document/src/tests/tensor_fieldvalue/partial_add/CMakeLists.txt index e1f0a9ede38..c726b58f134 100644 --- a/document/src/tests/tensor_fieldvalue/partial_add/CMakeLists.txt +++ b/document/src/tests/tensor_fieldvalue/partial_add/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(document_partial_add_test_app TEST SOURCES partial_add_test.cpp DEPENDS - document + vespa_document GTest::GTest ) vespa_add_test(NAME document_partial_add_test_app COMMAND document_partial_add_test_app) diff --git a/document/src/tests/tensor_fieldvalue/partial_modify/CMakeLists.txt b/document/src/tests/tensor_fieldvalue/partial_modify/CMakeLists.txt index b55fff1e23f..b2fcd318033 100644 --- a/document/src/tests/tensor_fieldvalue/partial_modify/CMakeLists.txt +++ b/document/src/tests/tensor_fieldvalue/partial_modify/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(document_partial_modify_test_app TEST SOURCES partial_modify_test.cpp DEPENDS - document + vespa_document GTest::GTest ) vespa_add_test(NAME document_partial_modify_test_app COMMAND document_partial_modify_test_app) diff --git a/document/src/tests/tensor_fieldvalue/partial_remove/CMakeLists.txt b/document/src/tests/tensor_fieldvalue/partial_remove/CMakeLists.txt index 9f87c7f3180..4df639b89bf 100644 --- a/document/src/tests/tensor_fieldvalue/partial_remove/CMakeLists.txt +++ b/document/src/tests/tensor_fieldvalue/partial_remove/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(document_partial_remove_test_app TEST SOURCES partial_remove_test.cpp DEPENDS - document + vespa_document GTest::GTest ) vespa_add_test(NAME document_partial_remove_test_app COMMAND document_partial_remove_test_app) diff --git a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp index 13f2a58e880..4cc4e5e6215 100644 --- a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp +++ b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp @@ -11,7 +11,7 @@ LOG_SETUP("fieldvalue_test"); #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using vespalib::eval::SimpleValue; diff --git a/document/src/vespa/document/CMakeLists.txt b/document/src/vespa/document/CMakeLists.txt index 581b8d078a5..b9c55a4b87b 100644 --- a/document/src/vespa/document/CMakeLists.txt +++ b/document/src/vespa/document/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(document +vespa_add_library(vespa_document SOURCES $<TARGET_OBJECTS:document_util> $<TARGET_OBJECTS:document_base> diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index fe146bc83d5..a3b7b463c3f 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -2,14 +2,14 @@ vespa_define_module( DEPENDS vespalog - config_cloudconfig + vespa_config vespalib - fnet - document - slobrok - messagebus - configdefinitions - vdslib + vespa_fnet + vespa_document + vespa_slobrok + vespa_messagebus + vespa_configdefinitions + vespa_vdslib LIBS src/vespa/documentapi @@ -18,7 +18,7 @@ vespa_define_module( src/vespa/documentapi/messagebus/policies TEST_DEPENDS - messagebus_messagebus-test + vespa_messagebus-test TESTS src/tests/messagebus diff --git a/documentapi/src/tests/messagebus/CMakeLists.txt b/documentapi/src/tests/messagebus/CMakeLists.txt index ef597fbee10..7341b22f9e9 100644 --- a/documentapi/src/tests/messagebus/CMakeLists.txt +++ b/documentapi/src/tests/messagebus/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(documentapi_messagebus_test_app TEST SOURCES messagebus_test.cpp DEPENDS - documentapi + vespa_documentapi GTest::gtest ) vespa_add_test(NAME documentapi_messagebus_test_app COMMAND documentapi_messagebus_test_app) diff --git a/documentapi/src/tests/messages/CMakeLists.txt b/documentapi/src/tests/messages/CMakeLists.txt index bbe29e4802a..b34a37e182e 100644 --- a/documentapi/src/tests/messages/CMakeLists.txt +++ b/documentapi/src/tests/messages/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(documentapi_messages_test_app TEST messages80test.cpp messages_app.cpp DEPENDS - documentapi + vespa_documentapi GTest::GTest ) vespa_add_test(NAME documentapi_messages_test_app COMMAND documentapi_messages_test_app) diff --git a/documentapi/src/tests/policies/CMakeLists.txt b/documentapi/src/tests/policies/CMakeLists.txt index 9c537c33db8..bcbdea9aa23 100644 --- a/documentapi/src/tests/policies/CMakeLists.txt +++ b/documentapi/src/tests/policies/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(documentapi_policies_test_app TEST testframe.cpp policies_test.cpp DEPENDS - documentapi + vespa_documentapi GTest::gtest ) vespa_add_test(NAME documentapi_policies_test_app COMMAND documentapi_policies_test_app) diff --git a/documentapi/src/tests/policyfactory/CMakeLists.txt b/documentapi/src/tests/policyfactory/CMakeLists.txt index 64add841c53..a1cf74c12f8 100644 --- a/documentapi/src/tests/policyfactory/CMakeLists.txt +++ b/documentapi/src/tests/policyfactory/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(documentapi_policyfactory_test_app TEST SOURCES policyfactory.cpp DEPENDS - documentapi + vespa_documentapi ) vespa_add_test(NAME documentapi_policyfactory_test_app COMMAND documentapi_policyfactory_test_app) diff --git a/documentapi/src/tests/policyfactory/policyfactory.cpp b/documentapi/src/tests/policyfactory/policyfactory.cpp index 1eaa3b86304..f18650ec6e3 100644 --- a/documentapi/src/tests/policyfactory/policyfactory.cpp +++ b/documentapi/src/tests/policyfactory/policyfactory.cpp @@ -6,7 +6,7 @@ #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::DocumentTypeRepo; using namespace documentapi; @@ -70,14 +70,9 @@ createMessage() // /////////////////////////////////////////////////////////////////////////////// -TEST_SETUP(Test); - const vespalib::duration TIMEOUT = 600s; -int -Test::Main() -{ - TEST_INIT("policyfactory_test"); +TEST("policyfactory_test") { std::shared_ptr<const DocumentTypeRepo> repo(new DocumentTypeRepo); mbus::Slobrok slobrok; @@ -106,6 +101,6 @@ Test::Main() fprintf(stderr, "%s", reply->getTrace().toString().c_str()); EXPECT_EQUAL(1u, reply->getNumErrors()); EXPECT_EQUAL((uint32_t)DocumentProtocol::ERROR_POLICY_FAILURE, reply->getError(0).getCode()); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/documentapi/src/tests/priority/CMakeLists.txt b/documentapi/src/tests/priority/CMakeLists.txt index 41946b2607c..1738fe100b5 100644 --- a/documentapi/src/tests/priority/CMakeLists.txt +++ b/documentapi/src/tests/priority/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(documentapi_priority_test_app TEST SOURCES priority.cpp DEPENDS - documentapi + vespa_documentapi ) vespa_add_test(NAME documentapi_priority_test_app COMMAND documentapi_priority_test_app) diff --git a/documentapi/src/tests/priority/priority.cpp b/documentapi/src/tests/priority/priority.cpp index a4167e6551c..d0290495035 100644 --- a/documentapi/src/tests/priority/priority.cpp +++ b/documentapi/src/tests/priority/priority.cpp @@ -1,18 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/documentapi/messagebus/priority.h> #include <fstream> #include <algorithm> using namespace documentapi; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("priority_test"); +TEST("priority_test") { std::vector<int32_t> expected; expected.push_back(Priority::PRI_HIGHEST); @@ -52,6 +47,6 @@ Test::Main() expected.erase(it); } ASSERT_TRUE(expected.empty()); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/documentapi/src/tests/replymerger/CMakeLists.txt b/documentapi/src/tests/replymerger/CMakeLists.txt index 41eaae643c4..9928537d9fa 100644 --- a/documentapi/src/tests/replymerger/CMakeLists.txt +++ b/documentapi/src/tests/replymerger/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(documentapi_replymerger_test_app TEST SOURCES replymerger_test.cpp DEPENDS - documentapi + vespa_documentapi GTest::gtest ) vespa_add_test(NAME documentapi_replymerger_test_app COMMAND documentapi_replymerger_test_app) diff --git a/documentapi/src/tests/routablefactory/CMakeLists.txt b/documentapi/src/tests/routablefactory/CMakeLists.txt index fa733e7a149..2be527c110c 100644 --- a/documentapi/src/tests/routablefactory/CMakeLists.txt +++ b/documentapi/src/tests/routablefactory/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(documentapi_routablefactory_test_app TEST SOURCES routablefactory.cpp DEPENDS - documentapi + vespa_documentapi + GTest::gtest ) vespa_add_test(NAME documentapi_routablefactory_test_app COMMAND documentapi_routablefactory_test_app) diff --git a/documentapi/src/tests/routablefactory/routablefactory.cpp b/documentapi/src/tests/routablefactory/routablefactory.cpp index a25c0867461..272d2965474 100644 --- a/documentapi/src/tests/routablefactory/routablefactory.cpp +++ b/documentapi/src/tests/routablefactory/routablefactory.cpp @@ -5,7 +5,7 @@ #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> using document::DocumentTypeRepo; using namespace documentapi; @@ -109,16 +109,6 @@ public: bool start(); }; -class Test : public vespalib::TestApp { -protected: - void testFactory(TestData &data); - -public: - int Main() override; -}; - -TEST_APPHOOK(Test); - TestData::TestData() : _repo(std::make_shared<DocumentTypeRepo>()), _slobrok(), @@ -153,19 +143,6 @@ TestData::start() return true; } -int -Test::Main() -{ - TEST_INIT("routablefactory_test"); - - TestData data; - ASSERT_TRUE(data.start()); - - testFactory(data); TEST_FLUSH(); - - TEST_DONE(); -} - /////////////////////////////////////////////////////////////////////////////// // // Tests @@ -174,9 +151,11 @@ Test::Main() const vespalib::duration TIMEOUT = 600s; -void -Test::testFactory(TestData &data) +TEST(RoutableFactoryTest, test_factory) { + TestData data; + ASSERT_TRUE(data.start()); + mbus::Route route = mbus::Route::parse("dst/session"); // Source should fail to encode the message. @@ -185,8 +164,8 @@ Test::testFactory(TestData &data) ASSERT_TRUE(reply); fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); ASSERT_TRUE(reply->hasErrors()); - EXPECT_EQUAL((uint32_t)mbus::ErrorCode::ENCODE_ERROR, reply->getError(0).getCode()); - EXPECT_EQUAL("", reply->getError(0).getService()); + EXPECT_EQ((uint32_t)mbus::ErrorCode::ENCODE_ERROR, reply->getError(0).getCode()); + EXPECT_EQ("", reply->getError(0).getService()); // Destination should fail to decode the message. data._srcProtocol->putRoutableFactory(MyMessage::TYPE, IRoutableFactory::SP(new MyMessageFactory()), @@ -196,8 +175,8 @@ Test::testFactory(TestData &data) ASSERT_TRUE(reply); fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); EXPECT_TRUE(reply->hasErrors()); - EXPECT_EQUAL((uint32_t)mbus::ErrorCode::DECODE_ERROR, reply->getError(0).getCode()); - EXPECT_EQUAL("dst/session", reply->getError(0).getService()); + EXPECT_EQ((uint32_t)mbus::ErrorCode::DECODE_ERROR, reply->getError(0).getCode()); + EXPECT_EQ("dst/session", reply->getError(0).getService()); // Destination should fail to encode the reply-> data._dstProtocol->putRoutableFactory(MyMessage::TYPE, IRoutableFactory::SP(new MyMessageFactory()), @@ -212,8 +191,8 @@ Test::testFactory(TestData &data) ASSERT_TRUE(reply); fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); EXPECT_TRUE(reply->hasErrors()); - EXPECT_EQUAL((uint32_t)mbus::ErrorCode::ENCODE_ERROR, reply->getError(0).getCode()); - EXPECT_EQUAL("dst/session", reply->getError(0).getService()); + EXPECT_EQ((uint32_t)mbus::ErrorCode::ENCODE_ERROR, reply->getError(0).getCode()); + EXPECT_EQ("dst/session", reply->getError(0).getService()); // Source should fail to decode the reply. data._dstProtocol->putRoutableFactory(MyReply::TYPE, IRoutableFactory::SP(new MyReplyFactory()), @@ -228,8 +207,8 @@ Test::testFactory(TestData &data) ASSERT_TRUE(reply); fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); EXPECT_TRUE(reply->hasErrors()); - EXPECT_EQUAL((uint32_t)mbus::ErrorCode::DECODE_ERROR, reply->getError(0).getCode()); - EXPECT_EQUAL("", reply->getError(0).getService()); + EXPECT_EQ((uint32_t)mbus::ErrorCode::DECODE_ERROR, reply->getError(0).getCode()); + EXPECT_EQ("", reply->getError(0).getService()); // All should succeed. data._srcProtocol->putRoutableFactory(MyReply::TYPE, IRoutableFactory::SP(new MyReplyFactory()), @@ -245,3 +224,5 @@ Test::testFactory(TestData &data) fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); EXPECT_TRUE(!reply->hasErrors()); } + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/documentapi/src/vespa/documentapi/CMakeLists.txt b/documentapi/src/vespa/documentapi/CMakeLists.txt index 8f9fbc4ac11..39034014618 100644 --- a/documentapi/src/vespa/documentapi/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(documentapi +vespa_add_library(vespa_documentapi SOURCES $<TARGET_OBJECTS:documentapi_documentapimessagebus> $<TARGET_OBJECTS:documentapi_documentapimessages> @@ -9,4 +9,4 @@ vespa_add_library(documentapi protobuf::libprotobuf ) -vespa_add_target_package_dependency(documentapi Protobuf) +vespa_add_target_package_dependency(vespa_documentapi Protobuf) diff --git a/eval/src/tests/ann/nns-l2.h b/eval/src/tests/ann/nns-l2.h index 5ce9434172f..4b9fb43d3da 100644 --- a/eval/src/tests/ann/nns-l2.h +++ b/eval/src/tests/ann/nns-l2.h @@ -3,7 +3,7 @@ #pragma once #include <cstring> #include <vespa/vespalib/util/arrayref.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> template <typename T, size_t VLEN> static double hw_l2_sq_dist(const T * af, const T * bf, size_t sz) @@ -34,9 +34,9 @@ static double hw_l2_sq_dist(const T * af, const T * bf, size_t sz) template <typename FltType = float> struct L2DistCalc { - const vespalib::hwaccelrated::IAccelrated & _hw; + const vespalib::hwaccelerated::IAccelerated & _hw; - L2DistCalc() : _hw(vespalib::hwaccelrated::IAccelrated::getAccelerator()) {} + L2DistCalc() : _hw(vespalib::hwaccelerated::IAccelerated::getAccelerator()) {} using Arr = vespalib::ArrayRef<FltType>; using ConstArr = vespalib::ConstArrayRef<FltType>; diff --git a/eval/src/vespa/eval/CMakeLists.txt b/eval/src/vespa/eval/CMakeLists.txt index 494359493ad..d3dcc97bdb2 100644 --- a/eval/src/vespa/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/CMakeLists.txt @@ -11,6 +11,7 @@ vespa_add_library(vespaeval $<TARGET_OBJECTS:eval_streamed> INSTALL lib64 DEPENDS + vespa_hwaccelerated onnxruntime ${VESPA_LLVM_LIB} ) diff --git a/eval/src/vespa/eval/instruction/generic_cell_cast.cpp b/eval/src/vespa/eval/instruction/generic_cell_cast.cpp index 16f47a56222..71cccf65e1e 100644 --- a/eval/src/vespa/eval/instruction/generic_cell_cast.cpp +++ b/eval/src/vespa/eval/instruction/generic_cell_cast.cpp @@ -5,6 +5,7 @@ #include <vespa/eval/eval/wrap_param.h> #include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <cassert> using namespace vespalib::eval::tensor_function; @@ -16,6 +17,8 @@ using Instruction = InterpretedFunction::Instruction; namespace { +using hwaccelerated::IAccelerated; + template <typename ICT, typename OCT> void my_generic_cell_cast_op(State &state, uint64_t param_in) { const auto &res_type = unwrap_param<ValueType>(param_in); @@ -31,6 +34,19 @@ void my_generic_cell_cast_op(State &state, uint64_t param_in) { state.pop_push(result_ref); } +template <> +void my_generic_cell_cast_op<BFloat16, float>(State &state, uint64_t param_in) { + const auto &res_type = unwrap_param<ValueType>(param_in); + const Value &a = state.peek(0); + auto input_cells = a.cells().typify<BFloat16>(); + auto output_cells = state.stash.create_uninitialized_array<float>(input_cells.size()); + static const IAccelerated & accelrator = IAccelerated::getAccelerator(); + accelrator.convert_bfloat16_to_float(reinterpret_cast<const uint16_t *>(input_cells.begin()), + output_cells.data(), output_cells.size()); + Value &result_ref = state.stash.create<ValueView>(res_type, a.index(), TypedCells(output_cells)); + state.pop_push(result_ref); +} + struct SelectGenericCellCastOp { template <typename ICT, typename OCT> static InterpretedFunction::op_function invoke() { @@ -60,7 +76,7 @@ GenericCellCast::make_instruction(const ValueType &result_type, assert(!input_type.is_double()); auto ¶m = stash.create<ValueType>(result_type); auto op = typify_invoke<2,TypifyCellType,SelectGenericCellCastOp>(from, to); - return Instruction(op, wrap_param<ValueType>(param)); + return {op, wrap_param<ValueType>(param)}; } } diff --git a/eval/src/vespa/eval/instruction/l2_distance.cpp b/eval/src/vespa/eval/instruction/l2_distance.cpp index 9490044a39b..5dcd4947e25 100644 --- a/eval/src/vespa/eval/instruction/l2_distance.cpp +++ b/eval/src/vespa/eval/instruction/l2_distance.cpp @@ -3,7 +3,7 @@ #include "l2_distance.h" #include <vespa/eval/eval/operation.h> #include <vespa/eval/eval/value.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <vespa/vespalib/util/require.h> #include <vespa/log/log.h> @@ -15,7 +15,7 @@ using namespace tensor_function; namespace { -static const auto &hw = hwaccelrated::IAccelrated::getAccelerator(); +static const auto &hw = hwaccelerated::IAccelerated::getAccelerator(); template <typename T> double sq_l2(const Value &lhs, const Value &rhs, size_t len) { diff --git a/eval/src/vespa/eval/instruction/mixed_l2_distance.cpp b/eval/src/vespa/eval/instruction/mixed_l2_distance.cpp index 1a5293789c5..d0fca54eff2 100644 --- a/eval/src/vespa/eval/instruction/mixed_l2_distance.cpp +++ b/eval/src/vespa/eval/instruction/mixed_l2_distance.cpp @@ -3,7 +3,7 @@ #include "mixed_l2_distance.h" #include <vespa/eval/eval/operation.h> #include <vespa/eval/eval/value.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <vespa/vespalib/util/require.h> #include <vespa/log/log.h> @@ -15,7 +15,7 @@ using namespace tensor_function; namespace { -static const auto &hw = hwaccelrated::IAccelrated::getAccelerator(); +static const auto &hw = hwaccelerated::IAccelerated::getAccelerator(); template <typename T> double h_sq_l2(const T *a, const T *b, size_t len) { diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt index 3928b11dd61..81a3800d4f6 100644 --- a/fileacquirer/CMakeLists.txt +++ b/fileacquirer/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_define_module( DEPENDS vespalog vespalib - config_cloudconfig + vespa_config LIBS src/vespa/fileacquirer diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a577bbe74df..67f1cf25510 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -48,14 +48,6 @@ public class Flags { private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); - public static final UnboundBooleanFlag ATHENZ_SERVICE_ACCOUNTS = defineFeatureFlag( - "athenz-service-accounts", false, - List.of("hakonhall"), "2024-05-06", "2024-07-06", - "Whether to provision new GCP VM instances with a service account that are independent " + - "of the zone, and aligned with the Athenz service names (configserver and tenant-host).", - "Takes effect when provisioning new VM instances", - APPLICATION, INSTANCE_ID); - public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, List.of("baldersheim"), "2020-12-02", "2024-12-31", @@ -208,7 +200,7 @@ public class Flags { public static final UnboundIntFlag MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS = defineIntFlag( "max-activation-inhibited-out-of-sync-groups", 0, - List.of("vekterli"), "2021-02-19", "2024-06-01", + List.of("vekterli"), "2021-02-19", "2025-02-01", "Allows replicas in up to N content groups to not be activated " + "for query visibility if they are out of sync with a majority of other replicas", "Takes effect at redeployment", @@ -216,7 +208,7 @@ public class Flags { public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, - List.of("geirst", "vekterli"), "2021-07-16", "2024-06-01", + List.of("geirst", "vekterli"), "2021-07-16", "2025-02-01", "Minimum ratio of nodes that have to be available (i.e. not Down) in any hierarchic content cluster group for the group to be Up", "Takes effect at redeployment", INSTANCE_ID); @@ -239,13 +231,6 @@ public class Flags { "Takes effect on next tick.", NODE_TYPE); - public static final UnboundStringFlag DIST_HOST = defineStringFlag( - "dist-host", "", - List.of("freva"), "2024-04-15", "2024-05-31", - "Sets dist_host YUM variable, empty means old behavior. Only effective in Public.", - "Provisioning of instance or next host-admin tick", - HOSTNAME, NODE_TYPE, CLOUD_ACCOUNT); - public static final UnboundBooleanFlag ENABLED_HORIZON_DASHBOARD = defineFeatureFlag( "enabled-horizon-dashboard", false, List.of("olaa"), "2021-09-13", "2024-09-01", @@ -314,32 +299,32 @@ public class Flags { public static final UnboundStringFlag CORE_ENCRYPTION_PUBLIC_KEY_ID = defineStringFlag( "core-encryption-public-key-id", "", - List.of("vekterli"), "2022-11-03", "2024-06-01", + List.of("vekterli"), "2022-11-03", "2025-02-01", "Specifies which public key to use for core dump encryption.", "Takes effect on the next tick.", NODE_TYPE, HOSTNAME); public static final UnboundListFlag<String> ZONAL_WEIGHTED_ENDPOINT_RECORDS = defineListFlag( - "zonal-weighted-endpoint-records", List.of(), String.class, List.of("jonmv"), "2023-12-15", "2024-06-01", + "zonal-weighted-endpoint-records", List.of(), String.class, List.of("jonmv"), "2023-12-15", "2024-12-01", "A list of weighted (application) endpoint fqdns for which we should use zonal endpoints as targets, not LBs.", "Takes effect at redeployment from controller"); public static final UnboundListFlag<String> WEIGHTED_ENDPOINT_RECORD_TTL = defineListFlag( - "weighted-endpoint-record-ttl", List.of(), String.class, List.of("jonmv"), "2023-05-16", "2024-06-01", + "weighted-endpoint-record-ttl", List.of(), String.class, List.of("jonmv"), "2023-05-16", "2024-12-01", "A list of endpoints and custom TTLs, on the form \"endpoint-fqdn:TTL-seconds\". " + "Where specified, CNAME records are used instead of the default ALIAS records, which have a default 60s TTL.", "Takes effect at redeployment from controller"); public static final UnboundBooleanFlag SORT_BLUEPRINTS_BY_COST = defineFeatureFlag( "sort-blueprints-by-cost", false, - List.of("baldersheim"), "2023-12-19", "2024-05-31", + List.of("baldersheim"), "2023-12-19", "2024-10-31", "If true blueprints are sorted based on cost estimate, rather that absolute estimated hits", "Takes effect at redeployment", INSTANCE_ID); public static final UnboundBooleanFlag ALWAYS_MARK_PHRASE_EXPENSIVE = defineFeatureFlag( "always-mark-phrase-expensive", false, - List.of("baldersheim"), "2023-11-20", "2024-05-31", + List.of("baldersheim"), "2023-11-20", "2024-10-31", "If true all phrases will be marked expensive, independent of parents", "Takes effect at redeployment", INSTANCE_ID); @@ -379,7 +364,7 @@ public class Flags { public static final UnboundIntFlag CONTENT_LAYER_METADATA_FEATURE_LEVEL = defineIntFlag( "content-layer-metadata-feature-level", 0, - List.of("vekterli"), "2022-09-12", "2024-06-01", + List.of("vekterli"), "2022-09-12", "2024-12-01", "Value semantics: 0) legacy behavior, 1) operation cancellation, 2) operation " + "cancellation and ephemeral content node sequence numbers for bucket replicas", "Takes effect at redeployment", @@ -401,7 +386,7 @@ public class Flags { public static final UnboundStringFlag ENDPOINT_CONFIG = defineStringFlag( "endpoint-config", "legacy", - List.of("mpolden", "tokle"), "2023-10-06", "2024-06-01", + List.of("mpolden", "tokle"), "2023-10-06", "2024-09-01", "Set the endpoint config to use for an application. Must be 'legacy', 'combined' or 'generated'. See EndpointConfig for further details", "Takes effect on next deployment through controller", TENANT_ID, APPLICATION, INSTANCE_ID); @@ -431,7 +416,7 @@ public class Flags { "Takes effect immediately"); public static final UnboundIntFlag PERSISTENCE_THREAD_MAX_FEED_OP_BATCH_SIZE = defineIntFlag( - "persistence-thread-max-feed-op-batch-size", 1, + "persistence-thread-max-feed-op-batch-size", 64, List.of("vekterli"), "2024-04-12", "2025-01-01", "Maximum number of enqueued feed operations (put/update/remove) bound "+ "towards the same bucket that can be async dispatched as part of the " + @@ -445,12 +430,6 @@ public class Flags { "Whether logserver container should run otel agent", "Takes effect at redeployment", INSTANCE_ID); - public static UnboundBooleanFlag ENCRYPT_DISK = defineFeatureFlag( - "encrypt-disk", true, - List.of("hmusum"), "2024-04-29", "2024-06-01", - "Whether to encrypt disk when provisioning new hosts", - "Will be read only on boot."); - public static UnboundBooleanFlag HUBSPOT_SYNC_TENANTS = defineFeatureFlag( "hubspot-sync-tenants", false, List.of("bjorncs"), "2024-05-07", "2025-01-01", @@ -463,6 +442,56 @@ public class Flags { "Whether EndpointDnsMaintainer should remove orphaned records instead of logging them", "Takes effect on next maintenance run"); + public static final UnboundBooleanFlag SYMMETRIC_PUT_AND_ACTIVATE_REPLICA_SELECTION = defineFeatureFlag( + "symmetric-put-and-activate-replica-selection", false, + List.of("vekterli"), "2024-05-23", "2024-08-01", + "Iff true there will be an 1-1 symmetry between the replicas chosen as feed targets " + + "for Put operations and the replica selection logic for bucket activation. If false, " + + "legacy feed behavior is used.", + "Takes effect immediately", + INSTANCE_ID); + + public static final UnboundBooleanFlag ENABLE_NEW_TRIAL = defineFeatureFlag( + "enable-new-trial", false, + List.of("bjorncs"), "2024-06-18", "2025-01-01", + "Whether to enable the new trial experience", + "Takes effect immediately", + TENANT_ID); + + public static final UnboundBooleanFlag ENFORCE_STRICTLY_INCREASING_CLUSTER_STATE_VERSIONS = defineFeatureFlag( + "enforce-strictly-increasing-cluster-state-versions", false, + List.of("vekterli"), "2024-06-03", "2024-08-01", + "Iff true, received cluster state versions that are lower than the current active " + + "state version on the node will be explicitly rejected.", + "Takes effect immediately", + INSTANCE_ID); + + public static final UnboundBooleanFlag USE_VESPA_ATHENZ_HOST_IDENTITY = defineFeatureFlag( + "use-vespa-athenz-host-identity", false, + List.of("freva"), "2024-06-12", "2024-08-01", + "Whether the host should get identity from Vespa Athenz. Only valid in public systems, noclave, AWS. Vespa version dimension refers to OS version.", + "Takes effect on next provisioning", + INSTANCE_ID, NODE_TYPE, VESPA_VERSION); + + public static final UnboundBooleanFlag LAUNCH_APPLICATION_ATHENZ_SERVICE = defineFeatureFlag( + "launch-application-athenz-service", false, + List.of("jonmv"), "2024-06-11", "2024-09-01", + "Whether to launch an Athenz service unique to the application. Only valid in public systems!", + "Takes effect on next deployment", + INSTANCE_ID); + + public static final UnboundBooleanFlag HUBSPOT_SYNC_CONTACTS = defineFeatureFlag( + "hubspot-sync-contacts", false, + List.of("bjorncs"), "2024-05-27", "2025-01-01", + "Whether to sync contacts to HubSpot", + "Takes effect immediately"); + + public static final UnboundBooleanFlag DELETE_EXPIRED_CONFIG_SESSIONS_NEW_PROCEDURE = defineFeatureFlag( + "delete-expired-config-sessions-new-procedure", false, + List.of("hmusum"), "2024-06-10", "2024-08-10", + "Whether to delete remote and local config sessions at the same time", + "Takes effect immediately"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, @@ -526,6 +555,15 @@ public class Flags { flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ + public static <T> UnboundListFlag<T> defineListFlag(String flagId, List<T> defaultValue, Class<T> elementClass, + List<String> owners, String createdAt, String expiresAt, + String description, String modificationEffect, + Predicate<List<T>> validator, Dimension... dimensions) { + return define((fid, dval, fvec) -> new UnboundListFlag<>(fid, dval, elementClass, fvec, validator), + flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); + } + @FunctionalInterface private interface TypedUnboundFlagFactory<T, U extends UnboundFlag<?, ?, ?>> { U create(FlagId id, T defaultValue, FetchVector defaultFetchVector); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/JacksonArraySerializer.java b/flags/src/main/java/com/yahoo/vespa/flags/JacksonArraySerializer.java index 8f7b7d71734..38e3b57d28b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/JacksonArraySerializer.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/JacksonArraySerializer.java @@ -4,20 +4,27 @@ package com.yahoo.vespa.flags; import com.fasterxml.jackson.databind.JavaType; import java.util.List; +import java.util.function.Predicate; /** * @author freva */ public class JacksonArraySerializer<T> implements FlagSerializer<List<T>> { private final JavaType type; + private final Predicate<List<T>> validator; - public JacksonArraySerializer(Class<T> clazz) { + public JacksonArraySerializer(Class<T> clazz, Predicate<List<T>> validator) { type = JsonNodeRawFlag.constructCollectionType(List.class, clazz); + this.validator = validator; } @Override public List<T> deserialize(RawFlag rawFlag) { - return JsonNodeRawFlag.fromJsonNode(rawFlag.asJsonNode()).toJacksonClass(type); + var list = JsonNodeRawFlag.fromJsonNode(rawFlag.asJsonNode()).<List<T>>toJacksonClass(type); + if (!validator.test(list)) { + throw new IllegalArgumentException("Invalid value: " + list); + } + return list; } @Override diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index f200940e52d..7d1c6bcecdb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -195,12 +195,6 @@ public class PermanentFlags { "Takes effect on next api request" ); - public static final UnboundBooleanFlag JVM_OMIT_STACK_TRACE_IN_FAST_THROW = defineFeatureFlag( - "jvm-omit-stack-trace-in-fast-throw", true, - "Controls JVM option OmitStackTraceInFastThrow (default feature flag value is true, which is the default JVM option value as well)", - "takes effect on JVM restart", - CLUSTER_TYPE, INSTANCE_ID); - public static final UnboundIntFlag MAX_TRIAL_TENANTS = defineIntFlag( "max-trial-tenants", -1, "The maximum nr. of tenants with trial plan, -1 is unlimited", @@ -372,6 +366,13 @@ public class PermanentFlags { "Takes effect immediately" ); + public static final UnboundLongFlag CONFIG_SERVER_SESSION_LIFETIME = defineLongFlag( + "config-server-session-lifetime", 3600, + "Lifetime / expiry time in seconds for config sessions. " + + "This can be lowered if there are incidents/bugs where one needs to delete sessions quickly", + "Takes effect immediately" + ); + public static final UnboundBooleanFlag NOTIFICATION_DISPATCH_FLAG = defineFeatureFlag( "dispatch-notifications", true, "Whether we should send notification for a given tenant", @@ -461,6 +462,13 @@ public class PermanentFlags { HOSTNAME ); + public static final UnboundListFlag<String> LOG_REQUEST_CONTENT = defineListFlag( + "log-request-content", List.of(), String.class, + "Include request content in access log for paths starting with any of these prefixes", + "Takes effect on next redeployment", + list -> list.stream().allMatch(s -> s.matches("^[a-zA-Z/.0-9-]+:(0(\\.\\d+)?|1(\\.0+)?):\\d+(B|kB|MB|GB)?$")), + INSTANCE_ID); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( @@ -503,5 +511,10 @@ public class PermanentFlags { return Flags.defineListFlag(flagId, defaultValue, elementClass, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } + private static <T> UnboundListFlag<T> defineListFlag( + String flagId, List<T> defaultValue, Class<T> elementClass, String description, String modificationEffect, Predicate<List<T>> validator, Dimension... dimensions) { + return Flags.defineListFlag(flagId, defaultValue, elementClass, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, validator, dimensions); + } + private static String toString(Instant instant) { return DateTimeFormatter.ISO_DATE.withZone(ZoneOffset.UTC).format(instant); } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundListFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundListFlag.java index f88a0648021..8411a286d54 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundListFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundListFlag.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.flags; import java.util.List; +import java.util.function.Predicate; /** * @author freva @@ -12,8 +13,13 @@ public class UnboundListFlag<T> extends UnboundFlagImpl<List<T>, ListFlag<T>, Un } public UnboundListFlag(FlagId id, List<T> defaultValue, Class<T> clazz, FetchVector defaultFetchVector) { + this(id, defaultValue, clazz, defaultFetchVector, __ -> true); + } + + public UnboundListFlag(FlagId id, List<T> defaultValue, Class<T> clazz, FetchVector defaultFetchVector, + Predicate<List<T>> validator) { super(id, defaultValue, defaultFetchVector, - new JacksonArraySerializer<T>(clazz), + new JacksonArraySerializer<T>(clazz, validator), (flagId, defVal, fetchVector) -> new UnboundListFlag<>(flagId, defVal, clazz, fetchVector), ListFlag::new); } diff --git a/fnet/src/examples/frt/rpc/CMakeLists.txt b/fnet/src/examples/frt/rpc/CMakeLists.txt index 8bbef0531b1..1268af42306 100644 --- a/fnet/src/examples/frt/rpc/CMakeLists.txt +++ b/fnet/src/examples/frt/rpc/CMakeLists.txt @@ -3,19 +3,19 @@ vespa_add_executable(fnet_rpc_server_app SOURCES rpc_server.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_executable(fnet_rpc_client_app SOURCES rpc_client.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_executable(fnet_echo_client_app SOURCES echo_client.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_executable(fnet_rpc_info_app SOURCES @@ -23,19 +23,19 @@ vespa_add_executable(fnet_rpc_info_app OUTPUT_NAME vespa-rpc-info INSTALL bin DEPENDS - fnet + vespa_fnet ) vespa_add_executable(fnet_rpc_callback_server_app SOURCES rpc_callback_server.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_executable(fnet_rpc_callback_client_app SOURCES rpc_callback_client.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_executable(fnet_rpc_invoke_app SOURCES @@ -43,5 +43,5 @@ vespa_add_executable(fnet_rpc_invoke_app OUTPUT_NAME vespa-rpc-invoke-bin INSTALL bin DEPENDS - fnet + vespa_fnet ) diff --git a/fnet/src/examples/ping/CMakeLists.txt b/fnet/src/examples/ping/CMakeLists.txt index 8256fcefe28..f215fbe7426 100644 --- a/fnet/src/examples/ping/CMakeLists.txt +++ b/fnet/src/examples/ping/CMakeLists.txt @@ -4,12 +4,12 @@ vespa_add_executable(fnet_pingserver_app packets.cpp pingserver.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_executable(fnet_pingclient_app SOURCES packets.cpp pingclient.cpp DEPENDS - fnet + vespa_fnet ) diff --git a/fnet/src/examples/timeout/CMakeLists.txt b/fnet/src/examples/timeout/CMakeLists.txt index 28b74bbc3b8..944d3f143d9 100644 --- a/fnet/src/examples/timeout/CMakeLists.txt +++ b/fnet/src/examples/timeout/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_executable(fnet_timeout_app SOURCES timeout.cpp DEPENDS - fnet + vespa_fnet ) diff --git a/fnet/src/tests/connect/CMakeLists.txt b/fnet/src/tests/connect/CMakeLists.txt index 41702292744..eeaa8f9b410 100644 --- a/fnet/src/tests/connect/CMakeLists.txt +++ b/fnet/src/tests/connect/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_connect_test_app TEST SOURCES connect_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_connect_test_app COMMAND fnet_connect_test_app) diff --git a/fnet/src/tests/connection_spread/CMakeLists.txt b/fnet/src/tests/connection_spread/CMakeLists.txt index 12ad0ab87bb..e221a194c76 100644 --- a/fnet/src/tests/connection_spread/CMakeLists.txt +++ b/fnet/src/tests/connection_spread/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_connection_spread_test_app TEST SOURCES connection_spread_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_connection_spread_test_app COMMAND fnet_connection_spread_test_app) diff --git a/fnet/src/tests/databuffer/CMakeLists.txt b/fnet/src/tests/databuffer/CMakeLists.txt index 9f5232b62a6..781abd2703e 100644 --- a/fnet/src/tests/databuffer/CMakeLists.txt +++ b/fnet/src/tests/databuffer/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_databuffer_test_app TEST SOURCES databuffer.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_databuffer_test_app COMMAND fnet_databuffer_test_app) diff --git a/fnet/src/tests/examples/CMakeLists.txt b/fnet/src/tests/examples/CMakeLists.txt index 2260bc1c323..f6d73849a0e 100644 --- a/fnet/src/tests/examples/CMakeLists.txt +++ b/fnet/src/tests/examples/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_examples_test_app TEST SOURCES examples_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_examples_test_app NO_VALGRIND COMMAND fnet_examples_test_app) diff --git a/fnet/src/tests/frt/detach_supervisor/CMakeLists.txt b/fnet/src/tests/frt/detach_supervisor/CMakeLists.txt index 3535a469ce8..5aa1febdbb3 100644 --- a/fnet/src/tests/frt/detach_supervisor/CMakeLists.txt +++ b/fnet/src/tests/frt/detach_supervisor/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_detach_supervisor_test_app TEST SOURCES detach_supervisor_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_detach_supervisor_test_app COMMAND fnet_detach_supervisor_test_app) diff --git a/fnet/src/tests/frt/method_pt/CMakeLists.txt b/fnet/src/tests/frt/method_pt/CMakeLists.txt index 17884a8b2d0..9ffce297edf 100644 --- a/fnet/src/tests/frt/method_pt/CMakeLists.txt +++ b/fnet/src/tests/frt/method_pt/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_method_pt_test_app TEST SOURCES method_pt.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_method_pt_test_app COMMAND fnet_method_pt_test_app) diff --git a/fnet/src/tests/frt/parallel_rpc/CMakeLists.txt b/fnet/src/tests/frt/parallel_rpc/CMakeLists.txt index 6a89f7bebc2..5c55fefeeb6 100644 --- a/fnet/src/tests/frt/parallel_rpc/CMakeLists.txt +++ b/fnet/src/tests/frt/parallel_rpc/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(fnet_parallel_rpc_test_app TEST SOURCES parallel_rpc_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_parallel_rpc_test_app COMMAND fnet_parallel_rpc_test_app) vespa_add_executable(fnet_tls_rpc_bench_app TEST SOURCES tls_rpc_bench.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_tls_rpc_bench_app COMMAND fnet_tls_rpc_bench_app BENCHMARK) diff --git a/fnet/src/tests/frt/rpc/CMakeLists.txt b/fnet/src/tests/frt/rpc/CMakeLists.txt index e424bcee590..48292a2df56 100644 --- a/fnet/src/tests/frt/rpc/CMakeLists.txt +++ b/fnet/src/tests/frt/rpc/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(fnet_invoke_test_app TEST SOURCES invoke.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_invoke_test_app COMMAND fnet_invoke_test_app) vespa_add_test(NAME fnet_invoke_test_app_tls COMMAND fnet_invoke_test_app ENVIRONMENT "CRYPTOENGINE=tls") @@ -19,13 +19,13 @@ vespa_add_executable(fnet_detach_return_invoke_test_app TEST SOURCES detach_return_invoke.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_detach_return_invoke_test_app COMMAND fnet_detach_return_invoke_test_app) vespa_add_executable(fnet_sharedblob_test_app TEST SOURCES sharedblob.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_sharedblob_test_app COMMAND fnet_sharedblob_test_app) diff --git a/fnet/src/tests/frt/values/CMakeLists.txt b/fnet/src/tests/frt/values/CMakeLists.txt index 62b3e44a19a..31ffc80af46 100644 --- a/fnet/src/tests/frt/values/CMakeLists.txt +++ b/fnet/src/tests/frt/values/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_values_test_app TEST SOURCES values_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_values_test_app COMMAND fnet_values_test_app) diff --git a/fnet/src/tests/info/CMakeLists.txt b/fnet/src/tests/info/CMakeLists.txt index aa7570cd2f3..ea4c1e0a48c 100644 --- a/fnet/src/tests/info/CMakeLists.txt +++ b/fnet/src/tests/info/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_info_test_app TEST SOURCES info.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_info_test_app COMMAND fnet_info_test_app) diff --git a/fnet/src/tests/locking/CMakeLists.txt b/fnet/src/tests/locking/CMakeLists.txt index 854be39801e..e4b69a653fa 100644 --- a/fnet/src/tests/locking/CMakeLists.txt +++ b/fnet/src/tests/locking/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(fnet_drainpackets_test_app TEST SOURCES drainpackets.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_drainpackets_test_app NO_VALGRIND COMMAND fnet_drainpackets_test_app) vespa_add_executable(fnet_lockspeed_test_app TEST @@ -11,13 +11,13 @@ vespa_add_executable(fnet_lockspeed_test_app TEST lockspeed.cpp dummy.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_lockspeed_test_app NO_VALGRIND COMMAND fnet_lockspeed_test_app) vespa_add_executable(fnet_castspeed_test_app TEST SOURCES castspeed.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_castspeed_test_app NO_VALGRIND COMMAND fnet_castspeed_test_app) diff --git a/fnet/src/tests/locking/lockspeed.cpp b/fnet/src/tests/locking/lockspeed.cpp index 78fe0869e22..eaf79ff38d4 100644 --- a/fnet/src/tests/locking/lockspeed.cpp +++ b/fnet/src/tests/locking/lockspeed.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include "dummy.h" #include <chrono> +#include <condition_variable> class SpinLock { private: @@ -202,16 +203,16 @@ TEST("lock speed") { start = clock::now(); for (i = 0; i < 1000000; i++) { - DummyObj *dummy0 = new DummyObj(); - DummyObj *dummy1 = new DummyObj(); - DummyObj *dummy2 = new DummyObj(); - DummyObj *dummy3 = new DummyObj(); - DummyObj *dummy4 = new DummyObj(); - DummyObj *dummy5 = new DummyObj(); - DummyObj *dummy6 = new DummyObj(); - DummyObj *dummy7 = new DummyObj(); - DummyObj *dummy8 = new DummyObj(); - DummyObj *dummy9 = new DummyObj(); + auto *dummy0 = new DummyObj(); + auto *dummy1 = new DummyObj(); + auto *dummy2 = new DummyObj(); + auto *dummy3 = new DummyObj(); + auto *dummy4 = new DummyObj(); + auto *dummy5 = new DummyObj(); + auto *dummy6 = new DummyObj(); + auto *dummy7 = new DummyObj(); + auto *dummy8 = new DummyObj(); + auto *dummy9 = new DummyObj(); delete dummy9; delete dummy8; delete dummy7; diff --git a/fnet/src/tests/printstuff/CMakeLists.txt b/fnet/src/tests/printstuff/CMakeLists.txt index 30d10f4d98c..65e585eace5 100644 --- a/fnet/src/tests/printstuff/CMakeLists.txt +++ b/fnet/src/tests/printstuff/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_printstuff_test_app TEST SOURCES printstuff_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_printstuff_test_app COMMAND fnet_printstuff_test_app) diff --git a/fnet/src/tests/scheduling/CMakeLists.txt b/fnet/src/tests/scheduling/CMakeLists.txt index 224581eda98..16e9930bb97 100644 --- a/fnet/src/tests/scheduling/CMakeLists.txt +++ b/fnet/src/tests/scheduling/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(fnet_schedule_test_app TEST SOURCES schedule.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_schedule_test_app COMMAND fnet_schedule_test_app) vespa_add_executable(fnet_sloweventloop_test_app TEST SOURCES sloweventloop.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_sloweventloop_test_app COMMAND fnet_sloweventloop_test_app) diff --git a/fnet/src/tests/scheduling/schedule.cpp b/fnet/src/tests/scheduling/schedule.cpp index 88b84bec67a..2890cc1d5f0 100644 --- a/fnet/src/tests/scheduling/schedule.cpp +++ b/fnet/src/tests/scheduling/schedule.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/fnet/scheduler.h> #include <vespa/fnet/task.h> +#include <cassert> using vespalib::steady_clock; using vespalib::steady_time; diff --git a/fnet/src/tests/sync_execute/CMakeLists.txt b/fnet/src/tests/sync_execute/CMakeLists.txt index c0482977b0d..29e955875b6 100644 --- a/fnet/src/tests/sync_execute/CMakeLists.txt +++ b/fnet/src/tests/sync_execute/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_sync_execute_test_app TEST SOURCES sync_execute.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_sync_execute_test_app COMMAND fnet_sync_execute_test_app) diff --git a/fnet/src/tests/thread_selection/CMakeLists.txt b/fnet/src/tests/thread_selection/CMakeLists.txt index 428437b0ec0..ec474f73fae 100644 --- a/fnet/src/tests/thread_selection/CMakeLists.txt +++ b/fnet/src/tests/thread_selection/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_thread_selection_test_app TEST SOURCES thread_selection_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_thread_selection_test_app COMMAND fnet_thread_selection_test_app) diff --git a/fnet/src/tests/time/CMakeLists.txt b/fnet/src/tests/time/CMakeLists.txt index 6276286dd8a..0f2c7c3c5c3 100644 --- a/fnet/src/tests/time/CMakeLists.txt +++ b/fnet/src/tests/time/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_timespeed_test_app TEST SOURCES timespeed.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_timespeed_test_app NO_VALGRIND COMMAND fnet_timespeed_test_app) diff --git a/fnet/src/tests/transport_debugger/CMakeLists.txt b/fnet/src/tests/transport_debugger/CMakeLists.txt index a5aebcdbc93..b252c408288 100644 --- a/fnet/src/tests/transport_debugger/CMakeLists.txt +++ b/fnet/src/tests/transport_debugger/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fnet_transport_debugger_test_app TEST SOURCES transport_debugger_test.cpp DEPENDS - fnet + vespa_fnet ) vespa_add_test(NAME fnet_transport_debugger_test_app COMMAND fnet_transport_debugger_test_app) diff --git a/fnet/src/vespa/fnet/CMakeLists.txt b/fnet/src/vespa/fnet/CMakeLists.txt index 3e829785544..492a48c4f33 100644 --- a/fnet/src/vespa/fnet/CMakeLists.txt +++ b/fnet/src/vespa/fnet/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fnet +vespa_add_library(vespa_fnet SOURCES channel.cpp channellookup.cpp diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java index b54e71993c9..f03354f6938 100644 --- a/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java +++ b/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java @@ -34,28 +34,35 @@ import static org.junit.jupiter.api.Assertions.assertTrue; */ class SignaturesTest { - private static final String ecPemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + - "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + - "-----END PUBLIC KEY-----\n"; - - private static final String ecPemPrivateKey = "-----BEGIN EC PRIVATE KEY-----\n" + - "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" + - "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" + - "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + - "-----END EC PRIVATE KEY-----\n"; - - private static final String otherEcPemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + - "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + - "-----END PUBLIC KEY-----\n"; - - private static final byte[] message = ("Hello,\n" + - "\n" + - "this is a secret message.\n" + - "\n" + - "Yours truly,\n" + - "∠( ᐛ 」∠)＿").getBytes(UTF_8); + private static final String ecPemPublicKey = """ + -----BEGIN PUBLIC KEY----- + MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9 + z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw== + -----END PUBLIC KEY----- + """; + + private static final String ecPemPrivateKey = """ + -----BEGIN EC PRIVATE KEY----- + MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49 + AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU + PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw== + -----END EC PRIVATE KEY----- + """; + + private static final String otherEcPemPublicKey = """ + -----BEGIN PUBLIC KEY----- + MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE + pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw== + -----END PUBLIC KEY----- + """; + + private static final byte[] message = (""" + Hello, + + this is a secret message. + + Yours truly, + ∠( ᐛ 」∠)＿""").getBytes(UTF_8); @Test void testHashing() throws Exception { diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index e83d5dd4ce9..60fa96d1fd6 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -54,6 +54,11 @@ <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> <build> <plugins> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java index 05ac73618e8..c2d41edf1b6 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java @@ -187,8 +187,8 @@ public class EmbedExpression extends Expression { if (targetType.rank() == 2 && targetType.mappedSubtype().rank() == 2) { if (embedderArguments.size() != 1) throw new VerificationException(this, "When the embedding target field is a 2d mapped tensor " + - "the name of the tensor dimension that corresponds to the input array elements must " + - "be given as a second argument to embed, e.g: ... | embed splade paragraph | ..."); + "the name of the tensor dimension that corresponds to the input array elements must " + + "be given as a second argument to embed, e.g: ... | embed splade paragraph | ..."); if ( ! targetType.mappedSubtype().dimensionNames().contains(embedderArguments.get(0))) { throw new VerificationException(this, "The dimension '" + embedderArguments.get(0) + "' given to embed " + "is not a sparse dimension of the target type " + targetType); @@ -254,7 +254,7 @@ public class EmbedExpression extends Expression { List<String> embedderIds = new ArrayList<>(); embedders.forEach((key, value) -> embedderIds.add(key)); embedderIds.sort(null); - return String.join(",", embedderIds); + return String.join(", ", embedderIds); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java index f6995ac5a72..dd0ec255c35 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -202,9 +202,9 @@ public class ScriptTestCase { "my input", "[110.0, 122.0, 33.0, 106.0]"); assertThrows(() -> testEmbedStatement("input myText | embed | attribute 'myTensor'", embedders, "input text", "[105, 110, 112, 117]"), - "Multiple embedders are provided but no embedder id is given. Valid embedders are emb1,emb2"); + "Multiple embedders are provided but no embedder id is given. Valid embedders are emb1, emb2"); assertThrows(() -> testEmbedStatement("input myText | embed emb3 | attribute 'myTensor'", embedders, "input text", "[105, 110, 112, 117]"), - "Can't find embedder 'emb3'. Valid embedders are emb1,emb2"); + "Can't find embedder 'emb3'. Valid embedders are emb1, emb2"); } private void testEmbedStatement(String expressionString, Map<String, Embedder> embedders, String input, String expected) { @@ -562,12 +562,12 @@ public class ScriptTestCase { } - private void assertThrows(Runnable r, String msg) { + private void assertThrows(Runnable r, String expectedMessage) { try { r.run(); fail(); } catch (IllegalStateException e) { - assertEquals(e.getMessage(), msg); + assertEquals(expectedMessage, e.getMessage()); } } diff --git a/integration/intellij/build.gradle.kts b/integration/intellij/build.gradle.kts index 6fff4e8f519..14e77ab4896 100644 --- a/integration/intellij/build.gradle.kts +++ b/integration/intellij/build.gradle.kts @@ -4,7 +4,7 @@ import org.jetbrains.grammarkit.tasks.GenerateParserTask plugins { id("java-library") - id("org.jetbrains.intellij") version "1.17.3" + id("org.jetbrains.intellij") version "1.17.4" id("org.jetbrains.grammarkit") version "2022.3.2.2" id("maven-publish") // to deploy the plugin into a Maven repo } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java index 6a106c6aa3f..515297ad20c 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudDataPlaneFilter.java @@ -2,6 +2,7 @@ package com.yahoo.jdisc.http.filter.security.cloud; import com.yahoo.component.annotation.Inject; +import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; @@ -31,6 +32,7 @@ import static com.yahoo.jdisc.http.filter.security.cloud.Permission.WRITE; * * @author bjorncs */ +@Provides("Vespa Cloud mTLS Authorization Filter") public class CloudDataPlaneFilter extends JsonSecurityRequestFilterBase { private static final Logger log = Logger.getLogger(CloudDataPlaneFilter.class.getName()); diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java index 699aa5c9187..d8263301049 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cloud/CloudTokenDataPlaneFilter.java @@ -2,6 +2,7 @@ package com.yahoo.jdisc.http.filter.security.cloud; import com.yahoo.component.annotation.Inject; +import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; @@ -32,6 +33,7 @@ import static com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler.CONT * * @author bjorncs */ +@Provides("Vespa Cloud Token Authorization Filter") public class CloudTokenDataPlaneFilter extends JsonSecurityRequestFilterBase { private static final Logger log = Logger.getLogger(CloudTokenDataPlaneFilter.class.getName()); diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/User.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/User.java index b573cf0c04a..616b9df4bd8 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/User.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/User.java @@ -2,95 +2,74 @@ package com.yahoo.jdisc.http.filter.security.misc; import java.time.LocalDate; +import java.util.Map; import java.util.Objects; +import java.util.TreeMap; /** * @author smorgrav */ -public class User { - +public record User(String email, String name, String nickname, String picture, boolean isVerified, int loginCount, + LocalDate lastLogin, Map<String, Object> extraAttributes) { public static final String ATTRIBUTE_NAME = "vespa.user.attributes"; public static final LocalDate NO_DATE = LocalDate.EPOCH; - private final String email; - private final String name; - private final String nickname; - private final String picture; - private final boolean isVerified; - private final int loginCount; - private final LocalDate lastLogin; - - public User(String email, String name, String nickname, String picture) { - this.email = Objects.requireNonNull(email); - this.name = name; - this.nickname = nickname; - this.picture = picture; - this.isVerified = false; - this.loginCount = -1; - this.lastLogin = NO_DATE; - } - - public User(String email, String name, String nickname, String picture, boolean isVerified, int loginCount, LocalDate lastLogin) { - this.email = Objects.requireNonNull(email); - this.name = name; - this.nickname = nickname; - this.picture = picture; - this.isVerified = isVerified; - this.loginCount = loginCount; - this.lastLogin = Objects.requireNonNull(lastLogin); - } - - public String name() { - return name; + public User { + Objects.requireNonNull(email); + Objects.requireNonNull(lastLogin); + extraAttributes = Map.copyOf(Objects.requireNonNull(extraAttributes)); } - public String email() { - return email; + public User(String email, String name, String nickname, String picture, boolean isVerified, int loginCount, + LocalDate lastLogin) { + this(email, name, nickname, picture, isVerified, loginCount, lastLogin, Map.of()); } - public String nickname() { - return nickname; - } - - public String picture() { - return picture; - } - - public LocalDate lastLogin() { return lastLogin; } - - public boolean isVerified() { return isVerified; } - - public int loginCount() { return loginCount; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - User user = (User) o; - return Objects.equals(name, user.name) && - Objects.equals(email, user.email) && - Objects.equals(nickname, user.nickname) && - Objects.equals(picture, user.picture) && - Objects.equals(lastLogin, user.lastLogin) && - loginCount == user.loginCount && - isVerified == user.isVerified; + public User(String email, String name, String nickname, String picture) { + this(email, name, nickname, picture, false, -1, NO_DATE, Map.of()); } - @Override - public int hashCode() { - return Objects.hash(name, email, nickname, picture, lastLogin, loginCount, isVerified); + private User(Builder builder) { + this(builder.email, builder.name, builder.nickname, builder.picture, builder.isVerified, builder.loginCount, + Objects.requireNonNullElse(builder.lastLogin, NO_DATE), builder.extraAttributes); } - @Override - public String toString() { - return "User{" + - "email='" + email + '\'' + - ", name='" + name + '\'' + - ", nickname='" + nickname + '\'' + - ", picture='" + picture + '\'' + - ", isVerified=" + isVerified + - ", loginCount=" + loginCount + - ", lastLogin=" + lastLogin + - '}'; + public static Builder builder() { return new Builder(); } + public static Builder builder(User u) { return new Builder(u); } + + public static class Builder { + private String email; + private String name; + private String nickname; + private String picture; + private boolean isVerified; + private int loginCount; + private LocalDate lastLogin; + private final Map<String, Object> extraAttributes = new TreeMap<>(); + + private Builder() {} + + private Builder(User u) { + email = u.email; + name = u.name; + nickname = u.nickname; + picture = u.picture; + isVerified = u.isVerified; + loginCount = u.loginCount; + lastLogin = u.lastLogin; + extraAttributes.putAll(u.extraAttributes); + } + + public Builder email(String email) { this.email = email; return this; } + public Builder name(String name) { this.name = name; return this; } + public Builder nickname(String nickname) { this.nickname = nickname; return this; } + public Builder picture(String picture) { this.picture = picture; return this; } + public Builder isVerified(boolean isVerified) { this.isVerified = isVerified; return this; } + public Builder loginCount(int loginCount) { this.loginCount = loginCount; return this; } + public Builder lastLogin(LocalDate lastLogin) { this.lastLogin = lastLogin; return this; } + public Builder extraAttribute(String key, Object value) { + extraAttributes.put(Objects.requireNonNull(key), Objects.requireNonNull(value)); return this; + } + public User build() { return new User(this); } } } diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties index 877edc19320..c393c64314f 100644 --- a/jdisc_core/src/test/resources/exportPackages.properties +++ b/jdisc_core/src/test/resources/exportPackages.properties @@ -1,3 +1,3 @@ #generated by com.yahoo.jdisc.core.ExportPackages #Fri Jul 07 16:04:11 CEST 2023 -exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", java.util.jar; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.security.auth.callback; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="33.2.0",com.google.common.base;version\="33.2.0";uses\:\="javax.annotation",com.google.common.cache;version\="33.2.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent,javax.annotation",com.google.common.collect;version\="33.2.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.escape;version\="33.2.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.eventbus;version\="33.2.0",com.google.common.graph;version\="33.2.0";uses\:\="com.google.common.collect,javax.annotation",com.google.common.hash;version\="33.2.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.html;version\="33.2.0";uses\:\="com.google.common.escape",com.google.common.io;version\="33.2.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash,javax.annotation",com.google.common.math;version\="33.2.0";uses\:\="javax.annotation",com.google.common.net;version\="33.2.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape,javax.annotation",com.google.common.primitives;version\="33.2.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.reflect;version\="33.2.0";uses\:\="com.google.common.collect,com.google.common.io,javax.annotation",com.google.common.util.concurrent;version\="33.2.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal,javax.annotation",com.google.common.xml;version\="33.2.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, com.sun.jna;version\=5.11.0, com.sun.jna.ptr;version\=5.11.0, com.sun.jna.win32;version\=5.11.0, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.llm.client.openai;version\=1.0.0,ai.vespa.llm.completion;version\=1.0.0,ai.vespa.llm.test;version\=1.0.0,ai.vespa.llm;version\=1.0.0,ai.vespa.net;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.helpers;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" +exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", java.util.jar; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.security.auth.callback; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="33.2.1",com.google.common.base;version\="33.2.1";uses\:\="javax.annotation",com.google.common.cache;version\="33.2.1";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent,javax.annotation",com.google.common.collect;version\="33.2.1";uses\:\="com.google.common.base,javax.annotation",com.google.common.escape;version\="33.2.1";uses\:\="com.google.common.base,javax.annotation",com.google.common.eventbus;version\="33.2.1",com.google.common.graph;version\="33.2.1";uses\:\="com.google.common.collect,javax.annotation",com.google.common.hash;version\="33.2.1";uses\:\="com.google.common.base,javax.annotation",com.google.common.html;version\="33.2.1";uses\:\="com.google.common.escape",com.google.common.io;version\="33.2.1";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash,javax.annotation",com.google.common.math;version\="33.2.1";uses\:\="javax.annotation",com.google.common.net;version\="33.2.1";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape,javax.annotation",com.google.common.primitives;version\="33.2.1";uses\:\="com.google.common.base,javax.annotation",com.google.common.reflect;version\="33.2.1";uses\:\="com.google.common.collect,com.google.common.io,javax.annotation",com.google.common.util.concurrent;version\="33.2.1";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal,javax.annotation",com.google.common.xml;version\="33.2.1";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, com.sun.jna;version\=5.11.0, com.sun.jna.ptr;version\=5.11.0, com.sun.jna.win32;version\=5.11.0, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.llm.client.openai;version\=1.0.0,ai.vespa.llm.completion;version\=1.0.0,ai.vespa.llm.test;version\=1.0.0,ai.vespa.llm;version\=1.0.0,ai.vespa.net;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.helpers;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" diff --git a/jrt_test/CMakeLists.txt b/jrt_test/CMakeLists.txt index 6c06774da61..27bd4ebee63 100644 --- a/jrt_test/CMakeLists.txt +++ b/jrt_test/CMakeLists.txt @@ -3,9 +3,9 @@ vespa_define_module( DEPENDS vespalog vespalib - fnet - slobrok - configdefinitions + vespa_fnet + vespa_slobrok + vespa_configdefinitions APPS src/binref diff --git a/jrt_test/src/tests/rpc-error/CMakeLists.txt b/jrt_test/src/tests/rpc-error/CMakeLists.txt index a86289c73bb..b0aab8a0eb9 100644 --- a/jrt_test/src/tests/rpc-error/CMakeLists.txt +++ b/jrt_test/src/tests/rpc-error/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(jrt_test_test-errors_app TEST SOURCES test-errors.cpp DEPENDS + GTest::gtest ) vespa_add_test(NAME jrt_test_test-errors_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/rpc-error_test.sh DEPENDS jrt_test_test-errors_app) diff --git a/jrt_test/src/tests/rpc-error/test-errors.cpp b/jrt_test/src/tests/rpc-error/test-errors.cpp index f7b8ff10185..cecdd09d718 100644 --- a/jrt_test/src/tests/rpc-error/test-errors.cpp +++ b/jrt_test/src/tests/rpc-error/test-errors.cpp @@ -1,171 +1,156 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> + #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> #include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/ref_counted.h> + +vespalib::string spec; -class TestErrors : public vespalib::TestApp +class TestErrors : public ::testing::Test { -private: - fnet::frt::StandaloneFRT server; - FRT_Supervisor *client; - FRT_Target *target; +protected: + static std::unique_ptr<fnet::frt::StandaloneFRT> server; + static vespalib::ref_counted<FRT_Target> target; + vespalib::ref_counted<FRT_RPCRequest> alloc_rpc_request() { + return vespalib::ref_counted<FRT_RPCRequest>::internal_attach(server->supervisor().AllocRPCRequest()); + } public: TestErrors(); ~TestErrors() override; - void init(const char *spec) { - client = & server.supervisor(); - target = client->GetTarget(spec); - } - void fini() { - target->internal_subref(); - target = nullptr; - client = nullptr; - } - - void testNoError(); - void testNoSuchMethod(); - void testWrongParameters(); - void testWrongReturnValues(); - void testMethodFailed(); - int Main() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); }; +std::unique_ptr<fnet::frt::StandaloneFRT> TestErrors::server; +vespalib::ref_counted<FRT_Target> TestErrors::target; + TestErrors::TestErrors() = default; TestErrors::~TestErrors() = default; void -TestErrors::testNoError() +TestErrors::SetUpTestSuite() { - FRT_RPCRequest *req1 = client->AllocRPCRequest(); + server = std::make_unique<fnet::frt::StandaloneFRT>(); + target = vespalib::ref_counted<FRT_Target>::internal_attach(server->supervisor().GetTarget(spec.c_str())); +} + +void +TestErrors::TearDownTestSuite() +{ + target.reset(); + server.reset(); +} + +TEST_F(TestErrors, no_error) +{ + auto req1 = alloc_rpc_request(); req1->SetMethodName("test"); req1->GetParams()->AddInt32(42); req1->GetParams()->AddInt32(0); req1->GetParams()->AddInt8(0); - target->InvokeSync(req1, 60.0); + target->InvokeSync(req1.get(), 60.0); EXPECT_TRUE(!req1->IsError()); - if (EXPECT_TRUE(1 == req1->GetReturn()->GetNumValues())) { - EXPECT_TRUE(42 == req1->GetReturn()->GetValue(0)._intval32); - } else { - EXPECT_TRUE(false); - } - req1->internal_subref(); + ASSERT_EQ(1, req1->GetReturn()->GetNumValues()); + ASSERT_EQ(42, req1->GetReturn()->GetValue(0)._intval32); } -void -TestErrors::testNoSuchMethod() +TEST_F(TestErrors, no_such_method) { - FRT_RPCRequest *req1 = client->AllocRPCRequest(); + auto req1 = alloc_rpc_request(); req1->SetMethodName("bogus"); - target->InvokeSync(req1, 60.0); + target->InvokeSync(req1.get(), 60.0); EXPECT_TRUE(req1->IsError()); EXPECT_TRUE(0 == req1->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_NO_SUCH_METHOD == req1->GetErrorCode()); - req1->internal_subref(); } -void -TestErrors::testWrongParameters() +TEST_F(TestErrors, wrong_parameters) { - FRT_RPCRequest *req1 = client->AllocRPCRequest(); + auto req1 = alloc_rpc_request(); req1->SetMethodName("test"); req1->GetParams()->AddInt32(42); req1->GetParams()->AddInt32(0); req1->GetParams()->AddInt32(0); - target->InvokeSync(req1, 60.0); + target->InvokeSync(req1.get(), 60.0); EXPECT_TRUE(req1->IsError()); - EXPECT_TRUE(0 == req1->GetReturn()->GetNumValues()); + EXPECT_EQ(0, req1->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_WRONG_PARAMS == req1->GetErrorCode()); - req1->internal_subref(); + req1.reset(); - FRT_RPCRequest *req2 = client->AllocRPCRequest(); + auto req2 = alloc_rpc_request(); req2->SetMethodName("test"); req2->GetParams()->AddInt32(42); req2->GetParams()->AddInt32(0); - target->InvokeSync(req2, 60.0); + target->InvokeSync(req2.get(), 60.0); EXPECT_TRUE(req2->IsError()); - EXPECT_TRUE(0 == req2->GetReturn()->GetNumValues()); + EXPECT_EQ(0, req2->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_WRONG_PARAMS == req2->GetErrorCode()); - req2->internal_subref(); + req2.reset(); - FRT_RPCRequest *req3 = client->AllocRPCRequest(); + auto req3 = alloc_rpc_request(); req3->SetMethodName("test"); req3->GetParams()->AddInt32(42); req3->GetParams()->AddInt32(0); req3->GetParams()->AddInt8(0); req3->GetParams()->AddInt8(0); - target->InvokeSync(req3, 60.0); + target->InvokeSync(req3.get(), 60.0); EXPECT_TRUE(req3->IsError()); - EXPECT_TRUE(0 == req3->GetReturn()->GetNumValues()); + EXPECT_EQ(0, req3->GetReturn()->GetNumValues()); EXPECT_TRUE(FRTE_RPC_WRONG_PARAMS == req3->GetErrorCode()); - req3->internal_subref(); } -void -TestErrors::testWrongReturnValues() +TEST_F(TestErrors, wrong_return_values) { - FRT_RPCRequest *req1 = client->AllocRPCRequest(); + auto req1 = alloc_rpc_request(); req1->SetMethodName("test"); req1->GetParams()->AddInt32(42); req1->GetParams()->AddInt32(0); req1->GetParams()->AddInt8(1); - target->InvokeSync(req1, 60.0); + target->InvokeSync(req1.get(), 60.0); EXPECT_TRUE(req1->IsError()); - EXPECT_TRUE(0 == req1->GetReturn()->GetNumValues()); - EXPECT_TRUE(FRTE_RPC_WRONG_RETURN == req1->GetErrorCode()); - req1->internal_subref(); + EXPECT_EQ(0, req1->GetReturn()->GetNumValues()); + EXPECT_TRUE(FRTE_RPC_WRONG_RETURN == req1->GetErrorCode()); } -void -TestErrors::testMethodFailed() +TEST_F(TestErrors, method_failed) { - FRT_RPCRequest *req1 = client->AllocRPCRequest(); + auto req1 = alloc_rpc_request(); req1->SetMethodName("test"); req1->GetParams()->AddInt32(42); req1->GetParams()->AddInt32(75000); req1->GetParams()->AddInt8(0); - target->InvokeSync(req1, 60.0); + target->InvokeSync(req1.get(), 60.0); EXPECT_TRUE(req1->IsError()); - EXPECT_TRUE(0 == req1->GetReturn()->GetNumValues()); - EXPECT_TRUE(75000 == req1->GetErrorCode()); - req1->internal_subref(); + EXPECT_EQ(0, req1->GetReturn()->GetNumValues()); + EXPECT_EQ(75000, req1->GetErrorCode()); - FRT_RPCRequest *req2 = client->AllocRPCRequest(); + auto req2 = alloc_rpc_request(); req2->SetMethodName("test"); req2->GetParams()->AddInt32(42); req2->GetParams()->AddInt32(75000); req2->GetParams()->AddInt8(1); - target->InvokeSync(req2, 60.0); + target->InvokeSync(req2.get(), 60.0); EXPECT_TRUE(req2->IsError()); - EXPECT_TRUE(0 == req2->GetReturn()->GetNumValues()); - EXPECT_TRUE(75000 == req2->GetErrorCode()); - req2->internal_subref(); + EXPECT_EQ(0, req2->GetReturn()->GetNumValues()); + EXPECT_EQ(75000, req2->GetErrorCode()); } - int -TestErrors::Main() +main(int argc, char* argv[]) { - if (_argc != 2) { - fprintf(stderr, "usage: %s spec", _argv[0]); + if (argc != 2) { + fprintf(stderr, "usage: %s spec\n", argv[0]); return 1; } - TEST_INIT("test_errors"); - init(_argv[1]); - testNoError(); - testNoSuchMethod(); - testWrongParameters(); - testWrongReturnValues(); - testMethodFailed(); - fini(); - TEST_DONE(); + spec = argv[1]; + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } - - -TEST_APPHOOK(TestErrors); diff --git a/linguistics/pom.xml b/linguistics/pom.xml index d07ff5d9fdb..40767bfff26 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -14,15 +14,19 @@ <packaging>container-plugin</packaging> <version>8-SNAPSHOT</version> <dependencies> + <!-- Compile Scope Dependencies --> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> + <scope>compile</scope> </dependency> + + <!-- Provided Scope Dependencies --> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -38,19 +42,19 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>annotations</artifactId> + <artifactId>configdefinitions</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>configdefinitions</artifactId> - <version>${project.version}</version> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>com.google.inject</groupId> @@ -58,14 +62,22 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> <scope>provided</scope> </dependency> + + <!-- Test Scope Dependencies --> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <scope>provided</scope> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.junit.vintage</groupId> @@ -73,8 +85,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter</artifactId> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> </dependencies> diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModel.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModel.java index 3244b8373ad..6e024c3025e 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModel.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModel.java @@ -8,6 +8,8 @@ import com.yahoo.language.significance.SignificanceModel; import java.io.IOException; import java.nio.file.Path; import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; /** * @@ -15,7 +17,7 @@ import java.util.HashMap; */ public class DefaultSignificanceModel implements SignificanceModel { private final long corpusSize; - private final HashMap<String, Long> frequencies; + private final Map<String, Long> frequencies; private String id; diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java index 72874c15d9e..6f3a108e9e1 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java @@ -7,9 +7,12 @@ import com.yahoo.language.Language; import com.yahoo.language.significance.SignificanceModel; import com.yahoo.language.significance.SignificanceModelRegistry; import com.yahoo.search.significance.config.SignificanceConfig; +import io.airlift.compress.zstd.ZstdInputStream; import java.io.IOException; import java.io.UncheckedIOException; +import java.io.FileInputStream; +import java.io.InputStream; import java.nio.file.Path; import java.util.EnumMap; import java.util.List; @@ -44,7 +47,11 @@ public class DefaultSignificanceModelRegistry implements SignificanceModelRegist public void addModel(Path path) { ObjectMapper objectMapper = new ObjectMapper(); try { - SignificanceModelFile file = objectMapper.readValue(path.toFile(), SignificanceModelFile.class); + InputStream in = path.toString().endsWith(".zst") ? + new ZstdInputStream(new FileInputStream(path.toFile())) : + new FileInputStream(path.toFile()); + + SignificanceModelFile file = objectMapper.readValue(in, SignificanceModelFile.class); for (var pair : file.languages().entrySet()) { this.models.put( Language.fromLanguageTag(pair.getKey()), diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/DocumentFrequencyFile.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/DocumentFrequencyFile.java index 9b7cbae834a..34e73e1b547 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/DocumentFrequencyFile.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/DocumentFrequencyFile.java @@ -7,6 +7,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; /** * @@ -19,13 +21,13 @@ public class DocumentFrequencyFile { private final long documentCount; - private final HashMap<String, Long> frequencies; + private final Map<String, Long> frequencies; @JsonCreator public DocumentFrequencyFile( @JsonProperty("description") String description, @JsonProperty("document-count") long documentCount, - @JsonProperty("document-frequencies") HashMap<String, Long> frequencies) { + @JsonProperty("document-frequencies") Map<String, Long> frequencies) { this.description = description; this.documentCount = documentCount; this.frequencies = frequencies; @@ -38,5 +40,5 @@ public class DocumentFrequencyFile { public long documentCount() { return documentCount; } @JsonProperty("document-frequencies") - public HashMap<String, Long> frequencies() { return frequencies; } + public Map<String, Long> frequencies() { return frequencies; } } diff --git a/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java b/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java index e8594885b9e..a5be567717e 100644 --- a/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java +++ b/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java @@ -55,6 +55,27 @@ public class DefaultSignificanceModelRegistryTest { } @Test + public void testDefaultSignificanceModelRegistryWithZSTDecompressing() { + List<Path> models = new ArrayList<>(); + + models.add(Path.of("src/test/models/docv1.json.zst")); + + DefaultSignificanceModelRegistry defaultSignificanceModelRegistry = new DefaultSignificanceModelRegistry(models); + + var optionalEnglishModel = defaultSignificanceModelRegistry.getModel(Language.ENGLISH); + assertTrue(optionalEnglishModel.isPresent()); + + var englishModel = optionalEnglishModel.get(); + + assertTrue( defaultSignificanceModelRegistry.getModel(Language.FRENCH).isEmpty()); + assertNotNull(englishModel); + assertEquals("test::1", englishModel.getId()); + assertEquals(2, englishModel.documentFrequency("test").frequency()); + assertEquals(10, englishModel.documentFrequency("test").corpusSize()); + + } + + @Test public void testDefaultSignificanceModelRegistryInOppsiteOrder() { List<Path> models = new ArrayList<>(); diff --git a/linguistics/src/test/models/docv1.json.zst b/linguistics/src/test/models/docv1.json.zst Binary files differnew file mode 100644 index 00000000000..61de6c7cb72 --- /dev/null +++ b/linguistics/src/test/models/docv1.json.zst diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index e80f821a426..d896c50f3d3 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_define_module( DEPENDS vespalog vespalib - config_cloudconfig + vespa_config APPS src/apps/logd diff --git a/logforwarder/CMakeLists.txt b/logforwarder/CMakeLists.txt index b8f7fb3c416..3d9c79fb6e7 100644 --- a/logforwarder/CMakeLists.txt +++ b/logforwarder/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_define_module( DEPENDS vespalog vespalib - config_cloudconfig + vespa_config APPS src/apps/vespa-logforwarder-start diff --git a/logforwarder/src/apps/vespa-logforwarder-start/CMakeLists.txt b/logforwarder/src/apps/vespa-logforwarder-start/CMakeLists.txt index bbe4f2c629e..b6258481658 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/CMakeLists.txt +++ b/logforwarder/src/apps/vespa-logforwarder-start/CMakeLists.txt @@ -9,7 +9,7 @@ vespa_add_executable(logforwarder-start_app OUTPUT_NAME vespa-logforwarder-start INSTALL bin DEPENDS - config_cloudconfig - configdefinitions + vespa_config + vespa_configdefinitions vespalib ) diff --git a/logforwarder/src/apps/vespa-otelcol-start/CMakeLists.txt b/logforwarder/src/apps/vespa-otelcol-start/CMakeLists.txt index d95ce0584c9..a2af9cada87 100644 --- a/logforwarder/src/apps/vespa-otelcol-start/CMakeLists.txt +++ b/logforwarder/src/apps/vespa-otelcol-start/CMakeLists.txt @@ -9,7 +9,7 @@ vespa_add_executable(otelcol_start_app OUTPUT_NAME vespa-otelcol-start INSTALL bin DEPENDS - config_cloudconfig - configdefinitions + vespa_config + vespa_configdefinitions vespalib ) diff --git a/logforwarder/src/apps/vespa-otelcol-start/cf-handler.cpp b/logforwarder/src/apps/vespa-otelcol-start/cf-handler.cpp index 9579271f88a..1d0dd2cc544 100644 --- a/logforwarder/src/apps/vespa-otelcol-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-otelcol-start/cf-handler.cpp @@ -3,7 +3,6 @@ #include "cf-handler.h" #include <vespa/config/common/configcontext.h> #include <vespa/config/common/configsystem.h> -#include <vespa/config/common/exceptions.h> #include <vespa/config/helper/legacy.h> #include <vespa/config/subscription/configsubscriber.hpp> @@ -52,16 +51,5 @@ constexpr std::chrono::milliseconds CONFIG_TIMEOUT_MS(30 * 1000); void CfHandler::start(const std::string &configId) { LOG(debug, "Reading configuration with id '%s'", configId.c_str()); - try { - subscribe(configId, CONFIG_TIMEOUT_MS); - } catch (config::ConfigTimeoutException & ex) { - LOG(warning, "Timout getting config, please check your setup. Will exit and restart: %s", ex.getMessage().c_str()); - std::_Exit(EXIT_FAILURE); - } catch (config::InvalidConfigException& ex) { - LOG(error, "Fatal: Invalid configuration, please check your setup: %s", ex.getMessage().c_str()); - std::_Exit(EXIT_FAILURE); - } catch (config::ConfigRuntimeException& ex) { - LOG(error, "Fatal: Could not get config, please check your setup: %s", ex.getMessage().c_str()); - std::_Exit(EXIT_FAILURE); - } + subscribe(configId, CONFIG_TIMEOUT_MS); } diff --git a/logforwarder/src/apps/vespa-otelcol-start/child-handler.cpp b/logforwarder/src/apps/vespa-otelcol-start/child-handler.cpp index 46ac3bea60e..a69f090e4df 100644 --- a/logforwarder/src/apps/vespa-otelcol-start/child-handler.cpp +++ b/logforwarder/src/apps/vespa-otelcol-start/child-handler.cpp @@ -58,9 +58,18 @@ void ChildHandler::startChild(const std::string &progPath, const std::string &cf return; } if (child == 0) { - std::string cfArg{"--config=file:" + cfPath}; - const char *cargv[] = { progPath.c_str(), cfArg.c_str(), nullptr }; - execv(progPath.c_str(), const_cast<char **>(cargv)); + std::string cfgPrefix{"--config=file:"}; + const char *gwCfg = "/etc/otelcol/gw-config.yaml"; + std::string cfArg1{cfgPrefix + gwCfg}; + std::string cfArg2{cfgPrefix + cfPath}; + if (access(gwCfg, R_OK) == 0) { + const char *cargv[] = { progPath.c_str(), cfArg1.c_str(), cfArg2.c_str(), nullptr }; + execv(progPath.c_str(), const_cast<char **>(cargv)); + } else { + fprintf(stderr, "info\tMissing config file: %s (running without it)\n", gwCfg); + const char *cargv[] = { progPath.c_str(), cfArg2.c_str(), nullptr }; + execv(progPath.c_str(), const_cast<char **>(cargv)); + } // if execv fails: perror(progPath.c_str()); std::_Exit(1); diff --git a/logforwarder/src/apps/vespa-otelcol-start/main.cpp b/logforwarder/src/apps/vespa-otelcol-start/main.cpp index 2e3e0659707..9310a0ceffa 100644 --- a/logforwarder/src/apps/vespa-otelcol-start/main.cpp +++ b/logforwarder/src/apps/vespa-otelcol-start/main.cpp @@ -3,6 +3,7 @@ #include "wrapper.h" #include <csignal> #include <unistd.h> +#include <vespa/config/common/exceptions.h> #include <vespa/vespalib/util/sig_catch.h> #include <vespa/defaults.h> @@ -12,10 +13,21 @@ LOG_SETUP("vespa-otelcol-start"); static void run(const char *configId) { vespalib::SigCatch catcher; Wrapper handler(configId); - handler.start(configId); - while (! catcher.receivedStopSignal()) { - handler.check(); - usleep(125000); // Avoid busy looping; + try { + handler.start(configId); + while (! catcher.receivedStopSignal()) { + handler.check(); + usleep(125000); // Avoid busy looping; + } + } catch (config::ConfigTimeoutException & ex) { + LOG(warning, "Timout getting config, please check your setup. Will exit and restart: %s", ex.getMessage().c_str()); + std::_Exit(EXIT_FAILURE); + } catch (config::InvalidConfigException& ex) { + LOG(error, "Fatal: Invalid configuration, please check your setup: %s", ex.getMessage().c_str()); + std::_Exit(EXIT_FAILURE); + } catch (config::ConfigRuntimeException& ex) { + LOG(error, "Fatal: Could not get config, please check your setup: %s", ex.getMessage().c_str()); + std::_Exit(EXIT_FAILURE); } handler.stop(); }; diff --git a/logforwarder/src/apps/vespa-otelcol-start/wrapper.cpp b/logforwarder/src/apps/vespa-otelcol-start/wrapper.cpp index 7228a3ea921..34b636546ce 100644 --- a/logforwarder/src/apps/vespa-otelcol-start/wrapper.cpp +++ b/logforwarder/src/apps/vespa-otelcol-start/wrapper.cpp @@ -69,7 +69,7 @@ void Wrapper::check() { void Wrapper::gotConfig(const OpenTelemetryConfig& config) { _childHandler.stopChild(); - std::string progPath = vespa::Defaults::underVespaHome("sbin/otelcol-contrib"); + std::string progPath = "/opt/vespa-deps/bin/vespa-otelcol"; std::string cfPath = cfFilePath(); writeConfig(config.yaml, cfPath); _childHandler.startChild(progPath, cfPath); diff --git a/lowercasing_test/CMakeLists.txt b/lowercasing_test/CMakeLists.txt index 2517c0811cd..664dd6e3605 100644 --- a/lowercasing_test/CMakeLists.txt +++ b/lowercasing_test/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_define_module( DEPENDS vespalog vespalib - searchlib + vespa_searchlib APPS src/binref diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 60cf163a8b4..6a7c5a5713b 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -14,7 +14,6 @@ com.google.j2objc:j2objc-annotations:${j2objc-annotations.vespa.version} commons-codec:commons-codec:${commons-codec.vespa.version} commons-io:commons-io:${commons-io.vespa.version} jakarta.inject:jakarta.inject-api:${jakarta.inject.vespa.version} -javax.annotation:javax.annotation-api:${javax.annotation.vespa.version} javax.inject:javax.inject:${javax.inject.vespa.version} junit:junit:${junit4.vespa.version} net.bytebuddy:byte-buddy-agent:${byte-buddy.vespa.version} @@ -32,8 +31,8 @@ org.apache.maven.resolver:maven-resolver-impl:${maven-resolver.vespa.version} org.apache.maven.resolver:maven-resolver-named-locks:${maven-resolver.vespa.version} org.apache.maven.resolver:maven-resolver-spi:${maven-resolver.vespa.version} org.apache.maven.resolver:maven-resolver-util:${maven-resolver.vespa.version} -org.apache.maven.shared:maven-dependency-tree:3.2.1 -org.apache.maven.shared:maven-shared-utils:3.3.4 +org.apache.maven.shared:maven-dependency-tree:${maven-dependency-tree.vespa.version} +org.apache.maven.shared:maven-shared-utils:${maven-shared-utils.vespa.version} org.apache.maven:maven-api-meta:${maven-xml-impl.vespa.version} org.apache.maven:maven-api-xml:${maven-xml-impl.vespa.version} org.apache.maven:maven-archiver:${maven-archiver.vespa.version} @@ -51,7 +50,7 @@ org.apache.maven:maven-xml-impl:${maven-xml-impl.vespa.version} org.apiguardian:apiguardian-api:${apiguardian.vespa.version} org.codehaus.plexus:plexus-archiver:${plexus-archiver.vespa.version} org.codehaus.plexus:plexus-cipher:2.0 -org.codehaus.plexus:plexus-classworlds:2.7.0 +org.codehaus.plexus:plexus-classworlds:${plexus-classworlds.vespa.version} org.codehaus.plexus:plexus-component-annotations:2.1.0 org.codehaus.plexus:plexus-interpolation:${plexus-interpolation.vespa.version} org.codehaus.plexus:plexus-io:${plexus-io.vespa.version} @@ -59,8 +58,6 @@ org.codehaus.plexus:plexus-sec-dispatcher:2.0 org.codehaus.plexus:plexus-utils:${plexus-utils.vespa.version} org.codehaus.plexus:plexus-xml:${plexus-xml.vespa.version} org.codehaus.woodstox:stax2-api:${stax2-api.vespa.version} -org.eclipse.aether:aether-api:1.0.0.v20140518 -org.eclipse.aether:aether-util:1.0.0.v20140518 org.eclipse.sisu:org.eclipse.sisu.inject:${eclipse-sisu.vespa.version} org.eclipse.sisu:org.eclipse.sisu.plexus:${eclipse-sisu.vespa.version} org.hamcrest:hamcrest-core:${hamcrest.vespa.version} diff --git a/messagebus/CMakeLists.txt b/messagebus/CMakeLists.txt index 253e16f0ec0..714f1a5f0e6 100644 --- a/messagebus/CMakeLists.txt +++ b/messagebus/CMakeLists.txt @@ -2,10 +2,10 @@ vespa_define_module( DEPENDS vespalog - config_cloudconfig + vespa_config vespalib - fnet - slobrok + vespa_fnet + vespa_slobrok LIBS src/vespa/messagebus diff --git a/messagebus/src/apps/printversion/CMakeLists.txt b/messagebus/src/apps/printversion/CMakeLists.txt index 7ac02531dd5..c9d504182d5 100644 --- a/messagebus/src/apps/printversion/CMakeLists.txt +++ b/messagebus/src/apps/printversion/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_executable(messagebus_printversion_app SOURCES printversion.cpp DEPENDS - messagebus + vespa_messagebus ) diff --git a/messagebus/src/tests/advancedrouting/CMakeLists.txt b/messagebus/src/tests/advancedrouting/CMakeLists.txt index ca2d3e4bba0..38edf80eee9 100644 --- a/messagebus/src/tests/advancedrouting/CMakeLists.txt +++ b/messagebus/src/tests/advancedrouting/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_advancedrouting_test_app TEST SOURCES advancedrouting.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_advancedrouting_test_app COMMAND messagebus_advancedrouting_test_app) diff --git a/messagebus/src/tests/auto-reply/CMakeLists.txt b/messagebus/src/tests/auto-reply/CMakeLists.txt index 92bbd098972..b715379bf5e 100644 --- a/messagebus/src/tests/auto-reply/CMakeLists.txt +++ b/messagebus/src/tests/auto-reply/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_auto-reply_test_app TEST SOURCES auto-reply.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_auto-reply_test_app COMMAND messagebus_auto-reply_test_app) diff --git a/messagebus/src/tests/auto-reply/auto-reply.cpp b/messagebus/src/tests/auto-reply/auto-reply.cpp index e704fcdb179..ea0ea373465 100644 --- a/messagebus/src/tests/auto-reply/auto-reply.cpp +++ b/messagebus/src/tests/auto-reply/auto-reply.cpp @@ -4,16 +4,11 @@ #include <vespa/messagebus/routablequeue.h> #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simplereply.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("auto-reply_test"); +TEST("auto-reply_test") { RoutableQueue q; { Message::UP msg(new SimpleMessage("test")); @@ -33,5 +28,6 @@ Test::Main() reply->pushHandler(q); } EXPECT_TRUE(q.size() == 2); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/blob/CMakeLists.txt b/messagebus/src/tests/blob/CMakeLists.txt index e6187d46435..f7b7fef16dc 100644 --- a/messagebus/src/tests/blob/CMakeLists.txt +++ b/messagebus/src/tests/blob/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_blob_test_app TEST SOURCES blob.cpp DEPENDS - messagebus - messagebus_messagebus-test + vespa_messagebus + vespa_messagebus-test ) vespa_add_test(NAME messagebus_blob_test_app COMMAND messagebus_blob_test_app) diff --git a/messagebus/src/tests/blob/blob.cpp b/messagebus/src/tests/blob/blob.cpp index e41199d28e8..2e0d1e3fa0a 100644 --- a/messagebus/src/tests/blob/blob.cpp +++ b/messagebus/src/tests/blob/blob.cpp @@ -1,13 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/blob.h> #include <vespa/messagebus/blobref.h> using mbus::Blob; using mbus::BlobRef; -TEST_SETUP(Test); - Blob makeBlob(const char *txt) { Blob b(strlen(txt) + 1); strcpy(b.data(), txt); @@ -26,10 +24,7 @@ BlobRef returnBlobRef(BlobRef br) { return br; } -int -Test::Main() -{ - TEST_INIT("blob_test"); +TEST("blob_test") { // create a blob Blob b = makeBlob("test"); @@ -71,6 +66,6 @@ Test::Main() EXPECT_TRUE(b2.data() == 0); EXPECT_TRUE(b.size() == strlen("test") + 1); EXPECT_TRUE(strcmp("test", b.data()) == 0); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/bucketsequence/CMakeLists.txt b/messagebus/src/tests/bucketsequence/CMakeLists.txt index cc138fb94cd..9e8e96b4bec 100644 --- a/messagebus/src/tests/bucketsequence/CMakeLists.txt +++ b/messagebus/src/tests/bucketsequence/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_bucketsequence_test_app TEST SOURCES bucketsequence.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_bucketsequence_test_app COMMAND messagebus_bucketsequence_test_app) diff --git a/messagebus/src/tests/bucketsequence/bucketsequence.cpp b/messagebus/src/tests/bucketsequence/bucketsequence.cpp index 7a58fe3d861..832f22ece54 100644 --- a/messagebus/src/tests/bucketsequence/bucketsequence.cpp +++ b/messagebus/src/tests/bucketsequence/bucketsequence.cpp @@ -7,7 +7,7 @@ #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/choke/CMakeLists.txt b/messagebus/src/tests/choke/CMakeLists.txt index c87c51f8c82..610528aef38 100644 --- a/messagebus/src/tests/choke/CMakeLists.txt +++ b/messagebus/src/tests/choke/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_choke_test_app TEST SOURCES choke.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_choke_test_app NO_VALGRIND COMMAND messagebus_choke_test_app) diff --git a/messagebus/src/tests/configagent/CMakeLists.txt b/messagebus/src/tests/configagent/CMakeLists.txt index 9b07222fc74..0ed02314b2e 100644 --- a/messagebus/src/tests/configagent/CMakeLists.txt +++ b/messagebus/src/tests/configagent/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_configagent_test_app TEST SOURCES configagent.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_configagent_test_app COMMAND messagebus_configagent_test_app) diff --git a/messagebus/src/tests/context/CMakeLists.txt b/messagebus/src/tests/context/CMakeLists.txt index ee0687f7bd7..0ed059c225c 100644 --- a/messagebus/src/tests/context/CMakeLists.txt +++ b/messagebus/src/tests/context/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_context_test_app TEST SOURCES context.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_context_test_app COMMAND messagebus_context_test_app) diff --git a/messagebus/src/tests/context/context.cpp b/messagebus/src/tests/context/context.cpp index 6adc7a56075..645f326cad0 100644 --- a/messagebus/src/tests/context/context.cpp +++ b/messagebus/src/tests/context/context.cpp @@ -9,7 +9,7 @@ #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <thread> using namespace mbus; @@ -36,12 +36,7 @@ RoutingSpec getRouting() { .addRoute(RouteSpec("test").addHop("test"))); } -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("context_test"); +TEST("context_test") { Slobrok slobrok; TestServer src(Identity(""), getRouting(), slobrok); @@ -92,5 +87,6 @@ Test::Main() ASSERT_TRUE(reply); EXPECT_EQUAL(reply->getContext().value.UINT64, 30u); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/emptyreply/CMakeLists.txt b/messagebus/src/tests/emptyreply/CMakeLists.txt index 5aebfd08aeb..b5d48280495 100644 --- a/messagebus/src/tests/emptyreply/CMakeLists.txt +++ b/messagebus/src/tests/emptyreply/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_emptyreply_test_app TEST SOURCES emptyreply.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_emptyreply_test_app COMMAND messagebus_emptyreply_test_app) diff --git a/messagebus/src/tests/emptyreply/emptyreply.cpp b/messagebus/src/tests/emptyreply/emptyreply.cpp index f8eb341c9d4..d373b27eb0c 100644 --- a/messagebus/src/tests/emptyreply/emptyreply.cpp +++ b/messagebus/src/tests/emptyreply/emptyreply.cpp @@ -1,18 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/emptyreply.h> using namespace mbus; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("emptyreply_test"); +TEST("emptyreply_test") { Reply::UP empty(new EmptyReply()); EXPECT_TRUE(empty->isReply()); EXPECT_TRUE(empty->getProtocol() == ""); EXPECT_TRUE(empty->getType() == 0); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/error/CMakeLists.txt b/messagebus/src/tests/error/CMakeLists.txt index 15b3f709710..453d73aba43 100644 --- a/messagebus/src/tests/error/CMakeLists.txt +++ b/messagebus/src/tests/error/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_error_test_app TEST SOURCES error.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_error_test_app COMMAND messagebus_error_test_app) diff --git a/messagebus/src/tests/error/error.cpp b/messagebus/src/tests/error/error.cpp index bfc4d4fc988..e7be09c3fcb 100644 --- a/messagebus/src/tests/error/error.cpp +++ b/messagebus/src/tests/error/error.cpp @@ -11,7 +11,7 @@ #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/identity/CMakeLists.txt b/messagebus/src/tests/identity/CMakeLists.txt index 39ce7f23d06..2771139df9a 100644 --- a/messagebus/src/tests/identity/CMakeLists.txt +++ b/messagebus/src/tests/identity/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_identity_test_app TEST SOURCES identity.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_identity_test_app COMMAND messagebus_identity_test_app) diff --git a/messagebus/src/tests/identity/identity.cpp b/messagebus/src/tests/identity/identity.cpp index eb5af399a9e..3be8ff8498c 100644 --- a/messagebus/src/tests/identity/identity.cpp +++ b/messagebus/src/tests/identity/identity.cpp @@ -1,15 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/network/identity.h> using namespace mbus; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("identity_test"); +TEST("identity_test") { Identity ident("foo/bar/baz"); EXPECT_TRUE(ident.getServicePrefix() == "foo/bar/baz"); { @@ -36,5 +31,6 @@ Test::Main() ASSERT_TRUE(tmp.size() == 1); EXPECT_TRUE(tmp[0] == ""); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/messagebus/CMakeLists.txt b/messagebus/src/tests/messagebus/CMakeLists.txt index f05141ca28f..c5240859d14 100644 --- a/messagebus/src/tests/messagebus/CMakeLists.txt +++ b/messagebus/src/tests/messagebus/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_messagebus_test_app TEST SOURCES messagebus.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_messagebus_test_app COMMAND messagebus_messagebus_test_app) diff --git a/messagebus/src/tests/messageordering/CMakeLists.txt b/messagebus/src/tests/messageordering/CMakeLists.txt index 7a429216d11..221dbcc42a0 100644 --- a/messagebus/src/tests/messageordering/CMakeLists.txt +++ b/messagebus/src/tests/messageordering/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_messageordering_test_app TEST SOURCES messageordering.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test( NAME messagebus_messageordering_test_app diff --git a/messagebus/src/tests/messageordering/messageordering.cpp b/messagebus/src/tests/messageordering/messageordering.cpp index 8b3754a8016..1083e5902f6 100644 --- a/messagebus/src/tests/messageordering/messageordering.cpp +++ b/messagebus/src/tests/messageordering/messageordering.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> @@ -15,8 +15,6 @@ LOG_SETUP("messageordering_test"); using namespace mbus; using namespace std::chrono_literals; -TEST_SETUP(Test); - RoutingSpec getRouting() { @@ -138,10 +136,7 @@ VerifyReplyReceptor::waitUntilDone(int waitForCount) const } } -int -Test::Main() -{ - TEST_INIT("messageordering_test"); +TEST("messageordering_test") { Slobrok slobrok; TestServer srcNet(Identity("test/src"), getRouting(), slobrok); @@ -178,6 +173,6 @@ Test::Main() src.waitUntilDone(messageCount); ASSERT_EQUAL(std::string(), src.getFailure()); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/messenger/CMakeLists.txt b/messagebus/src/tests/messenger/CMakeLists.txt index 633ee176062..3cf98f98c71 100644 --- a/messagebus/src/tests/messenger/CMakeLists.txt +++ b/messagebus/src/tests/messenger/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_messenger_test_app TEST SOURCES messenger.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_messenger_test_app COMMAND messagebus_messenger_test_app) diff --git a/messagebus/src/tests/messenger/messenger.cpp b/messagebus/src/tests/messenger/messenger.cpp index 92fd189aa51..ed080833882 100644 --- a/messagebus/src/tests/messenger/messenger.cpp +++ b/messagebus/src/tests/messenger/messenger.cpp @@ -2,7 +2,7 @@ #include <vespa/messagebus/messenger.h> #include <vespa/vespalib/util/barrier.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/protocolrepository/CMakeLists.txt b/messagebus/src/tests/protocolrepository/CMakeLists.txt index ae25e32a4cf..4609855c509 100644 --- a/messagebus/src/tests/protocolrepository/CMakeLists.txt +++ b/messagebus/src/tests/protocolrepository/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_protocolrepository_test_app TEST SOURCES protocolrepository.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_protocolrepository_test_app NO_VALGRIND COMMAND messagebus_protocolrepository_test_app) diff --git a/messagebus/src/tests/protocolrepository/protocolrepository.cpp b/messagebus/src/tests/protocolrepository/protocolrepository.cpp index e12188f2551..2a0493aec14 100644 --- a/messagebus/src/tests/protocolrepository/protocolrepository.cpp +++ b/messagebus/src/tests/protocolrepository/protocolrepository.cpp @@ -1,12 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/messagebus/protocolrepository.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; -TEST_SETUP(Test); - class TestProtocol : public IProtocol { private: const string _name; @@ -32,10 +30,7 @@ public: } }; -int -Test::Main() -{ - TEST_INIT("protocolrepository_test"); +TEST("protocolrepository_test") { ProtocolRepository repo; IProtocol::SP prev; @@ -49,6 +44,6 @@ Test::Main() policy = repo.getRoutingPolicy("foo", "bar", "baz"); ASSERT_FALSE(policy); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/queue/CMakeLists.txt b/messagebus/src/tests/queue/CMakeLists.txt index 097e5cc312a..d2a35985b9e 100644 --- a/messagebus/src/tests/queue/CMakeLists.txt +++ b/messagebus/src/tests/queue/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_queue_test_app TEST SOURCES queue.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_queue_test_app COMMAND messagebus_queue_test_app) diff --git a/messagebus/src/tests/queue/queue.cpp b/messagebus/src/tests/queue/queue.cpp index 19d485a5e5b..6f5a026b4a5 100644 --- a/messagebus/src/tests/queue/queue.cpp +++ b/messagebus/src/tests/queue/queue.cpp @@ -1,16 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/queue.h> using namespace mbus; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("queue_test"); +TEST("queue_test") { Queue<int> q; EXPECT_TRUE(q.size() == 0); q.push(1); @@ -84,5 +79,6 @@ Test::Main() EXPECT_TRUE(q.front() == 3); q.pop(); EXPECT_TRUE(q.size() == 0); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/replygate/CMakeLists.txt b/messagebus/src/tests/replygate/CMakeLists.txt index 5d503c0ca18..c548b56971b 100644 --- a/messagebus/src/tests/replygate/CMakeLists.txt +++ b/messagebus/src/tests/replygate/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_replygate_test_app TEST SOURCES replygate.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_replygate_test_app COMMAND messagebus_replygate_test_app) diff --git a/messagebus/src/tests/replygate/replygate.cpp b/messagebus/src/tests/replygate/replygate.cpp index 104c2f82518..e1b1a5ed1dd 100644 --- a/messagebus/src/tests/replygate/replygate.cpp +++ b/messagebus/src/tests/replygate/replygate.cpp @@ -5,7 +5,7 @@ #include <vespa/messagebus/replygate.h> #include <vespa/messagebus/routablequeue.h> #include <vespa/messagebus/testlib/simplemessage.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/resender/CMakeLists.txt b/messagebus/src/tests/resender/CMakeLists.txt index c364806b2dd..93150e01311 100644 --- a/messagebus/src/tests/resender/CMakeLists.txt +++ b/messagebus/src/tests/resender/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_resender_test_app TEST SOURCES resender.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_resender_test_app COMMAND messagebus_resender_test_app) diff --git a/messagebus/src/tests/result/CMakeLists.txt b/messagebus/src/tests/result/CMakeLists.txt index d4360f3826b..7d25570f33a 100644 --- a/messagebus/src/tests/result/CMakeLists.txt +++ b/messagebus/src/tests/result/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_result_test_app TEST SOURCES result.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_result_test_app COMMAND messagebus_result_test_app) diff --git a/messagebus/src/tests/retrypolicy/CMakeLists.txt b/messagebus/src/tests/retrypolicy/CMakeLists.txt index 2ef419109e0..a3113a5ade1 100644 --- a/messagebus/src/tests/retrypolicy/CMakeLists.txt +++ b/messagebus/src/tests/retrypolicy/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_retrypolicy_test_app TEST SOURCES retrypolicy.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_retrypolicy_test_app COMMAND messagebus_retrypolicy_test_app) diff --git a/messagebus/src/tests/retrypolicy/retrypolicy.cpp b/messagebus/src/tests/retrypolicy/retrypolicy.cpp index 12a49989d6d..635a81ef75e 100644 --- a/messagebus/src/tests/retrypolicy/retrypolicy.cpp +++ b/messagebus/src/tests/retrypolicy/retrypolicy.cpp @@ -2,16 +2,11 @@ #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/routing/retrytransienterrorspolicy.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("retrypolicy_test"); +TEST("retrypolicy_test") { constexpr double DELAY(0.001); RetryTransientErrorsPolicy policy; policy.setBaseDelay(DELAY); @@ -34,6 +29,6 @@ Test::Main() EXPECT_TRUE(!policy.canRetry(j)); } } - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/routable/CMakeLists.txt b/messagebus/src/tests/routable/CMakeLists.txt index 4641462c4f5..ad19bb22d2d 100644 --- a/messagebus/src/tests/routable/CMakeLists.txt +++ b/messagebus/src/tests/routable/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_routable_test_app TEST SOURCES routable.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_routable_test_app COMMAND messagebus_routable_test_app) diff --git a/messagebus/src/tests/routable/routable.cpp b/messagebus/src/tests/routable/routable.cpp index f45b212ffee..13f65a52b61 100644 --- a/messagebus/src/tests/routable/routable.cpp +++ b/messagebus/src/tests/routable/routable.cpp @@ -8,17 +8,12 @@ #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/error.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; using namespace std::chrono_literals; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("routable_test"); +TEST("routable_test") { { // Test message swap state. @@ -89,6 +84,6 @@ Test::Main() Reply::UP ap = handler.getReplyNow(); ASSERT_FALSE(ap); } - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/routablequeue/CMakeLists.txt b/messagebus/src/tests/routablequeue/CMakeLists.txt index 70f8ca6a993..7464b0490ad 100644 --- a/messagebus/src/tests/routablequeue/CMakeLists.txt +++ b/messagebus/src/tests/routablequeue/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_routablequeue_test_app TEST SOURCES routablequeue.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_routablequeue_test_app COMMAND messagebus_routablequeue_test_app) diff --git a/messagebus/src/tests/routablequeue/routablequeue.cpp b/messagebus/src/tests/routablequeue/routablequeue.cpp index 01932e4e488..01a95155483 100644 --- a/messagebus/src/tests/routablequeue/routablequeue.cpp +++ b/messagebus/src/tests/routablequeue/routablequeue.cpp @@ -3,7 +3,7 @@ #include <vespa/messagebus/routablequeue.h> #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simplereply.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; @@ -31,12 +31,7 @@ public: }; uint32_t TestReply::_cnt = 0; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("routablequeue_test"); +TEST("routablequeue_test") { { RoutableQueue rq; EXPECT_TRUE(rq.size() == 0); @@ -103,5 +98,6 @@ Test::Main() } EXPECT_TRUE(TestMessage::getCnt() == 0); EXPECT_TRUE(TestReply::getCnt() == 0); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/routeparser/CMakeLists.txt b/messagebus/src/tests/routeparser/CMakeLists.txt index 6c715d49f73..065e7674d1e 100644 --- a/messagebus/src/tests/routeparser/CMakeLists.txt +++ b/messagebus/src/tests/routeparser/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_routeparser_test_app TEST SOURCES routeparser.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_routeparser_test_app COMMAND messagebus_routeparser_test_app) diff --git a/messagebus/src/tests/routing/CMakeLists.txt b/messagebus/src/tests/routing/CMakeLists.txt index 6b013f7b9cc..db0c586bb09 100644 --- a/messagebus/src/tests/routing/CMakeLists.txt +++ b/messagebus/src/tests/routing/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_routing_test_app TEST SOURCES routing.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_routing_test_app COMMAND messagebus_routing_test_app) diff --git a/messagebus/src/tests/routingcontext/CMakeLists.txt b/messagebus/src/tests/routingcontext/CMakeLists.txt index 932d0020a3e..8e16c7bf489 100644 --- a/messagebus/src/tests/routingcontext/CMakeLists.txt +++ b/messagebus/src/tests/routingcontext/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_routingcontext_test_app TEST SOURCES routingcontext.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_routingcontext_test_app COMMAND messagebus_routingcontext_test_app) diff --git a/messagebus/src/tests/routingspec/CMakeLists.txt b/messagebus/src/tests/routingspec/CMakeLists.txt index c7115c16fd9..34fa9007fff 100644 --- a/messagebus/src/tests/routingspec/CMakeLists.txt +++ b/messagebus/src/tests/routingspec/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_routingspec_test_app TEST SOURCES routingspec.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_routingspec_test_app COMMAND messagebus_routingspec_test_app) diff --git a/messagebus/src/tests/rpcserviceaddress/CMakeLists.txt b/messagebus/src/tests/rpcserviceaddress/CMakeLists.txt index 9d9d0ded659..a667cd4c3c7 100644 --- a/messagebus/src/tests/rpcserviceaddress/CMakeLists.txt +++ b/messagebus/src/tests/rpcserviceaddress/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_rpcserviceaddress_test_app TEST SOURCES rpcserviceaddress.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_rpcserviceaddress_test_app COMMAND messagebus_rpcserviceaddress_test_app) diff --git a/messagebus/src/tests/rpcserviceaddress/rpcserviceaddress.cpp b/messagebus/src/tests/rpcserviceaddress/rpcserviceaddress.cpp index 8383505538b..82023b94564 100644 --- a/messagebus/src/tests/rpcserviceaddress/rpcserviceaddress.cpp +++ b/messagebus/src/tests/rpcserviceaddress/rpcserviceaddress.cpp @@ -1,16 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/messagebus/network/rpcserviceaddress.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("rpcserviceaddress_test"); +TEST("rpcserviceaddress_test") { { EXPECT_TRUE(RPCServiceAddress("", "bar").isMalformed()); EXPECT_TRUE(RPCServiceAddress("foo", "bar").isMalformed()); @@ -38,5 +33,6 @@ Test::Main() EXPECT_TRUE(addr.getConnectionSpec() == "tcp/foo.com:42"); EXPECT_TRUE(addr.getSessionName() == ""); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/sendadapter/CMakeLists.txt b/messagebus/src/tests/sendadapter/CMakeLists.txt index 9b73f282157..d0d4851d01f 100644 --- a/messagebus/src/tests/sendadapter/CMakeLists.txt +++ b/messagebus/src/tests/sendadapter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_sendadapter_test_app TEST SOURCES sendadapter.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_sendadapter_test_app COMMAND messagebus_sendadapter_test_app) diff --git a/messagebus/src/tests/sendadapter/sendadapter.cpp b/messagebus/src/tests/sendadapter/sendadapter.cpp index 48618ab3061..633cc6de314 100644 --- a/messagebus/src/tests/sendadapter/sendadapter.cpp +++ b/messagebus/src/tests/sendadapter/sendadapter.cpp @@ -6,7 +6,7 @@ #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/messagebus/network/rpcsendv2.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("sendadapter_test"); diff --git a/messagebus/src/tests/sequencer/CMakeLists.txt b/messagebus/src/tests/sequencer/CMakeLists.txt index 0c00209f3c7..6f5acb49d78 100644 --- a/messagebus/src/tests/sequencer/CMakeLists.txt +++ b/messagebus/src/tests/sequencer/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_sequencer_test_app TEST SOURCES sequencer.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_sequencer_test_app COMMAND messagebus_sequencer_test_app) diff --git a/messagebus/src/tests/serviceaddress/CMakeLists.txt b/messagebus/src/tests/serviceaddress/CMakeLists.txt index 9c7315dfdbd..df5c7b6ec2f 100644 --- a/messagebus/src/tests/serviceaddress/CMakeLists.txt +++ b/messagebus/src/tests/serviceaddress/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_serviceaddress_test_app TEST SOURCES serviceaddress.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_serviceaddress_test_app COMMAND messagebus_serviceaddress_test_app) diff --git a/messagebus/src/tests/serviceaddress/serviceaddress.cpp b/messagebus/src/tests/serviceaddress/serviceaddress.cpp index 376ee5fa38a..e2440cd917f 100644 --- a/messagebus/src/tests/serviceaddress/serviceaddress.cpp +++ b/messagebus/src/tests/serviceaddress/serviceaddress.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/messagebus/network/rpcservice.h> diff --git a/messagebus/src/tests/servicepool/CMakeLists.txt b/messagebus/src/tests/servicepool/CMakeLists.txt index 75886af2f69..addd8daaa81 100644 --- a/messagebus/src/tests/servicepool/CMakeLists.txt +++ b/messagebus/src/tests/servicepool/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_servicepool_test_app TEST SOURCES servicepool.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_servicepool_test_app COMMAND messagebus_servicepool_test_app) diff --git a/messagebus/src/tests/servicepool/servicepool.cpp b/messagebus/src/tests/servicepool/servicepool.cpp index 4a21f40e202..7439127e883 100644 --- a/messagebus/src/tests/servicepool/servicepool.cpp +++ b/messagebus/src/tests/servicepool/servicepool.cpp @@ -5,7 +5,7 @@ #include <vespa/messagebus/network/rpcservicepool.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; diff --git a/messagebus/src/tests/shutdown/CMakeLists.txt b/messagebus/src/tests/shutdown/CMakeLists.txt index 01d6df111fb..8cc0b9210fe 100644 --- a/messagebus/src/tests/shutdown/CMakeLists.txt +++ b/messagebus/src/tests/shutdown/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_shutdown_test_app TEST SOURCES shutdown.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_shutdown_test_app COMMAND messagebus_shutdown_test_app) diff --git a/messagebus/src/tests/shutdown/shutdown.cpp b/messagebus/src/tests/shutdown/shutdown.cpp index 8dc4aa555b4..424bd86e203 100644 --- a/messagebus/src/tests/shutdown/shutdown.cpp +++ b/messagebus/src/tests/shutdown/shutdown.cpp @@ -7,7 +7,7 @@ #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> using namespace mbus; diff --git a/messagebus/src/tests/simple-roundtrip/CMakeLists.txt b/messagebus/src/tests/simple-roundtrip/CMakeLists.txt index c0fc2b6a292..cef8a477e97 100644 --- a/messagebus/src/tests/simple-roundtrip/CMakeLists.txt +++ b/messagebus/src/tests/simple-roundtrip/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_simple-roundtrip_test_app TEST SOURCES simple-roundtrip.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_simple-roundtrip_test_app COMMAND messagebus_simple-roundtrip_test_app) diff --git a/messagebus/src/tests/simple-roundtrip/simple-roundtrip.cpp b/messagebus/src/tests/simple-roundtrip/simple-roundtrip.cpp index 37e24c5a329..acc6bc389ce 100644 --- a/messagebus/src/tests/simple-roundtrip/simple-roundtrip.cpp +++ b/messagebus/src/tests/simple-roundtrip/simple-roundtrip.cpp @@ -7,12 +7,10 @@ #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/messagebus.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace mbus; -TEST_SETUP(Test); - RoutingSpec getRouting() { return RoutingSpec() .addTable(RoutingTableSpec("Simple") @@ -21,10 +19,7 @@ RoutingSpec getRouting() { .addRoute(RouteSpec("test").addHop("pxy").addHop("dst"))); } -int -Test::Main() -{ - TEST_INIT("simple-roundtrip_test"); +TEST("simple-roundtrip_test") { Slobrok slobrok; TestServer srcNet(Identity("test/src"), getRouting(), slobrok); @@ -87,5 +82,6 @@ Test::Main() EXPECT_TRUE(reply->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(reply->getType() == SimpleProtocol::REPLY); EXPECT_TRUE(dynamic_cast<SimpleReply&>(*reply).getValue() == "test reply pxy"); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/simpleprotocol/CMakeLists.txt b/messagebus/src/tests/simpleprotocol/CMakeLists.txt index eaa7675d4a6..cd8b4d84932 100644 --- a/messagebus/src/tests/simpleprotocol/CMakeLists.txt +++ b/messagebus/src/tests/simpleprotocol/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_simpleprotocol_test_app TEST SOURCES simpleprotocol.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_simpleprotocol_test_app COMMAND messagebus_simpleprotocol_test_app) diff --git a/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp b/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp index 2dd3e28c35d..00d0c8ff961 100644 --- a/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp +++ b/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp @@ -7,17 +7,12 @@ #include <vespa/messagebus/testlib/testserver.h> #include <vespa/messagebus/ireplyhandler.h> #include <vespa/messagebus/routing/routingcontext.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/component/vtag.h> using namespace mbus; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("simpleprotocol_test"); +TEST("simpleprotocol_test") { vespalib::Version version = vespalib::Vtag::currentVersion; SimpleProtocol protocol; @@ -69,5 +64,6 @@ Test::Main() EXPECT_TRUE(tmp->getType() == SimpleProtocol::REPLY); EXPECT_TRUE(static_cast<SimpleReply&>(*tmp).getValue() == "reply"); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/slobrok/CMakeLists.txt b/messagebus/src/tests/slobrok/CMakeLists.txt index 52cfd22b986..a1dce99f3cc 100644 --- a/messagebus/src/tests/slobrok/CMakeLists.txt +++ b/messagebus/src/tests/slobrok/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_slobrok_test_app TEST SOURCES slobrok.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_slobrok_test_app COMMAND messagebus_slobrok_test_app) diff --git a/messagebus/src/tests/slobrok/slobrok.cpp b/messagebus/src/tests/slobrok/slobrok.cpp index a36039f458d..de9d966d3f1 100644 --- a/messagebus/src/tests/slobrok/slobrok.cpp +++ b/messagebus/src/tests/slobrok/slobrok.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/messagebus/network/rpcnetwork.h> @@ -56,12 +56,7 @@ compare(const IMirrorAPI &api, const string &pattern, SpecList expect) return false; } -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("slobrok_test"); +TEST("slobrok_test") { Slobrok slobrok; RPCNetwork net1(RPCNetworkParams(slobrok.config()) .setIdentity(Identity("net/a"))); @@ -128,5 +123,6 @@ Test::Main() net3.shutdown(); net2.shutdown(); net1.shutdown(); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/sourcesession/CMakeLists.txt b/messagebus/src/tests/sourcesession/CMakeLists.txt index 6f8b14667ee..8dd26a4ce16 100644 --- a/messagebus/src/tests/sourcesession/CMakeLists.txt +++ b/messagebus/src/tests/sourcesession/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_sourcesession_test_app TEST SOURCES sourcesession.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_sourcesession_test_app COMMAND messagebus_sourcesession_test_app) diff --git a/messagebus/src/tests/targetpool/CMakeLists.txt b/messagebus/src/tests/targetpool/CMakeLists.txt index 72cb63edb99..6990c10c806 100644 --- a/messagebus/src/tests/targetpool/CMakeLists.txt +++ b/messagebus/src/tests/targetpool/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_targetpool_test_app TEST SOURCES targetpool.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_targetpool_test_app COMMAND messagebus_targetpool_test_app) diff --git a/messagebus/src/tests/targetpool/targetpool.cpp b/messagebus/src/tests/targetpool/targetpool.cpp index dd86b279838..4265f9a946d 100644 --- a/messagebus/src/tests/targetpool/targetpool.cpp +++ b/messagebus/src/tests/targetpool/targetpool.cpp @@ -2,7 +2,7 @@ #include <vespa/messagebus/network/rpctargetpool.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("targetpool_test"); diff --git a/messagebus/src/tests/throttling/CMakeLists.txt b/messagebus/src/tests/throttling/CMakeLists.txt index 3ad0662d13b..56cb058f72a 100644 --- a/messagebus/src/tests/throttling/CMakeLists.txt +++ b/messagebus/src/tests/throttling/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_throttling_test_app TEST SOURCES throttling.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_throttling_test_app COMMAND messagebus_throttling_test_app) diff --git a/messagebus/src/tests/timeout/CMakeLists.txt b/messagebus/src/tests/timeout/CMakeLists.txt index 4aa3458cb14..b6620dfca4f 100644 --- a/messagebus/src/tests/timeout/CMakeLists.txt +++ b/messagebus/src/tests/timeout/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(messagebus_timeout_test_app TEST SOURCES timeout.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus GTest::gtest ) vespa_add_test(NAME messagebus_timeout_test_app COMMAND messagebus_timeout_test_app) diff --git a/messagebus/src/tests/trace-roundtrip/CMakeLists.txt b/messagebus/src/tests/trace-roundtrip/CMakeLists.txt index 034a6d7dbd2..30bda0803e8 100644 --- a/messagebus/src/tests/trace-roundtrip/CMakeLists.txt +++ b/messagebus/src/tests/trace-roundtrip/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_trace-roundtrip_test_app TEST SOURCES trace-roundtrip.cpp DEPENDS - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) vespa_add_test(NAME messagebus_trace-roundtrip_test_app COMMAND messagebus_trace-roundtrip_test_app) diff --git a/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp b/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp index bb86a94a2b6..5a8f57d35b7 100644 --- a/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp +++ b/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/sourcesession.h> @@ -72,8 +72,6 @@ Server::handleMessage(Message::UP msg) { //----------------------------------------------------------------------------- -TEST_SETUP(Test); - RoutingSpec getRouting() { return RoutingSpec() .addTable(RoutingTableSpec("Simple") @@ -82,10 +80,7 @@ RoutingSpec getRouting() { .addRoute(RouteSpec("test").addHop("pxy").addHop("dst"))); } -int -Test::Main() -{ - TEST_INIT("simple-roundtrip_test"); +TEST("simple-roundtrip_test") { Slobrok slobrok; TestServer srcNet(Identity("test/src"), getRouting(), slobrok); @@ -119,5 +114,6 @@ Test::Main() .addChild("Proxy reply") .addChild("Client reply"); EXPECT_TRUE(reply->getTrace().encode() == t.encode()); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/vespa/messagebus/CMakeLists.txt b/messagebus/src/vespa/messagebus/CMakeLists.txt index d9562ee2b40..52a1678e9fb 100644 --- a/messagebus/src/vespa/messagebus/CMakeLists.txt +++ b/messagebus/src/vespa/messagebus/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(messagebus +vespa_add_library(vespa_messagebus SOURCES blob.cpp blobref.cpp @@ -36,5 +36,5 @@ vespa_add_library(messagebus INSTALL lib64 DEPENDS ) -vespa_generate_config(messagebus ../../main/config/messagebus.def) +vespa_generate_config(vespa_messagebus ../../main/config/messagebus.def) install_config_definition(../../main/config/messagebus.def messagebus.messagebus.def) diff --git a/messagebus/src/vespa/messagebus/testlib/CMakeLists.txt b/messagebus/src/vespa/messagebus/testlib/CMakeLists.txt index f2805c03f73..39aceec4153 100644 --- a/messagebus/src/vespa/messagebus/testlib/CMakeLists.txt +++ b/messagebus/src/vespa/messagebus/testlib/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(messagebus_messagebus-test +vespa_add_library(vespa_messagebus-test SOURCES custompolicy.cpp receptor.cpp @@ -11,6 +11,6 @@ vespa_add_library(messagebus_messagebus-test testserver.cpp INSTALL lib64 DEPENDS - messagebus - slobrok_slobrokserver + vespa_messagebus + vespa_slobrok_slobrokserver ) diff --git a/messagebus_test/CMakeLists.txt b/messagebus_test/CMakeLists.txt index 337619274f8..e65b944748b 100644 --- a/messagebus_test/CMakeLists.txt +++ b/messagebus_test/CMakeLists.txt @@ -1,8 +1,8 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - slobrok_slobrokserver - messagebus + vespa_slobrok_slobrokserver + vespa_messagebus APPS src/binref diff --git a/messagebus_test/src/tests/compile-cpp/CMakeLists.txt b/messagebus_test/src/tests/compile-cpp/CMakeLists.txt index d9f27944396..3420786c0e7 100644 --- a/messagebus_test/src/tests/compile-cpp/CMakeLists.txt +++ b/messagebus_test/src/tests/compile-cpp/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(messagebus_test_compile-cpp_test_app TEST SOURCES compile-cpp.cpp DEPENDS - messagebus + vespa_messagebus ) vespa_add_test(NAME messagebus_test_compile-cpp_test_app NO_VALGRIND COMMAND messagebus_test_compile-cpp_test_app) diff --git a/messagebus_test/src/tests/compile-cpp/compile-cpp.cpp b/messagebus_test/src/tests/compile-cpp/compile-cpp.cpp index c8ab3c60ccb..3e145427408 100644 --- a/messagebus_test/src/tests/compile-cpp/compile-cpp.cpp +++ b/messagebus_test/src/tests/compile-cpp/compile-cpp.cpp @@ -1,15 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("compile-cpp_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/routing/route.h> -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("compile-cpp_test"); +TEST("compile-cpp_test") { mbus::Route r; - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus_test/src/tests/error/CMakeLists.txt b/messagebus_test/src/tests/error/CMakeLists.txt index d67e8fc89e8..1ad1852b00f 100644 --- a/messagebus_test/src/tests/error/CMakeLists.txt +++ b/messagebus_test/src/tests/error/CMakeLists.txt @@ -3,19 +3,19 @@ vespa_add_executable(messagebus_test_error_test_app TEST SOURCES error.cpp DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_add_executable(messagebus_test_cpp-server-error_app SOURCES cpp-server.cpp DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_add_executable(messagebus_test_cpp-client-error_app SOURCES cpp-client.cpp DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_add_test(NAME messagebus_test_error_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/error_test.sh DEPENDS messagebus_test_error_test_app messagebus_test_cpp-server-error_app messagebus_test_cpp-client-error_app) diff --git a/messagebus_test/src/tests/error/error.cpp b/messagebus_test/src/tests/error/error.cpp index 6e1c6b8ee70..d360849e8af 100644 --- a/messagebus_test/src/tests/error/error.cpp +++ b/messagebus_test/src/tests/error/error.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/messagebus_test/src/tests/errorcodes/CMakeLists.txt b/messagebus_test/src/tests/errorcodes/CMakeLists.txt index 1c38ccb723f..178265e1e60 100644 --- a/messagebus_test/src/tests/errorcodes/CMakeLists.txt +++ b/messagebus_test/src/tests/errorcodes/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(messagebus_test_dumpcodes_app TEST SOURCES dumpcodes.cpp DEPENDS - messagebus + vespa_messagebus ) vespa_add_test(NAME messagebus_test_dumpcodes_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/errorcodes_test.sh DEPENDS messagebus_test_dumpcodes_app) diff --git a/messagebus_test/src/tests/speed/CMakeLists.txt b/messagebus_test/src/tests/speed/CMakeLists.txt index 612e07e5ab3..79eb979558a 100644 --- a/messagebus_test/src/tests/speed/CMakeLists.txt +++ b/messagebus_test/src/tests/speed/CMakeLists.txt @@ -3,19 +3,19 @@ vespa_add_executable(messagebus_test_speed_test_app SOURCES speed.cpp DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_add_executable(messagebus_test_cpp-server-speed_app SOURCES cpp-server.cpp DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_add_executable(messagebus_test_cpp-client-speed_app SOURCES cpp-client.cpp DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_add_test(NAME messagebus_test_speed_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/speed_test.sh BENCHMARK DEPENDS messagebus_test_speed_test_app messagebus_test_cpp-server-speed_app messagebus_test_cpp-client-speed_app ) diff --git a/messagebus_test/src/tests/speed/speed.cpp b/messagebus_test/src/tests/speed/speed.cpp index 1b0b9002433..cb88be8ce34 100644 --- a/messagebus_test/src/tests/speed/speed.cpp +++ b/messagebus_test/src/tests/speed/speed.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("speed_test"); -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/vespalib/util/stringfmt.h> @@ -9,12 +8,7 @@ LOG_SETUP("speed_test"); using namespace mbus; using vespalib::make_string; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("speed_test"); +TEST("speed_test") { Slobrok slobrok; const std::string routing_template = TEST_PATH("routing-template.cfg"); @@ -51,5 +45,6 @@ Test::Main() fprintf(stderr, "STOPPING\n"); EXPECT_EQUAL(system((ctl_script + " stop server java").c_str()), 0); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus_test/src/tests/trace/CMakeLists.txt b/messagebus_test/src/tests/trace/CMakeLists.txt index 93660c5f345..66273b3a97c 100644 --- a/messagebus_test/src/tests/trace/CMakeLists.txt +++ b/messagebus_test/src/tests/trace/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(messagebus_test_trace_test_app TEST SOURCES trace.cpp DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_add_executable(messagebus_test_cpp-server-trace_app SOURCES cpp-server.cpp DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_add_test(NAME messagebus_test_trace_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/trace_test.sh DEPENDS messagebus_test_trace_test_app messagebus_test_cpp-server-trace_app) diff --git a/messagebus_test/src/tests/trace/trace.cpp b/messagebus_test/src/tests/trace/trace.cpp index a1b14281f94..c2d5a1cbdb5 100644 --- a/messagebus_test/src/tests/trace/trace.cpp +++ b/messagebus_test/src/tests/trace/trace.cpp @@ -1,6 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/vespalib/util/stringfmt.h> @@ -23,8 +22,6 @@ LOG_SETUP("trace_test"); using namespace mbus; using vespalib::make_string; -TEST_SETUP(Test); - bool waitSlobrok(RPCMessageBus &mbus, const std::string &pattern) { @@ -38,10 +35,7 @@ waitSlobrok(RPCMessageBus &mbus, const std::string &pattern) return false; } -int -Test::Main() -{ - TEST_INIT("trace_test"); +TEST("trace_test") { Slobrok slobrok; const std::string routing_template = TEST_PATH("routing-template.cfg"); const std::string ctl_script = TEST_PATH("ctl.sh"); @@ -115,5 +109,6 @@ Test::Main() EXPECT_TRUE(!reply->hasErrors()); EXPECT_EQUAL(reply->getTrace().encode(), expect.encode()); EXPECT_TRUE(system((ctl_script + " stop all").c_str()) == 0); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsJsonResponse.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsJsonResponse.java new file mode 100644 index 00000000000..b927db790b2 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/MetricsJsonResponse.java @@ -0,0 +1,31 @@ +package ai.vespa.metricsproxy.http; + +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.function.Consumer; + +/** + * @author jonmv + */ +public class MetricsJsonResponse extends HttpResponse { + + private final Consumer<OutputStream> modelWriter; + + public MetricsJsonResponse(int status, Consumer<OutputStream> modelWriter) { + super(status); + this.modelWriter = modelWriter; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + modelWriter.accept(outputStream); + } + + @Override + public long maxPendingBytes() { + return 1 << 20; + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/PrometheusResponse.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/PrometheusResponse.java new file mode 100644 index 00000000000..e0c74671c9c --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/PrometheusResponse.java @@ -0,0 +1,35 @@ +package ai.vespa.metricsproxy.http; + +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusModel; +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; + +/** + * @author jonmv + */ +public class PrometheusResponse extends HttpResponse { + + private final PrometheusModel model; + + public PrometheusResponse(int status, PrometheusModel model) { + super(status); + this.model = model; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + Writer writer = new OutputStreamWriter(outputStream); + model.serialize(writer); + writer.flush(); + } + + @Override + public long maxPendingBytes() { + return 1 << 20; + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java index 58b51020bb9..ace0d0abc65 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java @@ -3,7 +3,8 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.http.TextResponse; +import ai.vespa.metricsproxy.http.MetricsJsonResponse; +import ai.vespa.metricsproxy.http.PrometheusResponse; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; @@ -62,12 +63,12 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { return Optional.empty(); } - private JsonResponse applicationMetricsResponse(String requestedConsumer) { + private HttpResponse applicationMetricsResponse(String requestedConsumer) { try { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); var metricsByNode = metricsRetriever.getMetrics(consumer); - return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize()); + return new MetricsJsonResponse(OK, toGenericApplicationModel(metricsByNode)::serialize); } catch (Exception e) { log.log(Level.WARNING, "Got exception when retrieving metrics:", e); @@ -75,7 +76,7 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { } } - private TextResponse applicationPrometheusResponse(String requestedConsumer) { + private HttpResponse applicationPrometheusResponse(String requestedConsumer) { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); var metricsByNode = metricsRetriever.getMetrics(consumer); @@ -87,7 +88,7 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { .map(builder -> builder.putDimension(DimensionId.toDimensionId("hostname"), element.hostname)) .map(MetricsPacket.Builder::build)) .toList(); - return new TextResponse(200, toPrometheusModel(metricsForAllNodes).serialize()); + return new PrometheusResponse(200, toPrometheusModel(metricsForAllNodes)); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java index 3e4565c780b..50c1420edef 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java @@ -3,6 +3,7 @@ package ai.vespa.metricsproxy.http.metrics; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.http.MetricsJsonResponse; import ai.vespa.metricsproxy.http.ValuesFetcher; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.VespaServices; @@ -51,10 +52,10 @@ public class MetricsV1Handler extends HttpHandlerBase { return Optional.empty(); } - private JsonResponse valuesResponse(String consumer) { + private HttpResponse valuesResponse(String consumer) { try { List<MetricsPacket> metrics = valuesFetcher.fetch(consumer); - return new JsonResponse(OK, toGenericJsonModel(metrics).serialize()); + return new MetricsJsonResponse(OK, toGenericJsonModel(metrics)::serialize); } catch (Exception e) { log.log(Level.WARNING, "Got exception when rendering metrics:", e); return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java index 1f6cc17b2e1..7e9bba466df 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java @@ -3,6 +3,7 @@ package ai.vespa.metricsproxy.http.metrics; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.http.MetricsJsonResponse; import ai.vespa.metricsproxy.http.ValuesFetcher; import ai.vespa.metricsproxy.http.application.ClusterIdDimensionProcessor; import ai.vespa.metricsproxy.http.application.Node; @@ -62,7 +63,7 @@ public class MetricsV2Handler extends HttpHandlerBase { return Optional.empty(); } - private JsonResponse valuesResponse(String consumer) { + private HttpResponse valuesResponse(String consumer) { try { List<MetricsPacket> metrics = processAndBuild(valuesFetcher.fetchMetricsAsBuilders(consumer), new ServiceIdDimensionProcessor(), @@ -71,7 +72,7 @@ public class MetricsV2Handler extends HttpHandlerBase { Node localNode = new Node(nodeInfoConfig.role(), nodeInfoConfig.hostname(), 0, ""); Map<Node, List<MetricsPacket>> metricsByNode = Map.of(localNode, metrics); - return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize()); + return new MetricsJsonResponse(OK, toGenericApplicationModel(metricsByNode)::serialize); } catch (Exception e) { log.log(Level.WARNING, "Got exception when rendering metrics:", e); return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java index d73561b5eff..e609b54b916 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java @@ -3,6 +3,7 @@ package ai.vespa.metricsproxy.http.prometheus; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.http.PrometheusResponse; import ai.vespa.metricsproxy.http.TextResponse; import ai.vespa.metricsproxy.http.ValuesFetcher; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; @@ -56,11 +57,11 @@ public class PrometheusHandler extends HttpHandlerBase { return Optional.empty(); } - private TextResponse valuesResponse(String consumer) { + private HttpResponse valuesResponse(String consumer) { try { List<MetricsPacket> metrics = new ArrayList<>(valuesFetcher.fetch(consumer)); metrics.addAll(nodeMetricGatherer.gatherMetrics()); - return new TextResponse(OK, toPrometheusModel(metrics).serialize()); + return new PrometheusResponse(OK, toPrometheusModel(metrics)); } catch (Exception e) { log.log(Level.WARNING, "Got exception when rendering metrics:", e); return new TextResponse(INTERNAL_SERVER_ERROR, e.getMessage()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModel.java index 59d1f9e5c83..105724d17f8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModel.java @@ -5,10 +5,13 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.List; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; +import static java.nio.charset.StandardCharsets.UTF_8; /** * @author gjoranv @@ -21,8 +24,14 @@ public class GenericApplicationModel { public List<GenericJsonModel> nodes; public String serialize() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + serialize(out); + return out.toString(UTF_8); + } + + public void serialize(OutputStream out) { try { - return JacksonUtil.objectMapper().writeValueAsString(this); + JacksonUtil.objectMapper().writeValue(out, this); } catch (IOException e) { throw new JsonRenderingException("Could not render application nodes. Check the log for details.", e); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java index c4ba0f39a18..54616dff759 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java @@ -6,10 +6,13 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.List; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; +import static java.nio.charset.StandardCharsets.UTF_8; /** * @author gjoranv @@ -32,8 +35,14 @@ public class GenericJsonModel { public List<GenericService> services; public String serialize() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + serialize(out); + return out.toString(UTF_8); + } + + public void serialize(OutputStream out) { try { - return JacksonUtil.objectMapper().writeValueAsString(this); + JacksonUtil.objectMapper().writeValue(out, this); } catch (IOException e) { throw new JsonRenderingException("Could not render metrics. Check the log for details.", e); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java index fd00ad67ec0..0f3878821f3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java @@ -11,6 +11,7 @@ import io.prometheus.client.Collector.MetricFamilySamples.Sample; import io.prometheus.client.exporter.common.TextFormat; import java.io.StringWriter; +import java.io.Writer; import java.util.ArrayList; import java.util.Enumeration; import java.util.Iterator; @@ -50,12 +51,16 @@ public class PrometheusModel implements Enumeration<MetricFamilySamples> { public String serialize() { var writer = new StringWriter(); + serialize(writer); + return writer.toString(); + } + + public void serialize(Writer writer) { try { TextFormat.write004(writer, this); } catch (Exception e) { throw new PrometheusRenderingException("Could not render metrics. Check the log for details.", e); } - return writer.toString(); } private MetricFamilySamples createMetricFamily(MetricId metricId) { @@ -70,6 +75,7 @@ public class PrometheusModel implements Enumeration<MetricFamilySamples> { })); return new MetricFamilySamples(metricId.getIdForPrometheus(), Collector.Type.UNKNOWN, "", sampleList); } + private static Sample createSample(ServiceId serviceId, MetricId metricId, Number metric, Long timeStamp, Map<DimensionId, String> dimensions) { diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index 197fe7e9817..a92f2ed9192 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_define_module( DEPENDS vespalog vespalib - config_cloudconfig + vespa_config LIBS src/vespa/metrics diff --git a/metrics/src/main/java/ai/vespa/metrics/ClusterControllerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ClusterControllerMetrics.java index 12bdc50700a..f15b7412b24 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ClusterControllerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ClusterControllerMetrics.java @@ -13,6 +13,8 @@ public enum ClusterControllerMetrics implements VespaMetrics { STOPPING_COUNT("cluster-controller.stopping.count", Unit.NODE, "Number of content nodes currently stopping"), UP_COUNT("cluster-controller.up.count", Unit.NODE, "Number of content nodes up"), CLUSTER_STATE_CHANGE_COUNT("cluster-controller.cluster-state-change.count", Unit.NODE, "Number of nodes changing state"), + NODES_NOT_CONVERGED("cluster-controller.nodes-not-converged", Unit.NODE, "Number of nodes not converging to the latest cluster state version"), + CLUSTER_BUCKETS_OUT_OF_SYNC_RATIO("cluster-controller.cluster-buckets-out-of-sync-ratio", Unit.FRACTION, "Ratio of buckets in the cluster currently in need of syncing"), BUSY_TICK_TIME_MS("cluster-controller.busy-tick-time-ms", Unit.MILLISECOND, "Time busy"), IDLE_TICK_TIME_MS("cluster-controller.idle-tick-time-ms", Unit.MILLISECOND, "Time idle"), WORK_MS("cluster-controller.work-ms", Unit.MILLISECOND, "Time used for actual work"), diff --git a/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java index c413dc5e7d7..938fb9d982e 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java @@ -66,6 +66,9 @@ public enum ConfigServerMetrics implements VespaMetrics { CLUSTER_LOAD_IDEAL_CPU("cluster.load.ideal.cpu", Unit.FRACTION, "The ideal cpu load of a certain cluster"), CLUSTER_LOAD_IDEAL_MEMORY("cluster.load.ideal.memory", Unit.FRACTION, "The ideal memory load of a certain cluster"), CLUSTER_LOAD_IDEAL_DISK("cluster.load.ideal.disk", Unit.FRACTION, "The ideal disk load of a certain cluster"), + CLUSTER_LOAD_PEAK_CPU("cluster.load.peak.cpu", Unit.FRACTION, "The peak cpu load in the period considered of a certain cluster"), + CLUSTER_LOAD_PEAK_MEMORY("cluster.load.peak.memory", Unit.FRACTION, "The peak memory load in the period considered of a certain cluster"), + CLUSTER_LOAD_PEAK_DISK("cluster.load.peak.disk", Unit.FRACTION, "The peak disk load in the period considered of a certain cluster"), ZONE_WORKING("zone.working", Unit.BINARY, "The value 1 if zone is considered healthy, 0 if not. This is decided by considering the number of non-active nodes vs the number of active nodes in a zone"), CACHE_NODE_OBJECT_HIT_RATE("cache.nodeObject.hitRate", Unit.FRACTION, "The fraction of cache hits vs cache lookups for the node object cache"), @@ -90,6 +93,7 @@ public enum ConfigServerMetrics implements VespaMetrics { FAIL_REPORT("failReport", Unit.BINARY, "One if there is a fail report for the node, zero if not"), SUSPENDED("suspended", Unit.BINARY, "One if the node is suspended, zero if not"), SUSPENDED_SECONDS("suspendedSeconds", Unit.SECOND, "The number of seconds the node has been suspended"), + ACTIVE_SECONDS("activeSeconds", Unit.SECOND, "The number of seconds the node has been active"), NUMBER_OF_SERVICES_UP("numberOfServicesUp", Unit.INSTANCE, "The number of services confirmed to be running on a node"), NUMBER_OF_SERVICES_NOT_CHECKED("numberOfServicesNotChecked", Unit.INSTANCE, "The number of services supposed to run on a node, that has not checked"), NUMBER_OF_SERVICES_DOWN("numberOfServicesDown", Unit.INSTANCE, "The number of services confirmed to not be running on a node"), diff --git a/metrics/src/main/java/ai/vespa/metrics/StorageMetrics.java b/metrics/src/main/java/ai/vespa/metrics/StorageMetrics.java index 4d91cc1d989..8718dc0076a 100644 --- a/metrics/src/main/java/ai/vespa/metrics/StorageMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/StorageMetrics.java @@ -39,8 +39,8 @@ public enum StorageMetrics implements VespaMetrics { VDS_FILESTOR_ALLTHREADS_MERGEBUCKETS_FAILED("vds.filestor.allthreads.mergebuckets.failed", Unit.REQUEST, "Number of failed requests."), VDS_FILESTOR_ALLTHREADS_MERGEBUCKETS_LATENCY("vds.filestor.allthreads.mergebuckets.latency", Unit.MILLISECOND, "Latency of successful requests."), VDS_FILESTOR_ALLTHREADS_MERGELATENCYTOTAL("vds.filestor.allthreads.mergelatencytotal", Unit.MILLISECOND, "Latency of total merge operation, from master node receives it, until merge is complete and master node replies."), - VDS_FILESTOR_ALLTHREADS_MERGE_PUT_LATENCY("vds.filestor.allthreads.put_latency", Unit.MILLISECOND, "Latency of individual puts that are part of merge operations"), // TODO Vespa 9: Update metric name to include 'merge' - VDS_FILESTOR_ALLTHREADS_MERGE_REMOVE_LATENCY("vds.filestor.allthreads.remove_latency", Unit.MILLISECOND, "Latency of individual removes that are part of merge operations"), // TODO Vespa 9: Update metric name to include 'merge' + VDS_FILESTOR_ALLTHREADS_MERGE_PUT_LATENCY("vds.filestor.allthreads.merge_put_latency", Unit.MILLISECOND, "Latency of individual puts that are part of merge operations"), + VDS_FILESTOR_ALLTHREADS_MERGE_REMOVE_LATENCY("vds.filestor.allthreads.merge_remove_latency", Unit.MILLISECOND, "Latency of individual removes that are part of merge operations"), VDS_FILESTOR_ALLSTRIPES_THROTTLED_RPC_DIRECT_DISPATCHES("vds.filestor.allstripes.throttled_rpc_direct_dispatches", Unit.INSTANCE, "Number of times an RPC thread could not directly dispatch an async operation directly to Proton because it was disallowed by the throttle policy"), VDS_FILESTOR_ALLSTRIPES_THROTTLED_PERSISTENCE_THREAD_POLLS("vds.filestor.allstripes.throttled_persistence_thread_polls", Unit.INSTANCE, "Number of times a persistence thread could not immediately dispatch a queued async operation because it was disallowed by the throttle policy"), VDS_FILESTOR_ALLSTRIPES_TIMEOUTS_WAITING_FOR_THROTTLE_TOKEN("vds.filestor.allstripes.timeouts_waiting_for_throttle_token", Unit.INSTANCE, "Number of times a persistence thread timed out waiting for an available throttle policy token"), diff --git a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java index 8c500473678..08b7be0a421 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java @@ -67,6 +67,9 @@ public class InfrastructureMetricSet { addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_CPU.max()); addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_MEMORY.max()); addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_DISK.max()); + addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_PEAK_CPU.max()); + addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_PEAK_MEMORY.max()); + addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_PEAK_DISK.max()); addMetric(metrics, ConfigServerMetrics.NODES_EMPTY_EXCLUSIVE.max()); addMetric(metrics, ConfigServerMetrics.NODES_EXPIRED_DEPROVISIONED.count()); addMetric(metrics, ConfigServerMetrics.NODES_EXPIRED_DIRTY.count()); @@ -82,6 +85,7 @@ public class InfrastructureMetricSet { addMetric(metrics, ConfigServerMetrics.WANT_TO_DEPROVISION.max()); addMetric(metrics, ConfigServerMetrics.SUSPENDED.max()); addMetric(metrics, ConfigServerMetrics.SUSPENDED_SECONDS.count()); + addMetric(metrics, ConfigServerMetrics.ACTIVE_SECONDS.count()); addMetric(metrics, ConfigServerMetrics.SOME_SERVICES_DOWN.max()); addMetric(metrics, ConfigServerMetrics.NODE_FAILER_BAD_NODE.max()); addMetric(metrics, ConfigServerMetrics.LOCK_ATTEMPT_LOCKED_LOAD, EnumSet.of(max,average)); @@ -187,6 +191,8 @@ public class InfrastructureMetricSet { addMetric(metrics, ControllerMetrics.AUTH0_EXCEPTIONS.count()); addMetric(metrics, ControllerMetrics.BILLING_WEBHOOK_FAILURES.count()); addMetric(metrics, ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.max()); + addMetric(metrics, ControllerMetrics.CERTIFICATE_COUNT.max()); + addMetric(metrics, ControllerMetrics.CERTIFICATE_NAME_COUNT.max()); addMetric(metrics, ControllerMetrics.BILLING_EXCEPTIONS.count()); addMetric(metrics, ControllerMetrics.METERING_AGE_SECONDS.min()); diff --git a/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java index 957eaf8304f..0d5827369fd 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java @@ -226,6 +226,8 @@ public class Vespa9VespaMetricSet { addMetric(metrics, ClusterControllerMetrics.MAINTENANCE_COUNT.max()); addMetric(metrics, ClusterControllerMetrics.RETIRED_COUNT.max()); addMetric(metrics, ClusterControllerMetrics.UP_COUNT.max()); + addMetric(metrics, ClusterControllerMetrics.NODES_NOT_CONVERGED.max()); + addMetric(metrics, ClusterControllerMetrics.CLUSTER_BUCKETS_OUT_OF_SYNC_RATIO.max()); addMetric(metrics, ClusterControllerMetrics.CLUSTER_STATE_CHANGE_COUNT.baseName()); addMetric(metrics, ClusterControllerMetrics.BUSY_TICK_TIME_MS, EnumSet.of(max, sum, count)); addMetric(metrics, ClusterControllerMetrics.IDLE_TICK_TIME_MS, EnumSet.of(max, sum, count)); diff --git a/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java index 778d6963e19..0e6c537f56d 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java @@ -257,7 +257,9 @@ public class VespaMetricSet { addMetric(metrics, ClusterControllerMetrics.RETIRED_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ClusterControllerMetrics.STOPPING_COUNT.last()); addMetric(metrics, ClusterControllerMetrics.UP_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.NODES_NOT_CONVERGED.max()); addMetric(metrics, ClusterControllerMetrics.CLUSTER_STATE_CHANGE_COUNT.baseName()); + addMetric(metrics, ClusterControllerMetrics.CLUSTER_BUCKETS_OUT_OF_SYNC_RATIO.max()); addMetric(metrics, ClusterControllerMetrics.BUSY_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); // TODO: Vespa 9: Remove last addMetric(metrics, ClusterControllerMetrics.IDLE_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); // TODO: Vespa 9: Remove last diff --git a/metrics/src/tests/CMakeLists.txt b/metrics/src/tests/CMakeLists.txt index d46ff6ff0dd..9b31b60f78c 100644 --- a/metrics/src/tests/CMakeLists.txt +++ b/metrics/src/tests/CMakeLists.txt @@ -13,7 +13,7 @@ vespa_add_executable(metrics_gtest_runner_app TEST valuemetrictest.cpp gtest_runner.cpp DEPENDS - metrics + vespa_metrics GTest::GTest ) diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 06a5febaea7..417e79097a1 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(metrics +vespa_add_library(vespa_metrics SOURCES countmetric.cpp countmetricvalues.cpp @@ -24,5 +24,5 @@ vespa_add_library(metrics INSTALL lib64 DEPENDS ) -vespa_generate_config(metrics metricsmanager.def) +vespa_generate_config(vespa_metrics metricsmanager.def) install_config_definition(metricsmanager.def metrics.metricsmanager.def) diff --git a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java index fd02756e2ea..bbb82db7139 100644 --- a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java +++ b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java @@ -114,6 +114,8 @@ public class LocalLLM extends AbstractComponent implements LanguageModel { options.ifPresent("repeatpenalty", (v) -> inferParams.setRepeatPenalty(Float.parseFloat(v))); // Todo: more options? + inferParams.setUseChatTemplate(true); + var completionFuture = new CompletableFuture<Completion.FinishReason>(); var hasStarted = new AtomicBoolean(false); try { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 36221d978a1..402b5941701 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -627,7 +627,7 @@ public final class Node implements Nodelike { NodeResources allocated = all.subtract(freeHostCapacity.justNumbers()); return new Mean(allocated.vcpu() / all.vcpu(), - allocated.memoryGb() / all.memoryGb(), + allocated.memoryGiB() / all.memoryGiB(), allocated.diskGb() / all.diskGb()) .deviation(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index bed3d9fcb04..d282432d60f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -102,9 +102,9 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { return except(Set.of(node)); } - /** Returns the subset of nodes assigned to the given cluster type */ - public NodeList type(ClusterSpec.Type type) { - return matching(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().equals(type)); + /** Returns the subset of nodes assigned to the given cluster types */ + public NodeList type(ClusterSpec.Type... types) { + return matching(node -> node.allocation().isPresent() && Set.of(types).contains(node.allocation().get().membership().cluster().type())); } /** Returns the subset of nodes that run containers */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index 98c5af2688a..b7ad29af7fe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -48,8 +48,11 @@ public class AllocatableResources { } public AllocatableResources(NodeList nodes, NodeRepository nodeRepository) { + if (nodes.isEmpty()) { + throw new IllegalArgumentException("Expected a non-empty node list"); + } this.nodes = nodes.size(); - this.groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + this.groups = (int) nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); this.realResources = averageRealResourcesOf(nodes.asList(), nodeRepository); // Average since we average metrics over nodes this.advertisedResources = nodes.requestedResources(); this.clusterSpec = nodes.clusterSpec(); @@ -82,19 +85,6 @@ public class AllocatableResources { this.fulfilment = fulfilment; } - /** Returns this with the redundant node or group removed from counts. */ - public AllocatableResources withoutRedundancy() { - int groupSize = nodes / groups; - int nodesAdjustedForRedundancy = nodes > 1 ? (groups == 1 ? nodes - 1 : nodes - groupSize) : nodes; - int groupsAdjustedForRedundancy = nodes > 1 ? (groups == 1 ? 1 : groups - 1) : groups; - return new AllocatableResources(nodesAdjustedForRedundancy, - groupsAdjustedForRedundancy, - realResources, - advertisedResources, - clusterSpec, - fulfilment); - } - /** * Returns the resources which will actually be available per node in this cluster with this allocation. * These should be used for reasoning about allocation to meet measured demand. @@ -126,15 +116,23 @@ public class AllocatableResources { */ public double fulfilment() { return fulfilment; } + public boolean notFulfiled() { + return fulfilment < 0.9999999; + } + private static double fulfilment(ClusterResources realResources, ClusterResources idealResources) { double vcpuFulfilment = Math.min(1, realResources.totalResources().vcpu() / idealResources.totalResources().vcpu()); - double memoryGbFulfilment = Math.min(1, realResources.totalResources().memoryGb() / idealResources.totalResources().memoryGb()); + double memoryGbFulfilment = Math.min(1, realResources.totalResources().memoryGiB() / idealResources.totalResources().memoryGiB()); double diskGbFulfilment = Math.min(1, realResources.totalResources().diskGb() / idealResources.totalResources().diskGb()); - return (vcpuFulfilment + memoryGbFulfilment + diskGbFulfilment) / 3; + double fulfilment = (vcpuFulfilment + memoryGbFulfilment + diskGbFulfilment) / 3; + if (equal(fulfilment, 0)) return 0; + if (equal(fulfilment, 1)) return 1; + return fulfilment; } public boolean preferableTo(AllocatableResources other, ClusterModel model) { - if (other.fulfilment() < 1 || this.fulfilment() < 1) // always fulfil as much as possible + // always fulfil as much as possible unless fulfilment is considered to be equal + if ((other.fulfilment() < 1 || this.fulfilment() < 1) && ! equal(this.fulfilment(), other.fulfilment())) return this.fulfilment() > other.fulfilment(); return this.cost() * toHours(model.allocationDuration()) + this.costChangingFrom(model) @@ -165,7 +163,7 @@ public class AllocatableResources { } return nodes.get(0).allocation().get().requestedResources() .withVcpu(sum.vcpu() / nodes.size()) - .withMemoryGb(sum.memoryGb() / nodes.size()) + .withMemoryGiB(sum.memoryGiB() / nodes.size()) .withDiskGb(sum.diskGb() / nodes.size()) .withBandwidthGbps(sum.bandwidthGbps() / nodes.size()); } @@ -292,4 +290,8 @@ public class AllocatableResources { return true; } + private static boolean equal(double a, double b) { + return Math.abs(a - b) < 1e-9; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 82199888a48..186b8c76f84 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -133,7 +133,7 @@ public class AllocationOptimizer { var nonScaled = limits.isEmpty() || limits.min().nodeResources().isUnspecified() ? model.current().advertisedResources().nodeResources() : limits.min().nodeResources(); // min=max for non-scaled - return nonScaled.withVcpu(scaled.vcpu()).withMemoryGb(scaled.memoryGb()).withDiskGb(scaled.diskGb()); + return nonScaled.withVcpu(scaled.vcpu()).withMemoryGiB(scaled.memoryGiB()).withDiskGb(scaled.diskGb()); } /** Returns a copy of the given limits where the minimum nodes are at least the given value when allowed */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 29ab6d65b9f..88c0ac80726 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -69,11 +69,9 @@ public class Autoscaler { return Autoscaling.dontScale(Status.waiting, "Cluster change in progress", model); var loadAdjustment = model.loadAdjustment(); - if (enableDetailedLogging) { + if (enableDetailedLogging) log.info("Application: " + application.id().toShortString() + ", loadAdjustment: " + loadAdjustment.toString()); - } - // Ensure we only scale down if we'll have enough headroom to not scale up again given a small load increase var target = allocationOptimizer.findBestAllocation(loadAdjustment, model, limits, enableDetailedLogging); if (target.isEmpty()) @@ -98,8 +96,8 @@ public class Autoscaler { return Autoscaling.dontScale(Status.unavailable, "Autoscaling is disabled in single node clusters", model); if (! worthRescaling(model.current().realResources(), target.realResources())) { - if (target.fulfilment() < 0.9999999) - return Autoscaling.dontScale(Status.insufficient, "Configured limits prevents ideal scaling of this cluster", model); + if (target.notFulfiled()) + return Autoscaling.dontScale(Status.insufficient, "Cluster cannot be scaled to achieve ideal load", model); else if ( ! model.safeToScaleDown() && model.idealLoad().any(v -> v < 1.0)) return Autoscaling.dontScale(Status.ideal, "Cooling off before considering to scale down", model); else @@ -112,7 +110,7 @@ public class Autoscaler { public static boolean worthRescaling(ClusterResources from, ClusterResources to) { // *Increase* if needed with no regard for cost difference to prevent running out of a resource if (meaningfulIncrease(from.totalResources().vcpu(), to.totalResources().vcpu())) return true; - if (meaningfulIncrease(from.totalResources().memoryGb(), to.totalResources().memoryGb())) return true; + if (meaningfulIncrease(from.totalResources().memoryGiB(), to.totalResources().memoryGiB())) return true; if (meaningfulIncrease(from.totalResources().diskGb(), to.totalResources().diskGb())) return true; // Otherwise, only *decrease* if diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index e05b1c53c76..c6975ecbb83 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; -import io.questdb.Metrics; import java.time.Instant; import java.util.Objects; @@ -102,7 +101,7 @@ public class Autoscaling { @Override public String toString() { - return (resources.isPresent() ? "Autoscaling to " + resources : "Don't autoscale") + + return resources.map(r -> "Autoscaling to " + r).orElse("Don't autoscale") + (description.isEmpty() ? "" : ": " + description); } @@ -144,7 +143,7 @@ public class Autoscaling { public enum Status { - /** No status is available: Aautoscaling is disabled, or a brand new application. */ + /** No status is available: Autoscaling is disabled, or a brand new application. */ unavailable, /** Autoscaling is not taking any action at the moment due to recent changes or a lack of data */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 504965f1992..8ae06c8e7d2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -32,7 +32,7 @@ public class ClusterModel { static final double idealQueryCpuLoad = 0.75; static final double idealWriteCpuLoad = 0.95; - static final double idealContainerMemoryLoad = 0.8; + static final double idealContainerMemoryLoad = 0.9; static final double idealContentMemoryLoad = 0.65; static final double idealContainerDiskLoad = 0.95; @@ -125,6 +125,21 @@ public class ClusterModel { this.at = clock.instant(); } + + /** + * The central decision made in autoscaling. + * + * @return the relative load adjustment that should be made to this cluster given available measurements. + * For example, a load adjustment of 2 means we should allocate twice the amount of that resources. + */ + public Load loadAdjustment() { + if (nodeTimeseries().measurementsPerNode() < 0.5) return Load.one(); // Don't change based on very little data + Load adjustment = peakLoad().divide(idealLoad()); + if (! safeToScaleDown()) + adjustment = adjustment.map(v -> v < 1 ? 1 : v); + return adjustment; + } + public Application application() { return application; } public ClusterSpec clusterSpec() { return clusterSpec; } public CloudAccount cloudAccount() { return cluster.cloudAccount().orElse(CloudAccount.empty); } @@ -133,11 +148,7 @@ public class ClusterModel { private ClusterTimeseries clusterTimeseries() { return clusterTimeseries; } /** Returns the instant this model was created. */ - public Instant at() { return at;} - - public boolean isEmpty() { - return nodeTimeseries().isEmpty(); - } + public Instant at() { return at; } /** Returns the predicted duration of a rescaling of this cluster */ public Duration scalingDuration() { return scalingDuration; } @@ -148,9 +159,9 @@ public class ClusterModel { */ public Duration allocationDuration() { return allocationDuration; } - public boolean isContent() { - return clusterSpec.type().isContent(); - } + public boolean isEmpty() { return nodeTimeseries().isEmpty(); } + + public boolean isContent() { return clusterSpec.type().isContent(); } /** Returns the predicted duration of data redistribution in this cluster. */ public Duration redistributionDuration() { @@ -177,15 +188,6 @@ public class ClusterModel { return nodeRepository.exclusivity().allocation(clusterSpec); } - /** Returns the relative load adjustment that should be made to this cluster given available measurements. */ - public Load loadAdjustment() { - if (nodeTimeseries().measurementsPerNode() < 0.5) return Load.one(); // Don't change based on very little data - Load adjustment = peakLoad().divide(idealLoad()); - if (! safeToScaleDown()) - adjustment = adjustment.map(v -> v < 1 ? 1 : v); - return adjustment; - } - public boolean isStable(NodeRepository nodeRepository) { // The cluster is processing recent changes if (nodes.stream().anyMatch(node -> node.status().wantToRetire() || @@ -218,7 +220,6 @@ public class ClusterModel { .divide(redundancyAdjustment()); // correct for double redundancy adjustment } - /** * Returns the relative load adjustment accounting for redundancy given these nodes+groups * relative to node nodes+groups in this. @@ -443,11 +444,11 @@ public class ClusterModel { return nodeRepository.resourcesCalculator().requestToReal(initialResources, cloudAccount(), nodeRepository.exclusivity().allocation(clusterSpec), - false).memoryGb(); + false).memoryGiB(); } else { return nodes.stream() - .mapToDouble(node -> nodeRepository.resourcesCalculator().realResourcesOf(node, nodeRepository).memoryGb()) + .mapToDouble(node -> nodeRepository.resourcesCalculator().realResourcesOf(node, nodeRepository).memoryGiB()) .average() .getAsDouble(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index 51046df90af..960913e9e46 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -59,7 +59,7 @@ public class Limits { if (isEmpty()) return resources; if (min.nodeResources().isUnspecified()) return resources; // means max is also unspecified resources = resources.withVcpu(between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), resources.vcpu())); - resources = resources.withMemoryGb(between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), resources.memoryGb())); + resources = resources.withMemoryGiB(between(min.nodeResources().memoryGiB(), max.nodeResources().memoryGiB(), resources.memoryGiB())); resources = resources.withDiskGb(between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), resources.diskGb())); return resources; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java index 22c13795d18..e83016cc6f1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java @@ -41,7 +41,7 @@ public record Load(double cpu, double memory, double disk, double gpu, double gp } public Load multiply(NodeResources resources) { - return new Load(cpu * resources.vcpu(), memory * resources.memoryGb(), disk * resources.diskGb(), gpu * resources.gpuResources().count(), gpu * resources.gpuResources().memoryGb()); + return new Load(cpu * resources.vcpu(), memory * resources.memoryGiB(), disk * resources.diskGb(), gpu * resources.gpuResources().count(), gpu * resources.gpuResources().memoryGiB()); } public Load multiply(double factor) { return map(v -> v * factor); @@ -57,7 +57,7 @@ public record Load(double cpu, double memory, double disk, double gpu, double gp return map(v -> divide(v, divisor)); } public Load divide(NodeResources resources) { - return new Load(divide(cpu, resources.vcpu()), divide(memory, resources.memoryGb()), divide(disk, resources.diskGb()), divide(gpu, resources.gpuResources().count()), divide(gpuMemory, resources.gpuResources().memoryGb())); + return new Load(divide(cpu, resources.vcpu()), divide(memory, resources.memoryGiB()), divide(disk, resources.diskGb()), divide(gpu, resources.gpuResources().count()), divide(gpuMemory, resources.gpuResources().memoryGiB())); } /** Returns the load where the given function is applied to each dimension of this. */ @@ -85,7 +85,7 @@ public record Load(double cpu, double memory, double disk, double gpu, double gp public NodeResources scaled(NodeResources resources) { return resources.withVcpu(cpu * resources.vcpu()) - .withMemoryGb(memory * resources.memoryGb()) + .withMemoryGiB(memory * resources.memoryGiB()) .withDiskGb(disk * resources.diskGb()); } @@ -114,7 +114,7 @@ public record Load(double cpu, double memory, double disk, double gpu, double gp @Override public String toString() { - return "load: " + cpu + " cpu, " + memory + " memory, " + disk + " disk," + gpu + " gpu," + gpuMemory + " gpuMemory"; + return "load: " + cpu + " cpu, " + memory + " memory, " + disk + " disk, " + gpu + " gpu, " + gpuMemory + " gpuMemory"; } public static Load zero() { return new Load(0, 0, 0, 0, 0); } @@ -122,10 +122,10 @@ public record Load(double cpu, double memory, double disk, double gpu, double gp public static Load byDividing(NodeResources a, NodeResources b) { return new Load(divide(a.vcpu(), b.vcpu()), - divide(a.memoryGb(), b.memoryGb()), + divide(a.memoryGiB(), b.memoryGiB()), divide(a.diskGb(), b.diskGb()), divide(a.gpuResources().count(), b.gpuResources().count()), - divide(a.gpuResources().memoryGb(), b.gpuResources().memoryGb())); + divide(a.gpuResources().memoryGiB(), b.gpuResources().memoryGiB())); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index 0135f89c47e..1ec2e43bac0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -80,6 +80,17 @@ public class LoadBalancer { return new LoadBalancer(id, idSeed, Optional.of(instance), state, changedAt); } + @Override + public String toString() { + return "LoadBalancer{" + + "id=" + id + + ", idSeed='" + idSeed + '\'' + + ", instance=" + instance + + ", state=" + state + + ", changedAt=" + changedAt + + '}'; + } + /** Returns the effective container ID of given cluster. For combined clusters this returns the ID of the container cluster */ public static ClusterSpec.Id containerId(ClusterSpec cluster) { return cluster.combinedId().orElse(cluster.id()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 5c9c5fe30d7..471b9d0122f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -91,6 +91,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Cluster unchangedCluster = cluster; NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); + if (clusterNodes.isEmpty()) return true; // Cluster was removed since we started cluster = updateCompletion(cluster, clusterNodes); var current = new AllocatableResources(clusterNodes.not().retired(), nodeRepository()).advertisedResources(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index 83d5657c1f6..30c3c503cc2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -56,7 +56,7 @@ public class CapacityChecker { List<Node> overcommittedNodes = new ArrayList<>(); for (var entry : availableResources.entrySet()) { var resources = entry.getValue().nodeResources; - if (resources.vcpu() < 0 || resources.memoryGb() < 0 || resources.diskGb() < 0) { + if (resources.vcpu() < 0 || resources.memoryGiB() < 0 || resources.diskGb() < 0) { overcommittedNodes.add(entry.getKey()); } } @@ -299,7 +299,7 @@ public class CapacityChecker { if (l.vcpu() < r.vcpu()) reason.insufficientVcpu = true; - if (l.memoryGb() < r.memoryGb()) + if (l.memoryGiB() < r.memoryGiB()) reason.insufficientMemoryGb = true; if (l.diskGb() < r.diskGb()) reason.insufficientDiskGb = true; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java index 4026a294111..8a532131d90 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.ThrottleProvisioningException; import java.time.Duration; import java.util.logging.Level; @@ -48,6 +49,9 @@ public class HostDeprovisioner extends NodeRepositoryMaintainer { // if we want to support aborting deprovision if operator manually intervenes if (hostProvisioner.deprovision(host)) nodeRepository().nodes().removeRecursively(host, true); + } catch (ThrottleProvisioningException e) { + log.log(Level.INFO, "Failed to deprovision " + host.hostname() + ", will retry in " + interval() + ": " + e.getMessage()); + break; } catch (RuntimeException e) { failures++; log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index 2e3c6d1755a..08243290a28 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.ThrottleProvisioningException; import com.yahoo.yolean.Exceptions; import javax.naming.NamingException; @@ -60,6 +61,9 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { } catch (IllegalArgumentException | IllegalStateException e) { log.log(Level.INFO, "Could not provision " + host.hostname() + ", will retry in " + interval() + ": " + Exceptions.toMessageString(e)); + } catch (ThrottleProvisioningException e) { + log.log(Level.INFO, "Failed to provision " + host.hostname() + ", will retry in " + interval() + ": " + e.getMessage()); + break; } catch (FatalProvisioningException e) { // FatalProvisioningException is thrown if node is not found in the cloud, allow for // some time for the state to propagate diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index e3d72d1189e..e2931b70dec 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -6,6 +6,7 @@ import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; @@ -143,6 +144,9 @@ public class MetricsReporter extends NodeRepositoryMaintainer { metric.set(ConfigServerMetrics.CLUSTER_LOAD_IDEAL_CPU.baseName(), cluster.get().target().ideal().cpu(), context); metric.set(ConfigServerMetrics.CLUSTER_LOAD_IDEAL_MEMORY.baseName(), cluster.get().target().ideal().memory(), context); metric.set(ConfigServerMetrics.CLUSTER_LOAD_IDEAL_DISK.baseName(), cluster.get().target().ideal().disk(), context); + metric.set(ConfigServerMetrics.CLUSTER_LOAD_PEAK_CPU.baseName(), cluster.get().target().peak().cpu(), context); + metric.set(ConfigServerMetrics.CLUSTER_LOAD_PEAK_MEMORY.baseName(), cluster.get().target().peak().memory(), context); + metric.set(ConfigServerMetrics.CLUSTER_LOAD_PEAK_DISK.baseName(), cluster.get().target().peak().disk(), context); } private void updateZoneMetrics() { @@ -235,6 +239,16 @@ public class MetricsReporter extends NodeRepositoryMaintainer { .orElse(0L); metric.add(ConfigServerMetrics.SUSPENDED_SECONDS.baseName(), suspendedSeconds, context); }); + if (nodeRepository().zone().environment().isTest() && + node.state() == State.active && + node.type() == NodeType.tenant) { + node.history() + .event(History.Event.Type.activated) + .ifPresent(event -> { + var activeSeconds = Duration.between(event.at(), clock().instant()).getSeconds(); + metric.add(ConfigServerMetrics.ACTIVE_SECONDS.baseName(), activeSeconds, context); + }); + } long numberOfServices; List<ServiceInstance> services = serviceModel.getServiceInstancesByHostName().get(hostname); @@ -356,12 +370,12 @@ public class MetricsReporter extends NodeRepositoryMaintainer { private void updateContainerMetrics(NodeList nodes) { NodeResources totalCapacity = getCapacityTotal(nodes); metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_CPU.baseName(), totalCapacity.vcpu(), null); - metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_MEM.baseName(), totalCapacity.memoryGb(), null); + metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_MEM.baseName(), totalCapacity.memoryGiB(), null); metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_DISK.baseName(), totalCapacity.diskGb(), null); NodeResources totalFreeCapacity = getFreeCapacityTotal(nodes); metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_CPU.baseName(), totalFreeCapacity.vcpu(), null); - metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_MEM.baseName(), totalFreeCapacity.memoryGb(), null); + metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_MEM.baseName(), totalFreeCapacity.memoryGiB(), null); metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_DISK.baseName(), totalFreeCapacity.diskGb(), null); } @@ -378,7 +392,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { var context = getContext(dimensions(applicationId)); metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_CPU.baseName(), allocatedCapacity.vcpu(), context); - metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_MEM.baseName(), allocatedCapacity.memoryGb(), context); + metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_MEM.baseName(), allocatedCapacity.memoryGiB(), context); metric.set(ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_DISK.baseName(), allocatedCapacity.diskGb(), context); } ); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index 0ff8acdcf1f..7f13e3a5bca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -75,9 +76,10 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { MutableInteger failures, ApplicationId application) { if (exception != null) { - if (failures.get() < maxWarningsPerInvocation) + if (nodeRepository().zone().environment() == Environment.prod + && failures.get() < maxWarningsPerInvocation) log.log(Level.WARNING, "Could not update metrics for " + application + ": " + - Exceptions.toMessageString(exception)); + Exceptions.toMessageString(exception)); failures.add(1); } else if (response != null) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java index c7be505acaa..678e4aa5781 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java @@ -58,7 +58,7 @@ public abstract class NodeRepositoryMaintainer extends Maintainer { @Override public void completed(String job, double successFactor, long duration) { - var context = metric.createContext(Map.of("job", job)); + var context = metric.createContext(Map.of("maintainer", job)); metric.set("maintenance.successFactorDeviation", successFactor, context); metric.set("maintenance.duration", duration, context); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index ce641b5d650..07a36cf00f2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; +import ai.vespa.net.InetAddressUtil; import com.google.common.net.InetAddresses; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; @@ -452,7 +453,7 @@ public record IP() { /** Convert IP address to string. This uses :: for zero compression in IPv6 addresses. */ public static String asString(InetAddress inetAddress) { - return InetAddresses.toAddrString(inetAddress); + return InetAddressUtil.toString(inetAddress); } /** Returns whether given string is an IPv4 address */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java index 01eb24ec2ac..a8384faf97d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeResourcesSerializer.java @@ -26,7 +26,7 @@ public class NodeResourcesSerializer { static void toSlime(NodeResources resources, Cursor resourcesObject) { if (resources.isUnspecified()) return; resourcesObject.setDouble(vcpuKey, resources.vcpu()); - resourcesObject.setDouble(memoryKey, resources.memoryGb()); + resourcesObject.setDouble(memoryKey, resources.memoryGiB()); resourcesObject.setDouble(diskKey, resources.diskGb()); resourcesObject.setDouble(bandwidthKey, resources.bandwidthGbps()); resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); @@ -35,7 +35,7 @@ public class NodeResourcesSerializer { if (!resources.gpuResources().isDefault()) { Cursor gpuObject = resourcesObject.setObject(gpuKey); gpuObject.setLong(gpuCountKey, resources.gpuResources().count()); - gpuObject.setDouble(gpuMemoryKey, resources.gpuResources().memoryGb()); + gpuObject.setDouble(gpuMemoryKey, resources.gpuResources().memoryGiB()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 2ebc2350542..9aaf01f6fc7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterSpec.Type; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ParentHostUnavailableException; @@ -20,6 +21,7 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -159,7 +161,7 @@ class Activator { .collect(Collectors.toUnmodifiableSet()); } - private static void validateParentHosts(ApplicationId application, NodeList allNodes, NodeList potentialChildren) { + static void validateParentHosts(ApplicationId application, NodeList allNodes, NodeList potentialChildren) { Set<String> parentHostnames = potentialChildren.stream() .map(Node::parentHostname) .flatMap(Optional::stream) @@ -169,13 +171,24 @@ class Activator { .matching(node -> parentHostnames.contains(node.hostname())) .hostnames(); + Set<String> applicationParentHostnames = new HashSet<>(parentHostnames); + applicationParentHostnames.removeIf(host -> potentialChildren.childrenOf(host).type(Type.combined, Type.container, Type.content).isEmpty()); + + Set<String> nonActiveApplicationHosts = new HashSet<>(nonActiveHosts); + nonActiveApplicationHosts.removeIf(host -> potentialChildren.childrenOf(host).type(Type.combined, Type.container, Type.content).isEmpty()); + if (nonActiveHosts.size() > 0) { - long numActive = parentHostnames.size() - nonActiveHosts.size(); + int numActiveApplication = applicationParentHostnames.size() - nonActiveApplicationHosts.size(); + int numActiveAdmin = parentHostnames.size() - nonActiveHosts.size() - numActiveApplication; var messageBuilder = new StringBuilder() - .append(numActive).append("/").append(parentHostnames.size()) - .append(" hosts for ") - .append(application) - .append(" have completed provisioning and bootstrapping, still waiting for "); + .append(numActiveApplication).append("/").append(applicationParentHostnames.size()) + .append(" application hosts"); + if (parentHostnames.size() > applicationParentHostnames.size()) + messageBuilder.append(" and ") + .append(numActiveAdmin).append("/").append(parentHostnames.size() - applicationParentHostnames.size()) + .append(" admin hosts"); + messageBuilder.append(" for ").append(application) + .append(" have completed provisioning and bootstrapping, still waiting for "); if (nonActiveHosts.size() <= 5) { messageBuilder.append(nonActiveHosts.stream() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index fa0421555d4..2f5ecd32e8d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -247,7 +247,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat var n = node.resources(); var h = node.parent.get().resources(); if (h.vcpu() < n.vcpu() * 2) return false; - if (h.memoryGb() < n.memoryGb() * 2) return false; + if (h.memoryGiB() < n.memoryGiB() * 2) return false; if (h.diskGb() < n.diskGb() * 2) return false; return true; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java index 11a5ee39a3d..d7a382165a5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java @@ -24,8 +24,8 @@ public class NodeResourceComparator { @Override public int compare(NodeResources a, NodeResources b) { - if (a.memoryGb() > b.memoryGb()) return 1; - if (a.memoryGb() < b.memoryGb()) return -1; + if (a.memoryGiB() > b.memoryGiB()) return 1; + if (a.memoryGiB() < b.memoryGiB()) return -1; if (a.diskGb() > b.diskGb()) return 1; if (a.diskGb() < b.diskGb()) return -1; if (a.vcpu() > b.vcpu()) return 1; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 5eb8c2e7fd7..fb3671a74ff 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -31,15 +31,15 @@ public class NodeResourceLimits { boolean exclusive = nodeRepository.exclusivity().allocation(cluster); if (! requested.vcpuIsUnspecified() && requested.vcpu() < minAdvertisedVcpu(cluster, exclusive)) illegal(type, "vcpu", "", cluster, requested.vcpu(), minAdvertisedVcpu(cluster, exclusive)); - if (! requested.memoryGbIsUnspecified() && requested.memoryGb() < minAdvertisedMemoryGb(cluster, exclusive)) - illegal(type, "memoryGb", "Gb", cluster, requested.memoryGb(), minAdvertisedMemoryGb(cluster, exclusive)); - if (! requested.diskGbIsUnspecified() && requested.diskGb() < minAdvertisedDiskGb(requested, exclusive)) + if (! requested.memoryIsUnspecified() && requested.memoryGiB() < minAdvertisedMemoryGb(cluster, exclusive)) + illegal(type, "memoryGb", "Gb", cluster, requested.memoryGiB(), minAdvertisedMemoryGb(cluster, exclusive)); + if (! requested.diskIsUnspecified() && requested.diskGb() < minAdvertisedDiskGb(requested, exclusive)) illegal(type, "diskGb", "Gb", cluster, requested.diskGb(), minAdvertisedDiskGb(requested, exclusive)); } // TODO: Remove this when we are ready to fail, not just warn on this. */ public boolean isWithinAdvertisedDiskLimits(NodeResources requested, ClusterSpec cluster) { - if (requested.diskGbIsUnspecified() || requested.memoryGbIsUnspecified()) return true; + if (requested.diskIsUnspecified() || requested.memoryIsUnspecified()) return true; return requested.diskGb() >= minAdvertisedDiskGb(requested, cluster); } @@ -55,7 +55,7 @@ public class NodeResourceLimits { if (realResources.isUnspecified()) return true; if (realResources.vcpu() < minRealVcpu(cluster)) return false; - if (realResources.memoryGb() < minRealMemoryGb(cluster)) return false; + if (realResources.memoryGiB() < minRealMemoryGb(cluster)) return false; if (realResources.diskGb() < minRealDiskGb()) return false; return true; } @@ -67,7 +67,7 @@ public class NodeResourceLimits { requested = requested.withDiskGb(Math.max(minAdvertisedDiskGb(requested, cluster), requested.diskGb())); return requested.withVcpu(Math.max(minAdvertisedVcpu(cluster, exclusive), requested.vcpu())) - .withMemoryGb(Math.max(minAdvertisedMemoryGb(cluster, exclusive), requested.memoryGb())) + .withMemoryGiB(Math.max(minAdvertisedMemoryGb(cluster, exclusive), requested.memoryGiB())) .withDiskGb(Math.max(minAdvertisedDiskGb(requested, exclusive), requested.diskGb())); } @@ -89,7 +89,7 @@ public class NodeResourceLimits { // TODO: Move this check into the above when we are ready to fail, not just warn on this. */ private static double minAdvertisedDiskGb(NodeResources requested, ClusterSpec cluster) { - return requested.memoryGb() * switch (cluster.type()) { + return requested.memoryGiB() * switch (cluster.type()) { case combined, content -> 3; case container -> 2; default -> 0; // No constraint on other types diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ThrottleProvisioningException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ThrottleProvisioningException.java new file mode 100644 index 00000000000..c41e5c94fee --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ThrottleProvisioningException.java @@ -0,0 +1,16 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.TransientException; + +/** + * Thrown by {@link HostProvisioner} to indicate that (de)provisioning of a host has + * failed due request being throttled by the cloud provider. + * + * @author freva + */ +public class ThrottleProvisioningException extends TransientException { + public ThrottleProvisioningException(String message) { + super(message); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index dc70af9a84f..7c31f8bfda0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -244,7 +244,7 @@ public class NodePatcher { case "diskGb": return node.with(node.flavor().with(node.flavor().resources().withDiskGb(value.asDouble())), Agent.operator, clock.instant()); case "memoryGb": - return node.with(node.flavor().with(node.flavor().resources().withMemoryGb(value.asDouble())), Agent.operator, clock.instant()); + return node.with(node.flavor().with(node.flavor().resources().withMemoryGiB(value.asDouble())), Agent.operator, clock.instant()); case "vcpu": return node.with(node.flavor().with(node.flavor().resources().withVcpu(value.asDouble())), Agent.operator, clock.instant()); case "fastDisk": diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java index 518474c3711..adbd83158a1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java @@ -12,7 +12,7 @@ public class NodeResourcesSerializer { static void toSlime(NodeResources resources, Cursor object) { object.setDouble("vcpu", resources.vcpu()); - object.setDouble("memoryGb", resources.memoryGb()); + object.setDouble("memoryGb", resources.memoryGiB()); object.setDouble("diskGb", resources.diskGb()); object.setDouble("bandwidthGbps", resources.bandwidthGbps()); object.setString("diskSpeed", toString(resources.diskSpeed())); @@ -20,7 +20,7 @@ public class NodeResourcesSerializer { object.setString("architecture", toString(resources.architecture())); if (!resources.gpuResources().isDefault()) { object.setLong("gpuCount", resources.gpuResources().count()); - object.setDouble("gpuMemoryGb", resources.gpuResources().memoryGb()); + object.setDouble("gpuMemoryGb", resources.gpuResources().memoryGiB()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index cef28045a8b..c1d8d250ce8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -332,7 +332,7 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { if (resourcesInspector.field("vcpu").valid()) flavor = flavor.with(flavor.resources().withVcpu(resourcesInspector.field("vcpu").asDouble())); if (resourcesInspector.field("memoryGb").valid()) - flavor = flavor.with(flavor.resources().withMemoryGb(resourcesInspector.field("memoryGb").asDouble())); + flavor = flavor.with(flavor.resources().withMemoryGiB(resourcesInspector.field("memoryGb").asDouble())); if (resourcesInspector.field("diskGb").valid()) flavor = flavor.with(flavor.resources().withDiskGb(resourcesInspector.field("diskGb").asDouble())); if (resourcesInspector.field("bandwidthGbps").valid()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 6a48323cea4..de9e241bbfd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -229,7 +229,7 @@ public class MockHostProvisioner implements HostProvisioner { } public boolean compatible(Flavor flavor, NodeResources resources) { - NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb); + NodeResources resourcesToVerify = resources.withMemoryGiB(resources.memoryGiB() - memoryTaxGb); if (flavor.resources().storageType() == NodeResources.StorageType.remote && flavor.resources().diskGb() >= resources.diskGb()) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index e0c8199a882..c4327098378 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -154,6 +154,23 @@ public class AutoscalingTest { } @Test + public void test_containers_wont_scale_up_on_memory() { + var min = new ClusterResources(2, 1, new NodeResources(4, 8, 50, 0.1)); + var now = new ClusterResources(4, 1, new NodeResources(4, 8, 50, 0.1)); + var max = new ClusterResources(8, 1, new NodeResources(4, 8, 50, 0.1)); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(false) + .clusterType(ClusterSpec.Type.container) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); + fixture.tester().setScalingDuration(fixture.applicationId(), fixture.clusterSpec.id(), Duration.ofMinutes(5)); + + fixture.loader().applyLoad(new Load(0.1875, 1.0, 0.95, 0, 0), 50); + assertEquals(Autoscaling.Status.insufficient, fixture.autoscale().status()); + } + + @Test public void initial_deployment_with_host_sharing_flag() { var min = new ClusterResources(7, 1, new NodeResources(2.0, 10.0, 384.0, 0.1)); var max = new ClusterResources(7, 1, new NodeResources(2.4, 32.0, 768.0, 0.1)); @@ -267,7 +284,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(duration.negated()); fixture.loader().zeroTraffic(20, 1); fixture.tester().assertResources("Scaled down", - 2, 1, 2, 16, 100, + 2, 1, 2, 8, 100, fixture.autoscale()); } @@ -398,7 +415,7 @@ public class AutoscalingTest { fixture.loader().applyLoad(new Load(0.25, 0.95, 0.95, 0, 0), 120); fixture.tester().assertResources("Scaling up", 5, 1, - defaultResources.vcpu(), defaultResources.memoryGb(), defaultResources.diskGb(), + defaultResources.vcpu(), defaultResources.memoryGiB(), defaultResources.diskGb(), fixture.autoscale()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java index f2507786a8b..47793385eb8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java @@ -52,7 +52,7 @@ public class AwsHostResourcesCalculatorImpl implements HostResourcesCalculator { double diskOverhead = consideredFlavors.stream() .mapToDouble(flavor -> resourcesCalculator.diskOverhead(flavor, advertisedResources, false, exclusive)) .reduce(bestCase ? Double::min : Double::max).orElse(0); - return advertisedResources.withMemoryGb(advertisedResources.memoryGb() - memoryOverhead) + return advertisedResources.withMemoryGiB(advertisedResources.memoryGiB() - memoryOverhead) .withDiskGb(advertisedResources.diskGb() - diskOverhead); } @@ -63,7 +63,7 @@ public class AwsHostResourcesCalculatorImpl implements HostResourcesCalculator { for (VespaFlavor flavor : consideredFlavorsGivenReal(realResources)) { double memoryOverhead = resourcesCalculator.memoryOverhead(flavor, realResources, true); double diskOverhead = resourcesCalculator.diskOverhead(flavor, realResources, true, exclusive); - NodeResources advertised = realResources.withMemoryGb(realResources.memoryGb() + memoryOverhead) + NodeResources advertised = realResources.withMemoryGiB(realResources.memoryGiB() + memoryOverhead) .withDiskGb(realResources.diskGb() + diskOverhead); if ( ! flavor.advertisedResources().satisfies(advertised)) continue; if (bestCase ? memoryOverhead < chosenMemoryOverhead : memoryOverhead > chosenDiskOverhead) @@ -71,7 +71,7 @@ public class AwsHostResourcesCalculatorImpl implements HostResourcesCalculator { if (bestCase ? diskOverhead < chosenDiskOverhead : diskOverhead > chosenDiskOverhead) chosenDiskOverhead = diskOverhead; } - return realResources.withMemoryGb(realResources.memoryGb() + chosenMemoryOverhead) + return realResources.withMemoryGiB(realResources.memoryGiB() + chosenMemoryOverhead) .withDiskGb(realResources.diskGb() + chosenDiskOverhead); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java index 0e62faef2c1..ec62a10b293 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.autoscale.awsnodes; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.Zone; /** * Calculations and logic on node resources common to provision-service and host-admin (at least). @@ -28,7 +27,7 @@ public class AwsResourcesCalculator { public NodeResources realResourcesOfChildContainer(NodeResources resources, VespaFlavor hostFlavor) { // This must match realResourcesOfChildSaturatingHost() if exclusive is true, and vice versa boolean exclusive = saturates(hostFlavor, resources); - return resources.withMemoryGb(resources.memoryGb() - memoryOverhead(hostFlavor, resources, false)) + return resources.withMemoryGiB(resources.memoryGiB() - memoryOverhead(hostFlavor, resources, false)) .withDiskGb(resources.diskGb() - diskOverhead(hostFlavor, resources, false, exclusive)); } @@ -39,13 +38,13 @@ public class AwsResourcesCalculator { */ public double memoryOverhead(VespaFlavor hostFlavor, NodeResources resources, boolean real) { double hostMemoryOverhead = - hostFlavor.advertisedResources().memoryGb() - hostFlavor.realResources().memoryGb() + hostFlavor.advertisedResources().memoryGiB() - hostFlavor.realResources().memoryGiB() + hostMemory; // Approximate cost of host administration processes - if (hostMemoryOverhead > hostFlavor.advertisedResources().memoryGb()) // An unusably small flavor, - return resources.memoryGb(); // all will be overhead - double memoryShare = resources.memoryGb() / - ( hostFlavor.advertisedResources().memoryGb() - ( real ? hostMemoryOverhead : 0)); + if (hostMemoryOverhead > hostFlavor.advertisedResources().memoryGiB()) // An unusably small flavor, + return resources.memoryGiB(); // all will be overhead + double memoryShare = resources.memoryGiB() / + ( hostFlavor.advertisedResources().memoryGiB() - ( real ? hostMemoryOverhead : 0)); if (memoryShare > 1) // The real resources of the host cannot fit the requested real resources after overhead memoryShare = 1; @@ -69,7 +68,7 @@ public class AwsResourcesCalculator { private boolean saturates(VespaFlavor hostFlavor, NodeResources nodeResources) { NodeResources hostResources = hostFlavor.advertisedResources(); return equal(hostResources.vcpu(), nodeResources.vcpu()) || - equal(hostResources.memoryGb(), nodeResources.memoryGb()) || + equal(hostResources.memoryGiB(), nodeResources.memoryGiB()) || equal(hostResources.diskGb(), nodeResources.diskGb()) || equal(hostResources.bandwidthGbps(), nodeResources.bandwidthGbps()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index 0da8750ee1b..9090716cf57 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -1,8 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.config.provision.Exclusivity; -import com.yahoo.config.provision.SharedHosts; import com.yahoo.json.Jackson; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -105,7 +103,7 @@ public class CapacityCheckerTester { child.type = NodeType.tenant; NodeResources cnr = childResources.get(j % childResources.size()); child.minCpuCores = cnr.vcpu(); - child.minMainMemoryAvailableGb = cnr.memoryGb(); + child.minMainMemoryAvailableGb = cnr.memoryGiB(); child.minDiskAvailableGb = cnr.diskGb(); child.fastDisk = true; child.ipAddresses = List.of(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index 874db8c961d..c91564244de 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -24,7 +24,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.net.HostName; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.custom.ClusterCapacity; @@ -285,11 +284,11 @@ public class HostCapacityMaintainerTest { tester = new DynamicProvisioningTester(Cloud.builder().name(CloudName.AWS).dynamicProvisioning(true).allowHostSharing(false).build(), new MockNameResolver()); NodeResources resources1 = new NodeResources(24, 64, 100, 10); setPreprovisionCapacityFlag(tester, - new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), + new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGiB(), resources1.diskGb(), resources1.bandwidthGbps(), resources1.diskSpeed().name(), resources1.storageType().name(), resources1.architecture().name(), "container"), - new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), + new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGiB(), resources1.diskGb(), resources1.bandwidthGbps(), resources1.diskSpeed().name(), resources1.storageType().name(), resources1.architecture().name(), null)); @@ -325,7 +324,7 @@ public class HostCapacityMaintainerTest { tester = new DynamicProvisioningTester(Cloud.builder().name(CloudName.AWS).dynamicProvisioning(true).allowHostSharing(false).build(), new MockNameResolver()); NodeResources resources1 = new NodeResources(24, 64, 100, 10); setPreprovisionCapacityFlag(tester, - new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), + new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGiB(), resources1.diskGb(), resources1.bandwidthGbps(), resources1.diskSpeed().name(), resources1.storageType().name(), resources1.architecture().name(), null)); @@ -362,7 +361,7 @@ public class HostCapacityMaintainerTest { tester = new DynamicProvisioningTester(); NodeResources resources1 = new NodeResources(24, 64, 100, 10); setPreprovisionCapacityFlag(tester, - new ClusterCapacity(2, resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), + new ClusterCapacity(2, resources1.vcpu(), resources1.memoryGiB(), resources1.diskGb(), resources1.bandwidthGbps(), resources1.diskSpeed().name(), resources1.storageType().name(), resources1.architecture().name(), null)); @@ -420,7 +419,7 @@ public class HostCapacityMaintainerTest { setPreprovisionCapacityFlag(tester, new ClusterCapacity(3, resources1.vcpu() - applicationNodeResources.vcpu(), - resources1.memoryGb() - applicationNodeResources.memoryGb(), + resources1.memoryGiB() - applicationNodeResources.memoryGiB(), resources1.diskGb() - applicationNodeResources.diskGb(), resources1.bandwidthGbps() - applicationNodeResources.bandwidthGbps(), resources1.diskSpeed().name(), @@ -433,7 +432,7 @@ public class HostCapacityMaintainerTest { setPreprovisionCapacityFlag(tester, new ClusterCapacity(3, resources1.vcpu() - applicationNodeResources.vcpu() + 1, - resources1.memoryGb() - applicationNodeResources.memoryGb() + 1, + resources1.memoryGiB() - applicationNodeResources.memoryGiB() + 1, resources1.diskGb() - applicationNodeResources.diskGb() + 1, resources1.bandwidthGbps(), resources1.diskSpeed().name(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index c53a4d88c01..2fe458e139a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -73,13 +73,13 @@ public class ScalingSuggestionsMaintainerTest { new TestMetric()); maintainer.maintain(); - assertEquals("8 nodes with [vcpu: 3.3, memory: 4.5 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", + assertEquals("8 nodes with [vcpu: 3.3, memory: 4.0 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); assertEquals("7 nodes with [vcpu: 4.4, memory: 5.3 Gb, disk: 16.5 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app2, cluster2, tester).resources().get().toString()); // Secondary suggestions - assertEquals("9 nodes with [vcpu: 2.9, memory: 4.5 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", + assertEquals("9 nodes with [vcpu: 2.9, memory: 4.0 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionsOf(app1, cluster1, tester).get(1).resources().get().toString()); assertEquals("8 nodes with [vcpu: 3.8, memory: 4.7 Gb, disk: 14.2 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionsOf(app2, cluster2, tester).get(1).resources().get().toString()); @@ -89,7 +89,7 @@ public class ScalingSuggestionsMaintainerTest { addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Suggestion stays at the peak value observed", - "8 nodes with [vcpu: 3.3, memory: 4.5 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", + "8 nodes with [vcpu: 3.3, memory: 4.0 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); // Utilization is still way down and a week has passed tester.clock().advance(Duration.ofDays(7)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java index 6f58ecd2de3..240cf6bb08b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java @@ -101,13 +101,13 @@ public class AllocationVisualizer extends JPanel { if (isHost) { g.setColor(Color.GRAY); - for (int i = 0; i < node.resources().memoryGb(); i++) { + for (int i = 0; i < node.resources().memoryGiB(); i++) { g.fillRect(x, y - nodeHeight, nodeWidth, nodeHeight); y = y - (nodeHeight + 2); } } else { g.setColor(Color.YELLOW); - int multi = (int) node.resources().memoryGb(); + int multi = (int) node.resources().memoryGiB(); int height = multi * nodeHeight + ((multi - 1) * 2); g.fillRect(x, y - height, nodeWidth, height); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index a91902c8eba..ac141a81d58 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -428,7 +428,7 @@ public class DynamicAllocationTest { ApplicationId application = ProvisioningTester.applicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("1").build(); - List<HostSpec> hosts = tester.prepare(application, cluster, 36, 2, resources); + List<HostSpec> hosts = tester.prepare(application, cluster, 66, 2, resources); tester.activate(application, hosts); assertEquals(3, hosts.size()); assertEquals(1, hosts.stream().map(host -> host.membership().get().cluster().group().get()).distinct().count()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 76fc4078927..cb40a135af9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -11,7 +10,6 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.transaction.Mutex; @@ -185,7 +183,7 @@ public class DynamicProvisioningTester { NodeResources expectedResources, ClusterResources resources) { assertResources(message, nodeCount, groupCount, - expectedResources.vcpu(), expectedResources.memoryGb(), expectedResources.diskGb(), + expectedResources.vcpu(), expectedResources.memoryGiB(), expectedResources.diskGb(), resources); } @@ -196,7 +194,7 @@ public class DynamicProvisioningTester { assertTrue("Resources are present: " + message + " (" + autoscaling + ": " + autoscaling.status() + ")", autoscaling.resources().isPresent()); assertResources(message, nodeCount, groupCount, - expectedResources.vcpu(), expectedResources.memoryGb(), expectedResources.diskGb(), + expectedResources.vcpu(), expectedResources.memoryGiB(), expectedResources.diskGb(), autoscaling.resources().get()); } @@ -228,7 +226,7 @@ public class DynamicProvisioningTester { assertEquals("Node count in " + resources + ": " + message, nodeCount, resources.nodes()); assertEquals("Group count in " + resources+ ": " + message, groupCount, resources.groups()); assertEquals("Cpu in " + resources + ": " + message, approxCpu, Math.round(nodeResources.vcpu() * 10) / 10.0, delta); - assertEquals("Memory in " + resources + ": " + message, approxMemory, Math.round(nodeResources.memoryGb() * 10) / 10.0, delta); + assertEquals("Memory in " + resources + ": " + message, approxMemory, Math.round(nodeResources.memoryGiB() * 10) / 10.0, delta); assertEquals("Disk in: " + resources + ": " + message, approxDisk, Math.round(nodeResources.diskGb() * 10) / 10.0, delta); } @@ -254,7 +252,7 @@ public class DynamicProvisioningTester { @Override public NodeResources realResourcesOf(Nodelike node, NodeRepository nodeRepository) { if (zone.cloud().dynamicProvisioning()) - return node.resources().withMemoryGb(node.resources().memoryGb()); + return node.resources().withMemoryGiB(node.resources().memoryGiB()); else return node.resources(); } @@ -262,19 +260,19 @@ public class DynamicProvisioningTester { @Override public NodeResources advertisedResourcesOf(Flavor flavor) { if (zone.cloud().dynamicProvisioning()) - return flavor.resources().withMemoryGb(flavor.resources().memoryGb()); + return flavor.resources().withMemoryGiB(flavor.resources().memoryGiB()); else return flavor.resources(); } @Override public NodeResources requestToReal(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { - return resources.withMemoryGb(resources.memoryGb()); + return resources.withMemoryGiB(resources.memoryGiB()); } @Override public NodeResources realToRequest(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { - return resources.withMemoryGb(resources.memoryGb()); + return resources.withMemoryGiB(resources.memoryGiB()); } @Override diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 9e6ae9f010c..0f8292bb310 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -638,7 +638,7 @@ public class ProvisioningTest { ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(14, defaultResources).activateTenantHosts(); SystemState state = prepare(application, 1, 1, 1, 64, defaultResources, tester); // becomes 1, 1, 1, 1, 6 - assertEquals(9, state.allHosts.size()); + assertEquals(6, state.allHosts.size()); tester.activate(application, state.allHosts); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 7c5859ad686..81ec7861cbb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -8,14 +8,12 @@ import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; @@ -26,7 +24,6 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -332,8 +329,8 @@ public class ProvisioningTester { expected.justNonNumbers().compatibleWith(node.resources().justNonNumbers())); assertEquals(explanation + ": Vcpu: Expected " + expected.vcpu() + " but was " + node.resources().vcpu(), expected.vcpu(), node.resources().vcpu(), 0.05); - assertEquals(explanation + ": Memory: Expected " + expected.memoryGb() + " but was " + node.resources().memoryGb(), - expected.memoryGb(), node.resources().memoryGb(), 0.05); + assertEquals(explanation + ": Memory: Expected " + expected.memoryGiB() + " but was " + node.resources().memoryGiB(), + expected.memoryGiB(), node.resources().memoryGiB(), 0.05); assertEquals(explanation + ": Disk: Expected " + expected.diskGb() + " but was " + node.resources().diskGb(), expected.diskGb(), node.resources().diskGb(), 0.05); } @@ -772,14 +769,14 @@ public class ProvisioningTester { FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); flavor.name(flavorName); flavor.minCpuCores(resources.vcpu()); - flavor.minMainMemoryAvailableGb(resources.memoryGb()); + flavor.minMainMemoryAvailableGb(resources.memoryGiB()); flavor.minDiskAvailableGb(resources.diskGb()); flavor.bandwidth(resources.bandwidthGbps() * 1000); flavor.fastDisk(resources.diskSpeed().compatibleWith(com.yahoo.config.provision.NodeResources.DiskSpeed.fast)); flavor.remoteStorage(resources.storageType().compatibleWith(com.yahoo.config.provision.NodeResources.StorageType.remote)); flavor.architecture(resources.architecture().toString()); flavor.gpuCount(resources.gpuResources().count()); - flavor.gpuMemoryGb(resources.gpuResources().memoryGb()); + flavor.gpuMemoryGb(resources.gpuResources().memoryGiB()); return flavor; } @@ -799,7 +796,7 @@ public class ProvisioningTester { public NodeResources realResourcesOf(Nodelike node, NodeRepository nodeRepository) { NodeResources resources = node.resources(); if (node.type() == NodeType.host) return resources; - return resources.withMemoryGb(resources.memoryGb() - memoryTaxGb) + return resources.withMemoryGiB(resources.memoryGiB() - memoryTaxGb) .withDiskGb(resources.diskGb() - ( resources.storageType() == local ? localDiskTax : 0)); } @@ -807,18 +804,18 @@ public class ProvisioningTester { public NodeResources advertisedResourcesOf(Flavor flavor) { NodeResources resources = flavor.resources(); if ( ! flavor.isConfigured()) return resources; - return resources.withMemoryGb(resources.memoryGb() + memoryTaxGb); + return resources.withMemoryGiB(resources.memoryGiB() + memoryTaxGb); } @Override public NodeResources requestToReal(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { - return resources.withMemoryGb(resources.memoryGb() - memoryTaxGb) + return resources.withMemoryGiB(resources.memoryGiB() - memoryTaxGb) .withDiskGb(resources.diskGb() - ( resources.storageType() == local ? localDiskTax : 0) ); } @Override public NodeResources realToRequest(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { - return resources.withMemoryGb(resources.memoryGb() + memoryTaxGb) + return resources.withMemoryGiB(resources.memoryGiB() + memoryTaxGb) .withDiskGb(resources.diskGb() + ( resources.storageType() == local ? localDiskTax : 0) ); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java index 1bf42d72180..9310511e687 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java @@ -87,8 +87,8 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { } NodeResources realResourcesOf(NodeResources advertisedResources) { - return advertisedResources.withMemoryGb(advertisedResources.memoryGb() - - memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGb(), advertisedResources, false)) + return advertisedResources.withMemoryGiB(advertisedResources.memoryGiB() - + memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGiB(), advertisedResources, false)) .withDiskGb(advertisedResources.diskGb() - diskOverhead(advertisedResourcesOf(hostFlavor).diskGb(), advertisedResources, false)); } @@ -96,25 +96,25 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { @Override public NodeResources requestToReal(NodeResources advertisedResources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { - double memoryOverhead = memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGb(), advertisedResources, false); + double memoryOverhead = memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGiB(), advertisedResources, false); double diskOverhead = diskOverhead(advertisedResourcesOf(hostFlavor).diskGb(), advertisedResources, false); - return advertisedResources.withMemoryGb(advertisedResources.memoryGb() - memoryOverhead) + return advertisedResources.withMemoryGiB(advertisedResources.memoryGiB() - memoryOverhead) .withDiskGb(advertisedResources.diskGb() - diskOverhead); } @Override public NodeResources advertisedResourcesOf(Flavor flavor) { if ( ! flavor.equals(hostFlavor)) return flavor.resources(); // Node 'flavors' just wrap the advertised resources - return hostFlavor.resources().withMemoryGb(hostFlavor.resources().memoryGb() + memoryOverhead) + return hostFlavor.resources().withMemoryGiB(hostFlavor.resources().memoryGiB() + memoryOverhead) .withDiskGb(hostFlavor.resources().diskGb() + diskOverhead); } @Override public NodeResources realToRequest(NodeResources realResources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { - double memoryOverhead = memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGb(), realResources, true); + double memoryOverhead = memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGiB(), realResources, true); double diskOverhead = diskOverhead(advertisedResourcesOf(hostFlavor).diskGb(), realResources, true); - return realResources.withMemoryGb(realResources.memoryGb() + memoryOverhead) + return realResources.withMemoryGiB(realResources.memoryGiB() + memoryOverhead) .withDiskGb(realResources.diskGb() + diskOverhead); } @@ -127,7 +127,7 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { * @param real true if the given resources are in real values, false if they are in advertised */ private double memoryOverhead(double hostAdvertisedMemoryGb, NodeResources resources, boolean real) { - double memoryShare = resources.memoryGb() / + double memoryShare = resources.memoryGiB() / ( hostAdvertisedMemoryGb - (real ? memoryOverhead : 0)); return memoryOverhead * memoryShare; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index 83ff33b06a0..ddaf5679cf3 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -84,33 +84,32 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { // Non-private for testing purposes SuspensionLimit getConcurrentSuspensionLimit(ClusterApi clusterApi) { - // Possible service clusters on a node as of 2021-01-22: + // Possible service clusters on a node as of 2024-06-09: // // CLUSTER ID SERVICE TYPE HEALTH ASSOCIATION // 1 CCN-controllers container-clustercontrollers Slobrok 1, 3, or 6 in content cluster // 2 CCN distributor Slobrok content cluster // 3 CCN storagenode Slobrok content cluster // 4 CCN searchnode Slobrok content cluster - // 5 CCN transactionlogserver not checked content cluster - // 6 JCCN container Slobrok jdisc container cluster - // 7 admin slobrok not checked 1-3 in jdisc container cluster - // 8 metrics metricsproxy-container Slobrok application - // 9 admin logd not checked application - // 10 admin config-sentinel not checked application - // 11 admin configproxy not checked application - // 12 admin logforwarder not checked application - // 13 controller controller state/v1 controllers - // 14 zone-config-servers configserver state/v1 config servers - // 15 controller-host hostadmin state/v1 controller hosts - // 16 configserver-host hostadmin state/v1 config server hosts - // 17 tenant-host hostadmin state/v1 tenant hosts - // 18 proxy-host hostadmin state/v1 proxy hosts + // 5 JCCN container Slobrok jdisc container cluster + // 6 admin slobrok not checked 1-3 in jdisc container cluster + // 7 metrics metricsproxy-container Slobrok application + // 8 admin logd not checked application + // 9 admin config-sentinel not checked application + // 10 admin configproxy not checked application + // 11 admin logforwarder not checked application + // 12 controller controller state/v1 controllers + // 13 zone-config-servers configserver state/v1 config servers + // 14 controller-host hostadmin state/v1 controller hosts + // 15 configserver-host hostadmin state/v1 config server hosts + // 16 tenant-host hostadmin state/v1 tenant hosts + // 17 proxy-host hostadmin state/v1 proxy hosts // // CCN refers to the content cluster's name, as specified in services.xml. // JCCN refers to the jdisc container cluster's name, as specified in services.xml. // - // For instance a content node will have 2-5 and 8-12 and possibly 1, while a combined - // cluster node may have all 1-12. + // For instance a content node will have 2-4 and 7-11 and possibly 1, while a combined + // cluster node may have all 1-11. // // The services on a node can be categorized into these main types, ref association column above: // A content @@ -132,7 +131,7 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { return SuspensionLimit.fromAllowedDown(1); } - if (Set.of(ServiceType.STORAGE, ServiceType.SEARCH, ServiceType.DISTRIBUTOR, ServiceType.TRANSACTION_LOG_SERVER) + if (Set.of(ServiceType.STORAGE, ServiceType.SEARCH, ServiceType.DISTRIBUTOR) .contains(clusterApi.serviceType())) { // Delegate to the cluster controller return SuspensionLimit.fromAllowedDownRatio(1); diff --git a/parent/pom.xml b/parent/pom.xml index 6049ac9f4a4..8bded0b0430 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -64,6 +64,13 @@ <groupId>org.apache.felix</groupId> <artifactId>maven-bundle-plugin</artifactId> <version>${maven-bundle-plugin.vespa.version}</version> + <configuration> + <obrRepository>NONE</obrRepository> + <!-- Hide warnings for multi-release jars, e.g. bouncycastle --> + <instructions> + <_fixupmessages>"Classes found in the wrong directory"</_fixupmessages> + </instructions> + </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -237,7 +244,7 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> - <version>3.2.0</version> + <version>3.3.0</version> </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> @@ -317,7 +324,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.31.0</version> + <version>5.34.1</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> @@ -327,7 +334,7 @@ <dependency> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-testing-frameworks</artifactId> - <version>2.9.0</version> + <version>2.12.2</version> </dependency> </dependencies> </plugin> @@ -717,6 +724,21 @@ </dependency> <dependency> <groupId>io.netty</groupId> + <artifactId>netty-buffer</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-codec</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-common</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> <artifactId>netty-handler</artifactId> <version>${netty.vespa.version}</version> </dependency> @@ -727,6 +749,16 @@ </dependency> <dependency> <groupId>io.netty</groupId> + <artifactId>netty-transport</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-transport-classes-epoll</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> <artifactId>netty-transport-native-epoll</artifactId> <version>${netty.vespa.version}</version> <classifier>linux-x86_64</classifier> @@ -738,6 +770,11 @@ </dependency> <dependency> <groupId>io.netty</groupId> + <artifactId>netty-transport-native-unix-common</artifactId> + <version>${netty.vespa.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> <artifactId>netty-tcnative</artifactId> <version>${netty-tcnative.vespa.version}</version> </dependency> @@ -915,7 +952,12 @@ <dependency> <groupId>org.apache.maven.shared</groupId> <artifactId>maven-dependency-tree</artifactId> - <version>3.2.1</version> + <version>${maven-dependency-tree.vespa.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven.shared</groupId> + <artifactId>maven-shared-utils</artifactId> + <version>${maven-shared-utils.vespa.version}</version> </dependency> <dependency> <groupId>org.apache.maven.surefire</groupId> @@ -1005,6 +1047,11 @@ </dependency> <dependency> <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-classworlds</artifactId> + <version>${plexus-classworlds.vespa.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.plexus</groupId> <artifactId>plexus-interpolation</artifactId> <version>${plexus-interpolation.vespa.version}</version> </dependency> @@ -1199,7 +1246,7 @@ See pluginManagement of rewrite-maven-plugin for more details --> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-recipe-bom</artifactId> - <version>2.11.0</version> + <version>2.13.2</version> <type>pom</type> <scope>import</scope> </dependency> diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index b801b16f60a..db34bc6d02b 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_define_module( DEPENDS vespalog vespalib - document - vdslib + vespa_document + vespa_vdslib LIBS src/vespa/persistence diff --git a/persistence/src/tests/dummyimpl/CMakeLists.txt b/persistence/src/tests/dummyimpl/CMakeLists.txt index 3391766ca10..34e44762327 100644 --- a/persistence/src/tests/dummyimpl/CMakeLists.txt +++ b/persistence/src/tests/dummyimpl/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(persistence_dummyimpl_conformance_test_app TEST dummyimpltest.cpp DEPENDS persistence_persistence_conformancetest - persistence + vespa_persistence ) vespa_add_test( @@ -16,6 +16,6 @@ vespa_add_executable(persistence_dummypersistence_test_app TEST SOURCES dummypersistence_test.cpp DEPENDS - persistence + vespa_persistence ) vespa_add_test(NAME persistence_dummypersistence_test_app COMMAND persistence_dummypersistence_test_app) diff --git a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp index fa0669a994a..835ac26d5c1 100644 --- a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp +++ b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp @@ -2,7 +2,7 @@ // Unit tests for dummypersistence. #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/documentid.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/vdslib/distribution/distribution.h> diff --git a/persistence/src/tests/spi/CMakeLists.txt b/persistence/src/tests/spi/CMakeLists.txt index 6f6981f2cce..fdb8e9d0b1b 100644 --- a/persistence/src/tests/spi/CMakeLists.txt +++ b/persistence/src/tests/spi/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_library(persistence_testspi SOURCES clusterstatetest.cpp DEPENDS - persistence + vespa_persistence ) diff --git a/persistence/src/vespa/persistence/CMakeLists.txt b/persistence/src/vespa/persistence/CMakeLists.txt index d95ee466344..fc71fb6b20f 100644 --- a/persistence/src/vespa/persistence/CMakeLists.txt +++ b/persistence/src/vespa/persistence/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(persistence +vespa_add_library(vespa_persistence SOURCES $<TARGET_OBJECTS:persistence_dummyimpl> $<TARGET_OBJECTS:persistence_spi> @@ -10,6 +10,6 @@ vespa_add_library(persistence_persistence_conformancetest SOURCES $<TARGET_OBJECTS:persistence_conformancetest_lib> DEPENDS - persistence + vespa_persistence GTest::GTest ) diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp index 7d106e93f57..ca4c172d2e0 100644 --- a/persistence/src/vespa/persistence/spi/clusterstate.cpp +++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp @@ -17,7 +17,7 @@ ClusterState::ClusterState(const lib::ClusterState& state, const lib::Distribution& distribution, bool maintenanceInAllSpaces) : _state(std::make_unique<lib::ClusterState>(state)), - _distribution(std::make_unique<lib::Distribution>(distribution.serialize())), + _distribution(std::make_unique<lib::Distribution>(distribution.serialized())), _nodeIndex(nodeIndex), _maintenanceInAllSpaces(maintenanceInAllSpaces) { @@ -99,7 +99,7 @@ void ClusterState::serialize(vespalib::nbostream& o) const { vespalib::asciistream tmp; _state->serialize(tmp); o << tmp.str() << _nodeIndex; - o << _distribution->serialize(); + o << _distribution->serialized(); } } diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/HitsVerificationBenchmark.java b/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/HitsVerificationBenchmark.java index 37683b1055e..dc00b6d4a14 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/HitsVerificationBenchmark.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/HitsVerificationBenchmark.java @@ -63,7 +63,7 @@ public class HitsVerificationBenchmark { writeOutputToStandardOut(output); } - private static PredicateIndex getIndex(BenchmarkArguments args, Config config, Map<String, Object> output) throws IOException { + static PredicateIndex getIndex(BenchmarkArguments args, Config config, Map<String, Object> output) throws IOException { if (args.feedFile != null) { PredicateIndexBuilder builder = new PredicateIndexBuilder(config); AtomicInteger idCounter = new AtomicInteger(); @@ -170,13 +170,13 @@ public class HitsVerificationBenchmark { "Query format. Valid formats are either 'vespa' (obsolete query property format) or 'json'.") public Format format = Format.VESPA; - @Option(name = {"-ff", "--feed-file"}, description = "File path to feed file (Vespa XML feed)") + @Option(name = {"-ff", "--feed-file"}, description = "File path to feed file (Vespa Json feed)") public String feedFile; @Option(name = {"-if", "--index-file"}, description = "File path to index file (Serialized index)") public String indexFile; - @Option(name = {"-quf", "--query-file"}, description = "File path to a query file") + @Option(name = {"-quf", "--query-file"}, description = "File path to query file") public String queryFile; @Arguments(title = "Output file", description = "File path to output file") diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/TargetingQueryFileConverter.java b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/TargetingQueryFileConverter.java index fefa35a0e1c..57635f2c3b2 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/TargetingQueryFileConverter.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/TargetingQueryFileConverter.java @@ -263,14 +263,11 @@ public class TargetingQueryFileConverter { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof Feature)) return false; - - Feature feature = (Feature) o; + if (!(o instanceof Feature feature)) return false; if (longValue != feature.longValue) return false; if (!key.equals(feature.key)) return false; return !(strValue != null ? !strValue.equals(feature.strValue) : feature.strValue != null); - } @Override diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedParser.java b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedParser.java index 0e15aff5869..bb0695f2285 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedParser.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedParser.java @@ -9,9 +9,10 @@ import java.io.IOException; import java.util.function.Consumer; /** - * Parses a feed file containing documents in XML format. Its implementation is based on the following assumptions: + * Parses a feed file containing documents in JSON format. Its implementation is based on the following assumptions: * 1. Each document has single predicate field. * 2. The predicate is stored in a field named "boolean". + * 3. There is just one "boolean" field on each line * * @author bjorncs */ @@ -20,26 +21,30 @@ public class VespaFeedParser { public static int parseDocuments(String feedFile, int maxDocuments, Consumer<Predicate> consumer) throws IOException { int documentCount = 0; try (BufferedReader reader = new BufferedReader(new FileReader(feedFile), 8 * 1024)) { - reader.readLine(); - reader.readLine(); // Skip to start of first document String line = reader.readLine(); - while (!line.startsWith("</vespafeed>") && documentCount < maxDocuments) { - while (!line.startsWith("<boolean>")) { + while (!line.startsWith("]") && documentCount < maxDocuments) { + while (!line.contains("\"boolean\":")) { line = reader.readLine(); } - Predicate predicate = Predicate.fromString(extractBooleanExpression(line)); - consumer.accept(predicate); - ++documentCount; - while (!line.startsWith("<document") && !line.startsWith("</vespafeed>")) { - line = reader.readLine(); + String booleanExpression = extractBooleanExpression(line); + try { + var predicate = Predicate.fromString(booleanExpression); + consumer.accept(predicate); + ++documentCount; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Failed to parse predicate: " + booleanExpression, e); } + line = reader.readLine(); } } return documentCount; } private static String extractBooleanExpression(String line) { - return line.substring(9, line.length() - 10); + String field = "\"boolean\":"; + var start = line.indexOf(field); + var end = line.indexOf("\"", start + field.length() + 1); + return line.substring(start + field.length() +1 , end); } } diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java deleted file mode 100644 index 7d4a6867405..00000000000 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.predicate.utils; - -import com.google.common.html.HtmlEscapers; -import com.yahoo.document.predicate.Predicate; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.Writer; - -/** - * @author Magnar Nedland - */ -public class VespaFeedWriter extends BufferedWriter { - - private String namespace; - private String documentType; - - VespaFeedWriter(Writer writer, String namespace, String documentType) throws IOException { - super(writer); - this.namespace = namespace; - this.documentType = documentType; - - this.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n"); - this.append("<vespafeed>\n"); - } - - @Override - public void close() throws IOException { - this.append("</vespafeed>\n"); - super.close(); - } - - public void writePredicateDocument(int id, String fieldName, Predicate predicate) { - try { - this.append(String.format("<document documenttype=\"%2$s\" documentid=\"id:%1$s:%2$s::%3$d\">\n", - namespace, documentType, id)); - this.append("<" + fieldName + ">" + HtmlEscapers.htmlEscaper().escape(predicate.toString()) + "</" + fieldName + ">\n"); - this.append("</document>\n"); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/benchmarks/HitsVerificationBenchmarkTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/benchmarks/HitsVerificationBenchmarkTest.java new file mode 100644 index 00000000000..340723a92b7 --- /dev/null +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/benchmarks/HitsVerificationBenchmarkTest.java @@ -0,0 +1,44 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.predicate.benchmarks; + +import com.google.common.primitives.Ints; +import com.yahoo.search.predicate.Config; +import com.yahoo.search.predicate.PredicateIndex; +import com.yahoo.search.predicate.index.BoundsPostingList; +import com.yahoo.search.predicate.index.IntervalWithBounds; +import com.yahoo.search.predicate.index.PredicateIntervalStore; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class HitsVerificationBenchmarkTest { + + @Test + void testTargeting() throws IOException { + HitsVerificationBenchmark.BenchmarkArguments args = new HitsVerificationBenchmark.BenchmarkArguments(); + args.feedFile = "src/test/resources/targeting.json"; + Config config = new Config.Builder().build(); + Map<String, Object> output = new HashMap<>(); + HitsVerificationBenchmark.getIndex(args, config, output); + assertEquals(46, output.get("Interval index entries")); + } + + @Test + void testFeed() throws IOException { + HitsVerificationBenchmark.BenchmarkArguments args = new HitsVerificationBenchmark.BenchmarkArguments(); + args.feedFile = "src/test/resources/vespa-feed.json"; + Config config = new Config.Builder().build(); + Map<String, Object> output = new HashMap<>(); + HitsVerificationBenchmark.getIndex(args, config, output); + assertEquals(206, output.get("Interval index entries")); + } + +} diff --git a/predicate-search/src/test/resources/targeting.json b/predicate-search/src/test/resources/targeting.json new file mode 100644 index 00000000000..f2f820e8e8d --- /dev/null +++ b/predicate-search/src/test/resources/targeting.json @@ -0,0 +1,46 @@ +[ + {"id":"id:rise:targeting::1","fields":{"boolean":"subseg in [1087]"}}, + {"id":"id:rise:targeting::2","fields":{"boolean":"subseg in [15045]"}}, + {"id":"id:rise:targeting::3","fields":{"boolean":"subseg in [15062]"}}, + {"id":"id:rise:targeting::4","fields":{"boolean":"subseg in [15068]"}}, + {"id":"id:rise:targeting::5","fields":{"boolean":"subseg in [15087]"}}, + {"id":"id:rise:targeting::6","fields":{"boolean":"subseg in [15090]"}}, + {"id":"id:rise:targeting::7","fields":{"boolean":"subseg in [15052]"}}, + {"id":"id:rise:targeting::8","fields":{"boolean":"subseg in [15053]"}}, + {"id":"id:rise:targeting::9","fields":{"boolean":"subseg in [15091]"}}, + {"id":"id:rise:targeting::10","fields":{"boolean":"subseg in [15092]"}}, + {"id":"id:rise:targeting::11","fields":{"boolean":"subseg in [15093]"}}, + {"id":"id:rise:targeting::12","fields":{"boolean":"subseg in [15104]"}}, + {"id":"id:rise:targeting::13","fields":{"boolean":"subseg in [15105]"}}, + {"id":"id:rise:targeting::14","fields":{"boolean":"subseg in [15038]"}}, + {"id":"id:rise:targeting::15","fields":{"boolean":"subseg in [15106]"}}, + {"id":"id:rise:targeting::16","fields":{"boolean":"subseg in [15145]"}}, + {"id":"id:rise:targeting::17","fields":{"boolean":"subseg in [15238]"}}, + {"id":"id:rise:targeting::18","fields":{"boolean":"subseg in [15108]"}}, + {"id":"id:rise:targeting::19","fields":{"boolean":"subseg in [1954297]"}}, + {"id":"id:rise:targeting::20","fields":{"boolean":"subseg in [15136]"}}, + {"id":"id:rise:targeting::21","fields":{"boolean":"subseg in [15125] or subseg in [15126]"}}, + {"id":"id:rise:targeting::22","fields":{"boolean":"subseg in [15138]"}}, + {"id":"id:rise:targeting::23","fields":{"boolean":"subseg in [15140]"}}, + {"id":"id:rise:targeting::24","fields":{"boolean":"subseg in [15088]"}}, + {"id":"id:rise:targeting::25","fields":{"boolean":"subseg in [15089]"}}, + {"id":"id:rise:targeting::26","fields":{"boolean":"subseg in [15143]"}}, + {"id":"id:rise:targeting::27","fields":{"boolean":"subseg in [15149]"}}, + {"id":"id:rise:targeting::28","fields":{"boolean":"subseg in [15158]"}}, + {"id":"id:rise:targeting::29","fields":{"boolean":"subseg in [15060]"}}, + {"id":"id:rise:targeting::30","fields":{"boolean":"subseg in [15081]"}}, + {"id":"id:rise:targeting::31","fields":{"boolean":"subseg in [15144]"}}, + {"id":"id:rise:targeting::32","fields":{"boolean":"subseg in [1550224]"}}, + {"id":"id:rise:targeting::33","fields":{"boolean":"subseg in [15049]"}}, + {"id":"id:rise:targeting::34","fields":{"boolean":"subseg in [1549646]"}}, + {"id":"id:rise:targeting::35","fields":{"boolean":"subseg in [15154]"}}, + {"id":"id:rise:targeting::36","fields":{"boolean":"subseg in [15155]"}}, + {"id":"id:rise:targeting::37","fields":{"boolean":"subseg in [15160]"}}, + {"id":"id:rise:targeting::38","fields":{"boolean":"subseg in [15161]"}}, + {"id":"id:rise:targeting::39","fields":{"boolean":"subseg in [15162]"}}, + {"id":"id:rise:targeting::40","fields":{"boolean":"subseg in [15163]"}}, + {"id":"id:rise:targeting::41","fields":{"boolean":"subseg in [15167]"}}, + {"id":"id:rise:targeting::42","fields":{"boolean":"subseg in [1552820] or subseg in [1552821]"}}, + {"id":"id:rise:targeting::43","fields":{"boolean":"subseg in [15194]"}}, + {"id":"id:rise:targeting::44","fields":{"boolean":"subseg in [1954296]"}} +] diff --git a/predicate-search/src/test/resources/vespa-feed.json b/predicate-search/src/test/resources/vespa-feed.json new file mode 100644 index 00000000000..bb6b9d26796 --- /dev/null +++ b/predicate-search/src/test/resources/vespa-feed.json @@ -0,0 +1,3 @@ +[ + {"id":"id:rise:gd::77908","fields":{"boolean":"dur in ['_dy10_mo2_yr2012', '_dy10_mo5_yr2009', '_dy11_mo2_yr2012', '_dy11_mo5_yr2009', '_dy12_mo2_yr2012', '_dy12_mo5_yr2009', '_dy13_mo2_yr2012', '_dy13_mo5_yr2009', '_dy14_mo2_yr2012', '_dy14_mo5_yr2009', '_dy15_mo2_yr2012', '_dy15_mo5_yr2009', '_dy16_mo2_yr2012', '_dy16_mo5_yr2009', '_dy17_mo2_yr2012', '_dy17_mo5_yr2009', '_dy18_mo2_yr2012', '_dy18_mo5_yr2009', '_dy19_mo2_yr2012', '_dy19_mo5_yr2009', '_dy1_mo2_yr2012', '_dy20_mo2_yr2012', '_dy20_mo5_yr2009', '_dy21_mo2_yr2012', '_dy21_mo5_yr2009', '_dy22_mo2_yr2012', '_dy22_mo5_yr2009', '_dy23_mo2_yr2012', '_dy23_mo5_yr2009', '_dy24_mo2_yr2012', '_dy24_mo5_yr2009', '_dy25_mo2_yr2012', '_dy25_mo5_yr2009', '_dy26_mo2_yr2012', '_dy26_mo5_yr2009', '_dy27_mo5_yr2009', '_dy28_mo5_yr2009', '_dy29_mo5_yr2009', '_dy2_mo2_yr2012', '_dy30_mo5_yr2009', '_dy31_mo5_yr2009', '_dy3_mo2_yr2012', '_dy4_mo2_yr2012', '_dy5_mo2_yr2012', '_dy6_mo2_yr2012', '_dy7_mo2_yr2012', '_dy8_mo2_yr2012', '_dy8_mo5_yr2009', '_dy9_mo2_yr2012', '_dy9_mo5_yr2009', '_hr0_dy27_mo2_yr2012', '_hr10_dy7_mo5_yr2009', '_hr11_dy7_mo5_yr2009', '_hr12_dy7_mo5_yr2009', '_hr13_dy7_mo5_yr2009', '_hr14_dy7_mo5_yr2009', '_hr15_dy7_mo5_yr2009', '_hr16_dy7_mo5_yr2009', '_hr17_dy7_mo5_yr2009', '_hr18_dy7_mo5_yr2009', '_hr19_dy7_mo5_yr2009', '_hr1_dy27_mo2_yr2012', '_hr20_dy7_mo5_yr2009', '_hr21_dy7_mo5_yr2009', '_hr22_dy7_mo5_yr2009', '_hr23_dy7_mo5_yr2009', '_hr2_dy27_mo2_yr2012', '_hr3_dy27_mo2_yr2012', '_hr4_dy27_mo2_yr2012', '_hr4_dy7_mo5_yr2009', '_hr5_dy7_mo5_yr2009', '_hr6_dy7_mo5_yr2009', '_hr7_dy7_mo5_yr2009', '_hr8_dy7_mo5_yr2009', '_hr9_dy7_mo5_yr2009', '_mo10_yr2009', '_mo11_yr2009', '_mo12_yr2009', '_mo1_yr2012', '_mo6_yr2009', '_mo7_yr2009', '_mo8_yr2009', '_mo9_yr2009', '_yr2010', '_yr2011'] and ((ymdevice in [sonyericsson] and ymmodel in ['"k550i"', '"k610i"', '"k800i"', '"s500i"', '"w200i"', '"w300i"', '"w350i"', '"w580i"', '"w760i"', '"w810i"', '"w880i"', '"w910i"']) or (ymdevice in [samsung] and ymmodel in ['"blackjack-ii"', '"sch-i910"', '"sgh-a237"', '"sgh-a737"', '"sgh-a767"', '"sgh-a837"', '"sgh-a867"', '"sgh-a877"', '"sgh-t229"', '"sgh-t429"', '"sgh-t459"', '"sgh-t729"', '"sgh-t739"', '"sgh-t819"', '"sgh-t919"', '"sgh-t929"', '"sph-m800"']) or (ymdevice in [blackberry] and ymmodel in ['"blackberry-7100"', '"blackberry-7130c"', '"blackberry-7130g"', '"blackberry-7250"', '"blackberry-7290"', '"blackberry-8100"', '"blackberry-8110"', '"blackberry-8120"', '"blackberry-8130"', '"blackberry-8220"', '"blackberry-8230f"', '"blackberry-8300"', '"blackberry-8310"', '"blackberry-8320"', '"blackberry-8330"', '"blackberry-8330m"', '"blackberry-8350"', '"blackberry-8530"', '"blackberry-8700"', '"blackberry-8700r"', '"blackberry-8703e"', '"blackberry-8705g"', '"blackberry-8707"', '"blackberry-8800"', '"blackberry-8803"', '"blackberry-8820"', '"blackberry-8830"', '"blackberry-8900"', '"blackberry-9000"', '"blackberry-9020"', '"blackberry-9100"', '"blackberry-9500"', '"blackberry-9530"', '"blackberry-9550"', '"blackberry-9700"']) or (ymdevice in [palm] and ymmodel in ['"pre"', '"pre-pixi"']) or (ymdevice in ['nokia-others'] and ymmodel in ['"2610"', '"3110c"', '"5300"', '"5310"', '"5800-xpressmusic"', '"6085"', '"6120c"', '"6131"', '"6233"', '"6300"', '"6500slide"', '"6650d"', '"6680"', '"6682"', '"7610"']) or (ymdevice in ['nokia-smartphone'] and ymmodel in ['"e51"', '"e61i"', '"e66"', '"e71"', '"e71x"', '"n70"', '"n73"', '"n78"', '"n81"', '"n82"', '"n95"', '"n95_8gb"', '"n96"']) or (ymdevice in [motorola] and ymmodel in ['"q"', '"q9/q9h"', '"v3"', '"v3i"', '"v3xx"', '"v9"', '"w385"']) or (ymdevice in [iphone] and ymmodel in ['"iphone"', '"iphone-2.0"']) or (ymdevice in [lg] and ymmodel in ['"cu515"', '"cu720"', '"cu920"', '"gr500"', '"vx-9100"', '"vx9800"'])) and sme in [0] and ypos in [wp] and yspaceidbase in [954003845]","id":77908}} +] diff --git a/renovate.json b/renovate.json index b815ae4bf88..6e992909ba0 100644 --- a/renovate.json +++ b/renovate.json @@ -21,7 +21,6 @@ { "description": "Disable automatic PRs for artifacts, e.g. fixed version required like ZK dependencies or released to frequently. PRs can still be created manually from dependency dashboard.", "matchPackageNames": [ - "github.com/go-json-experiment/json", "javax.servlet:javax.servlet-api", "io.dropwizard.metrics:metrics-core", "org.apache.zookeeper:zookeeper" diff --git a/screwdriver.yaml b/screwdriver.yaml index 4edef83b63b..edc3cfee3c1 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -496,7 +496,7 @@ jobs: fi verify-opensource-rpm-installable: - image: quay.io/centos/centos:stream8 + image: almalinux:8 annotations: screwdriver.cd/buildPeriodically: H 0 * * * steps: diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 45554882653..6a5d3915d6a 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -1,17 +1,17 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fnet + vespa_fnet vespalog vespalib - metrics - config_cloudconfig - configdefinitions - document - searchlib - messagebus - persistence - searchsummary + vespa_metrics + vespa_config + vespa_configdefinitions + vespa_document + vespa_searchlib + vespa_messagebus + vespa_persistence + vespa_searchsummary fileacquirer LIBS @@ -68,11 +68,8 @@ vespa_define_module( src/tests/proton/bucketdb/bucketdb src/tests/proton/common src/tests/proton/common/alloc_config - src/tests/proton/common/attribute_updater - src/tests/proton/common/document_type_inspector src/tests/proton/common/hw_info_sampler src/tests/proton/common/operation_rate_tracker - src/tests/proton/common/state_reporter_utils src/tests/proton/common/timer src/tests/proton/docsummary src/tests/proton/document_iterator @@ -101,7 +98,6 @@ vespa_define_module( src/tests/proton/documentmetastore/lid_allocator src/tests/proton/documentmetastore/lid_state_vector src/tests/proton/feed_and_search - src/tests/proton/feedoperation src/tests/proton/feedtoken src/tests/proton/flushengine src/tests/proton/flushengine/prepare_restart_flush_strategy @@ -121,9 +117,6 @@ vespa_define_module( src/tests/proton/matching/request_context src/tests/proton/matching/same_element_builder src/tests/proton/matching/unpacking_iterators_optimizer - src/tests/proton/metrics/documentdb_job_trackers - src/tests/proton/metrics/job_load_sampler - src/tests/proton/metrics/job_tracked_flush src/tests/proton/metrics/metrics_engine src/tests/proton/persistenceconformance src/tests/proton/persistenceengine @@ -152,14 +145,13 @@ vespa_define_module( src/tests/proton/server/memory_flush_config_updater src/tests/proton/server/memoryflush src/tests/proton/server/shared_threading_service - src/tests/proton/statusreport src/tests/proton/summaryengine src/tests/proton/verify_ranksetup src/tests/index/disk_indexes src/tests/index/index_disk_layout TEST_DEPENDS - messagebus_messagebus-test + vespa_messagebus-test ) vespa_install_script(src/apps/vespa-remove-indexes/vespa-remove-index.sh vespa-remove-index bin) diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt index 13e4092c2ad..84ff9594510 100644 --- a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt +++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt @@ -1,14 +1,14 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcore_verify_ranksetup +vespa_add_library(vespa_searchcore_verify_ranksetup SOURCES verify_ranksetup.cpp INSTALL lib64 DEPENDS - streamingvisitors + vespa_streamingvisitors searchcore_matching searchcore_documentmetastore ) -vespa_generate_config(searchcore_verify_ranksetup verify-ranksetup.def) +vespa_generate_config(vespa_searchcore_verify_ranksetup verify-ranksetup.def) vespa_add_executable(searchcore_verify_ranksetup_app SOURCES @@ -16,5 +16,5 @@ vespa_add_executable(searchcore_verify_ranksetup_app OUTPUT_NAME vespa-verify-ranksetup-bin INSTALL bin DEPENDS - searchcore_verify_ranksetup + vespa_searchcore_verify_ranksetup ) diff --git a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp index 49c7d2d3867..38fba787bff 100644 --- a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp +++ b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp @@ -45,7 +45,6 @@ public: int usage(const char *self) { fprintf(stderr, "usage: %s <port|spec|--local|--id=name> <cmd> [args]\n", self); - fprintf(stderr, "die\n"); fprintf(stderr, "getProtonStatus\n"); fprintf(stderr, "getState\n"); fprintf(stderr, "triggerFlush\n"); @@ -331,8 +330,6 @@ public: if (! _req->IsError()) { printf("OK: prepareRestart enabled\n"); } - } else if (strcmp(argv[2], "die") == 0) { - _req->SetMethodName("pandora.rtc.die"); } else { finiRPC(); return usage(argv[0]); diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp index e25c675ff64..af37aca4702 100644 --- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -13,7 +13,7 @@ #include <vespa/searchcommon/attribute/i_multi_value_attribute.h> #include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("attribute_initializer_test"); diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 634bc2c93b5..df27c4d7cc2 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -32,7 +32,7 @@ #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/hw_info.h> diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp index 2cdfd908bb1..0d011797395 100644 --- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp @@ -13,7 +13,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/hw_info.h> diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp index 15c26797f15..a20e08d5be2 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -3,7 +3,7 @@ #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchlib/attribute/address_space_components.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp index 6615a0e583a..6f84b793608 100644 --- a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp @@ -14,7 +14,7 @@ #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchcore/proton/attribute/document_field_extractor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::Field; using document::DataType; diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp index 08f9bfdb52d..6d522f254b0 100644 --- a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp @@ -2,7 +2,7 @@ #include <vespa/document/datatype/datatype.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> #include <vespa/searchlib/attribute/attributefactory.h> diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp index d97b2c2a3e7..6d2f375e410 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/attribute/imported_attributes_context.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp index d0b7ac8e688..0447db9c89e 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("imported_attributes_repo_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/basictype.h> #include <vespa/searchcommon/attribute/iattributevector.h> diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp index 4f6d09e6ffa..213d9c38290 100644 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp +++ b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp @@ -6,7 +6,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("bucketdb_test"); diff --git a/searchcore/src/tests/proton/common/.gitignore b/searchcore/src/tests/proton/common/.gitignore index 9ce51ef2178..e69de29bb2d 100644 --- a/searchcore/src/tests/proton/common/.gitignore +++ b/searchcore/src/tests/proton/common/.gitignore @@ -1,2 +0,0 @@ -searchcore_cachedselect_test_app -searchcore_selectpruner_test_app diff --git a/searchcore/src/tests/proton/common/CMakeLists.txt b/searchcore/src/tests/proton/common/CMakeLists.txt index 658afa38247..7eec733214b 100644 --- a/searchcore/src/tests/proton/common/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/CMakeLists.txt @@ -1,24 +1,23 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_selectpruner_test_app TEST +vespa_add_executable(searchcore_proton_common_vespa_test_app TEST SOURCES + vespa_testrunner.cpp selectpruner_test.cpp - DEPENDS - searchcore_pcommon - searchlib_test -) -vespa_add_test(NAME searchcore_selectpruner_test_app COMMAND searchcore_selectpruner_test_app) -vespa_add_executable(searchcore_cachedselect_test_app TEST - SOURCES cachedselect_test.cpp - DEPENDS - searchcore_pcommon - searchlib_test -) -vespa_add_test(NAME searchcore_cachedselect_test_app COMMAND searchcore_cachedselect_test_app) -vespa_add_executable(pendinglidtracker_test_app TEST - SOURCES pendinglidtracker_test.cpp + attribute_updater_test.cpp + state_reporter_utils_test.cpp + document_type_inspector_test.cpp + feedoperation_test.cpp + documentdb_job_trackers_test.cpp + job_load_sampler_test.cpp + job_tracked_flush_test.cpp + statusreport_test.cpp DEPENDS + searchcore_proton_metrics + searchcore_feedoperation searchcore_pcommon + searchcore_test + searchlib_test ) -vespa_add_test(NAME pendinglidtracker_test_app COMMAND pendinglidtracker_test_app) +vespa_add_test(NAME searchcore_proton_common_vespa_test_app COMMAND searchcore_proton_common_vespa_test_app) diff --git a/searchcore/src/tests/proton/common/attribute_updater/.gitignore b/searchcore/src/tests/proton/common/attribute_updater/.gitignore deleted file mode 100644 index 3c6e15d6808..00000000000 --- a/searchcore/src/tests/proton/common/attribute_updater/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -attribute_updater_test -searchcore_attribute_updater_test_app diff --git a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt deleted file mode 100644 index be0da1012d0..00000000000 --- a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_updater_test_app TEST - SOURCES - attribute_updater_test.cpp - DEPENDS - searchcore_pcommon - searchlib_test -) -vespa_add_test(NAME searchcore_attribute_updater_test_app COMMAND searchcore_attribute_updater_test_app) diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater_test.cpp index 0efeaf18c65..432386af0e6 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater_test.cpp @@ -38,10 +38,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("attribute_updater_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using document::config_builder::Array; @@ -472,6 +469,3 @@ TEST_F("require that tensor remove update is applied", } } - -TEST_MAIN() { TEST_RUN_ALL(); } - diff --git a/searchcore/src/tests/proton/common/cachedselect_test.cpp b/searchcore/src/tests/proton/common/cachedselect_test.cpp index a0c8fef3b83..70cec30392b 100644 --- a/searchcore/src/tests/proton/common/cachedselect_test.cpp +++ b/searchcore/src/tests/proton/common/cachedselect_test.cpp @@ -21,10 +21,10 @@ #include <vespa/searchlib/attribute/singlenumericenumattribute.hpp> #include <vespa/searchlib/attribute/singlenumericpostattribute.h> #include <vespa/searchlib/test/mock_attribute_manager.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> -LOG_SETUP("cachedselect_test"); +LOG_SETUP(".cachedselect_test"); using document::DataType; using document::Document; @@ -145,9 +145,7 @@ checkSelect(const NodeUP &sel, } std::ostringstream os; EXPECT_TRUE(sel->trace(ctx, os) == exp); - LOG(info, - "trace output: '%s'", - os.str().c_str()); + LOG(info, "trace output: '%s'", os.str().c_str()); return false; } @@ -679,7 +677,4 @@ TEST_F("Test performance when using attributes", TestFixture) } - } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/document_type_inspector/.gitignore b/searchcore/src/tests/proton/common/document_type_inspector/.gitignore deleted file mode 100644 index 49db4ae7746..00000000000 --- a/searchcore/src/tests/proton/common/document_type_inspector/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_document_type_inspector_test_app diff --git a/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt b/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt deleted file mode 100644 index 339574dc906..00000000000 --- a/searchcore/src/tests/proton/common/document_type_inspector/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_document_type_inspector_test_app TEST - SOURCES - document_type_inspector_test.cpp - DEPENDS - searchcore_pcommon -) -vespa_add_test(NAME searchcore_document_type_inspector_test_app COMMAND searchcore_document_type_inspector_test_app) diff --git a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp b/searchcore/src/tests/proton/common/document_type_inspector_test.cpp index 83106747623..46022ed9273 100644 --- a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp +++ b/searchcore/src/tests/proton/common/document_type_inspector_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/common/document_type_inspector.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/field.h> #include <vespa/document/datatype/datatypes.h> @@ -127,8 +127,3 @@ TEST_F("require that struct addition is detected", Fixture(false, false)) EXPECT_FALSE(inspector.hasUnchangedField("map.key")); EXPECT_FALSE(inspector.hasUnchangedField("map.value")); } - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp b/searchcore/src/tests/proton/common/documentdb_job_trackers_test.cpp index c32b3439c5d..89c3b164e96 100644 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp +++ b/searchcore/src/tests/proton/common/documentdb_job_trackers_test.cpp @@ -3,12 +3,9 @@ #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> #include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h> #include <vespa/searchcore/proton/test/dummy_flush_target.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <thread> -#include <vespa/log/log.h> -LOG_SETUP("documentdb_job_trackers_test"); - using namespace proton; using namespace searchcorespi; @@ -115,5 +112,3 @@ TEST_F("require that un-known flush targets are not tracked", Fixture) EXPECT_EQUAL(1u, output.size()); EXPECT_EQUAL(&*output[0].get(), &*input[0]); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/common/feedoperation_test.cpp index b4cf29f67e0..48893aa7da3 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/common/feedoperation_test.cpp @@ -24,7 +24,7 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::BucketId; using document::DataType; @@ -357,5 +357,3 @@ TEST_F("require that we can serialize and deserialize remove by gid operations", } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp index d23505dae9c..ad74039a5ee 100644 --- a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp +++ b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using proton::HwInfoSampler; using search::test::DirectoryHandler; diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp b/searchcore/src/tests/proton/common/job_load_sampler_test.cpp index 2b74d1425a1..b6fcd3fe092 100644 --- a/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp +++ b/searchcore/src/tests/proton/common/job_load_sampler_test.cpp @@ -1,9 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("job_load_sampler_test"); #include <vespa/searchcore/proton/metrics/job_load_sampler.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <chrono> using namespace proton; @@ -101,5 +99,3 @@ TEST_F("require that multiple jobs that starts and ends in several intervals get f.end(45); EXPECT_APPROX(0.5, f.sample(50), EPS); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp b/searchcore/src/tests/proton/common/job_tracked_flush_test.cpp index fa6b158136f..608fbb60a70 100644 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp +++ b/searchcore/src/tests/proton/common/job_tracked_flush_test.cpp @@ -5,14 +5,11 @@ #include <vespa/searchcore/proton/test/dummy_flush_target.h> #include <vespa/searchcore/proton/test/simple_job_tracker.h> #include <vespa/searchlib/common/flush_token.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/gate.h> -#include <vespa/log/log.h> -LOG_SETUP("job_tracked_flush_test"); - using namespace proton; using namespace searchcorespi; using search::SerialNum; @@ -134,5 +131,3 @@ TEST_F("require that nullptr flush task is not tracked", Fixture) FlushTask::UP task = f._trackedFlush.initFlush(0, std::make_shared<search::FlushToken>()); EXPECT_TRUE(task.get() == nullptr); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp b/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp index 1aac149b7e7..f8d0d218670 100644 --- a/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp +++ b/searchcore/src/tests/proton/common/pendinglidtracker_test.cpp @@ -1,11 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/common/pendinglidtracker.h> -#include <vespa/log/log.h> -LOG_SETUP("pendinglidtracker_test"); - using namespace proton; constexpr uint32_t LID_1 = 1u; @@ -76,5 +73,3 @@ TEST("test pendinglidtracker for needcommit") { EXPECT_EQUAL(ILidCommitState::State::COMPLETED, tracker.getState(LID_1)); EXPECT_EQUAL(ILidCommitState::State::COMPLETED, tracker.getState(LIDV_2_1_3)); } - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp index 1f71da5aeda..09854af99e1 100644 --- a/searchcore/src/tests/proton/common/selectpruner_test.cpp +++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp @@ -10,10 +10,10 @@ #include <vespa/document/select/parser.h> #include <vespa/document/select/cloningvisitor.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> -LOG_SETUP("selectpruner_test"); +LOG_SETUP(".selectpruner_test"); using document::DataType; using document::Document; @@ -824,5 +824,3 @@ TEST_F("Complex imported field references return Invalid", TestFixture) } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore b/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore deleted file mode 100644 index bb0963e5ec3..00000000000 --- a/searchcore/src/tests/proton/common/state_reporter_utils/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_state_reporter_utils_test_app diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt b/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt deleted file mode 100644 index 1bdb0b613cf..00000000000 --- a/searchcore/src/tests/proton/common/state_reporter_utils/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_state_reporter_utils_test_app TEST - SOURCES - state_reporter_utils_test.cpp - DEPENDS - searchcore_pcommon -) -vespa_add_test(NAME searchcore_state_reporter_utils_test_app COMMAND searchcore_state_reporter_utils_test_app) diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp b/searchcore/src/tests/proton/common/state_reporter_utils_test.cpp index 749f8b147ac..6c9025d276f 100644 --- a/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp +++ b/searchcore/src/tests/proton/common/state_reporter_utils_test.cpp @@ -1,10 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("state_reporter_utils_test"); #include <vespa/searchcore/proton/common/state_reporter_utils.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace proton; using namespace vespalib::slime; @@ -44,4 +42,3 @@ TEST("require that advanced status report is correctly converted to slime") message("foo")))); } -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/statusreport/statusreport_test.cpp b/searchcore/src/tests/proton/common/statusreport_test.cpp index d1ef6c3af29..052eb795529 100644 --- a/searchcore/src/tests/proton/statusreport/statusreport_test.cpp +++ b/searchcore/src/tests/proton/common/statusreport_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/common/statusreport.h> namespace proton { @@ -37,5 +37,3 @@ TEST("require that custom status report works") } } // namespace proton - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/vespa_testrunner.cpp b/searchcore/src/tests/proton/common/vespa_testrunner.cpp new file mode 100644 index 00000000000..1e4e79047c3 --- /dev/null +++ b/searchcore/src/tests/proton/common/vespa_testrunner.cpp @@ -0,0 +1,8 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Unit tests for predicate_index. +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/log/log.h> +LOG_SETUP("proton_common_test"); + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp index 173cfbbf052..361e37278da 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp @@ -65,7 +65,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/encoding/base64.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp index c428f350d1a..e09a0d2ea2b 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchcore/proton/test/test.h> #include <vespa/persistence/spi/test.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("buckethandler_test"); diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp index 3904156170d..c713108dcc0 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchcore/proton/test/test.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("combiningfeedview_test"); diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 2504bca17e4..7c87e16bb35 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-summary.h> #include <vespa/document/datatype/documenttype.h> diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp index 56fb1ab3914..e3ab2737fe5 100644 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp @@ -3,7 +3,7 @@ #include <vespa/searchcore/proton/server/document_scan_iterator.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/documentid.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index 75ef56299d9..73d27672380 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -5,7 +5,7 @@ #include <vespa/config-rank-profiles.h> #include <vespa/searchcore/proton/server/documentdbconfig.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-summary.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp index 95ad4bd143b..862792bf274 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/server/documentdbconfig.h> #include <vespa/searchcore/proton/server/documentdbconfigscout.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-attributes.h> #include <ostream> diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 808c5743538..cdd275e898d 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -38,7 +38,7 @@ #include <vespa/searchlib/test/doc_builder.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/net/socket_spec.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp index b92560c57e9..e1ec3d2bb29 100644 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp +++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/server/i_blockable_maintenance_job.h> #include <vespa/searchcore/proton/server/job_tracked_maintenance_job.h> #include <vespa/searchcore/proton/test/simple_job_tracker.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/threadstackexecutor.h> diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 509210679da..6c67848ae51 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -35,7 +35,7 @@ #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/lambdatask.h> diff --git a/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp b/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp index 62530e9de7b..766158bd178 100644 --- a/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp +++ b/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp @@ -2,7 +2,7 @@ #include <vespa/searchcore/proton/server/i_blockable_maintenance_job.h> #include <vespa/searchcore/proton/server/move_operation_limiter.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <queue> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index e6923674584..db30c8e03fb 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -17,7 +17,7 @@ #include <vespa/searchlib/test/doc_builder.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("storeonlyfeedview_test"); diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index 40b1904cd1a..7a16e6b17cb 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -2,7 +2,7 @@ #include <vespa/config-proton.h> #include <vespa/searchcore/proton/server/threading_service_config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/hw_info.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index 4bcfb2eedd9..335119927f7 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/documentmetastore/i_store.h> #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h> #include <vespa/searchcore/proton/test/thread_utils.h> diff --git a/searchcore/src/tests/proton/feedoperation/.gitignore b/searchcore/src/tests/proton/feedoperation/.gitignore deleted file mode 100644 index cfdeb9049b2..00000000000 --- a/searchcore/src/tests/proton/feedoperation/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*_test -.depend -Makefile -searchcore_feedoperation_test_app diff --git a/searchcore/src/tests/proton/feedoperation/CMakeLists.txt b/searchcore/src/tests/proton/feedoperation/CMakeLists.txt deleted file mode 100644 index fe9d3bad302..00000000000 --- a/searchcore/src/tests/proton/feedoperation/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_feedoperation_test_app TEST - SOURCES - feedoperation_test.cpp - DEPENDS - searchcore_feedoperation - searchcore_pcommon -) -vespa_add_test(NAME searchcore_feedoperation_test_app COMMAND searchcore_feedoperation_test_app) diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index 9a0dc47771f..dc9dcd3e0b0 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -13,7 +13,7 @@ #include <vespa/searchlib/common/flush_token.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <mutex> #include <thread> diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 06a2ebec958..7abac088011 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/flushengine/active_flush_stats.h> #include <vespa/searchcore/proton/flushengine/flush_target_candidate.h> diff --git a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp index 39175e8c27c..70a8887b9bb 100644 --- a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp +++ b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/common/i_compactable_lid_space.h> diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp index 51465c59ac7..c52eb940817 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/test/mock_index_manager.h> #include <vespa/searchlib/test/doc_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/initializer/task_runner_test.cpp b/searchcore/src/tests/proton/initializer/task_runner_test.cpp index 82cce924832..a8b9969299f 100644 --- a/searchcore/src/tests/proton/initializer/task_runner_test.cpp +++ b/searchcore/src/tests/proton/initializer/task_runner_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("task_runner_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> #include <vespa/searchcore/proton/initializer/task_runner.h> #include <vespa/vespalib/stllike/string.h> diff --git a/searchcore/src/tests/proton/matching/CMakeLists.txt b/searchcore/src/tests/proton/matching/CMakeLists.txt index c35e9498986..be9e10c45a0 100644 --- a/searchcore/src/tests/proton/matching/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/CMakeLists.txt @@ -13,7 +13,7 @@ vespa_add_executable(searchcore_matching_test_app TEST searchcore_grouping searchlib_test ) -vespa_add_test(NAME searchcore_matching_test_app COMMAND searchcore_matching_test_app) +vespa_add_test(NAME searchcore_matching_test_app COMMAND searchcore_matching_test_app COST 100) vespa_add_executable(searchcore_sessionmanager_test_app TEST SOURCES sessionmanager_test.cpp diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt b/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt index b545023ce97..b5b71836581 100644 --- a/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/match_loop_communicator/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchcore_match_loop_communicator_test_app TEST match_loop_communicator_test.cpp DEPENDS searchcore_matching + GTest::gtest ) vespa_add_test(NAME searchcore_match_loop_communicator_test_app COMMAND searchcore_match_loop_communicator_test_app) diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp index d5ee88e1617..dc05471a1eb 100644 --- a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp +++ b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp @@ -1,7 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/match_loop_communicator.h> +#include <vespa/searchlib/features/first_phase_rank_lookup.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/test/nexus.h> #include <algorithm> +#include <atomic> using namespace proton::matching; @@ -12,10 +15,22 @@ using Hit = MatchLoopCommunicator::Hit; using Hits = MatchLoopCommunicator::Hits; using TaggedHit = MatchLoopCommunicator::TaggedHit; using TaggedHits = MatchLoopCommunicator::TaggedHits; +using search::features::FirstPhaseRankLookup; using search::queryeval::SortedHitSequence; +using vespalib::test::Nexus; + +namespace search::queryeval { + +void PrintTo(const Scores& scores, std::ostream* os) { + *os << "{" << scores.low << "," << scores.high << "}"; +} + +} std::vector<Hit> hit_vec(std::vector<Hit> list) { return list; } +auto do_nothing = []() noexcept {}; + Hits makeScores(size_t id) { switch (id) { case 0: return {{1, 5.4}, {2, 4.4}, {3, 3.4}, {4, 2.4}, {5, 1.4}}; @@ -27,6 +42,13 @@ Hits makeScores(size_t id) { return {}; } +Hits make_first_scores(size_t id, size_t size) { + auto result = makeScores(id); + EXPECT_LE(size, result.size()); + result.resize(size); + return result; +} + std::tuple<size_t,Hits,RangePair> second_phase(MatchLoopCommunicator &com, const Hits &hits, size_t thread_id, double delta = 0.0) { std::vector<uint32_t> refs; for (size_t i = 0; i < hits.size(); ++i) { @@ -60,25 +82,6 @@ size_t my_work_size(MatchLoopCommunicator &com, const Hits &hits, size_t thread_ return work_size; } -void equal(size_t count, const Hits & a, const Hits & b) { - EXPECT_EQUAL(count, b.size()); - for (size_t i(0); i < count; i++) { - EXPECT_EQUAL(a[i].first, b[i].first); - EXPECT_EQUAL(a[i].second , b[i].second); - } -} - -void equal_range(const Range &a, const Range &b) { - EXPECT_EQUAL(a.isValid(), b.isValid()); - EXPECT_EQUAL(a.low, b.low); - EXPECT_EQUAL(a.high, b.high); -} - -void equal_ranges(const RangePair &a, const RangePair &b) { - TEST_DO(equal_range(a.first, b.first)); - TEST_DO(equal_range(a.second, b.second)); -} - struct EveryOdd : public search::queryeval::IDiversifier { bool accepted(uint32_t docId) override { return docId & 0x01; @@ -89,122 +92,242 @@ struct None : public search::queryeval::IDiversifier { bool accepted(uint32_t) override { return false; } }; -TEST_F("require that selectBest gives appropriate results for single thread", MatchLoopCommunicator(num_threads, 3)) { - TEST_DO(equal(2u, hit_vec({{1, 5}, {2, 4}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}}), thread_id))); - TEST_DO(equal(3u, hit_vec({{1, 5}, {2, 4}, {3, 3}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}, {3, 3}}), thread_id))); - TEST_DO(equal(3u, hit_vec({{1, 5}, {2, 4}, {3, 3}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}, {3, 3}, {4, 2}}), thread_id))); +TEST(MatchLoopCommunicatorTest, require_that_selectBest_gives_appropriate_results_for_single_thread) +{ + constexpr size_t num_threads = 1; + constexpr size_t thread_id = 0; + MatchLoopCommunicator f1(num_threads, 3); + EXPECT_EQ(hit_vec({{1, 5}, {2, 4}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}}), thread_id)); + EXPECT_EQ(hit_vec({{1, 5}, {2, 4}, {3, 3}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}, {3, 3}}), thread_id)); + EXPECT_EQ(hit_vec({{1, 5}, {2, 4}, {3, 3}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}, {3, 3}, {4, 2}}), thread_id)); } -TEST_F("require that selectBest gives appropriate results for single thread with filter", - MatchLoopCommunicator(num_threads, 3, std::make_unique<EveryOdd>())) +TEST(MatchLoopCommunicatorTest, require_that_selectBest_gives_appropriate_results_for_single_thread_with_filter) { - TEST_DO(equal(1u, hit_vec({{1, 5}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}}), thread_id))); - TEST_DO(equal(2u, hit_vec({{1, 5}, {3, 3}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}, {3, 3}}), thread_id))); - TEST_DO(equal(3u, hit_vec({{1, 5}, {3, 3}, {5, 1}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}, {3, 3}, {4, 2}, {5, 1}, {6, 0}}), thread_id))); + constexpr size_t num_threads = 1; + constexpr size_t thread_id = 0; + MatchLoopCommunicator f1(num_threads, 3, std::make_unique<EveryOdd>(), nullptr, do_nothing); + EXPECT_EQ(hit_vec({{1, 5}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}}), thread_id)); + EXPECT_EQ(hit_vec({{1, 5}, {3, 3}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}, {3, 3}}), thread_id)); + EXPECT_EQ(hit_vec({{1, 5}, {3, 3}, {5, 1}}), selectBest(f1, hit_vec({{1, 5}, {2, 4}, {3, 3}, {4, 2}, {5, 1}, {6, 0}}), thread_id)); } -TEST_MT_F("require that selectBest works with no hits", 10, MatchLoopCommunicator(num_threads, 10)) { - EXPECT_TRUE(selectBest(f1, hit_vec({}), thread_id).empty()); +TEST(MatchLoopCommunicatorTest, require_that_selectBest_works_with_no_hits) +{ + constexpr size_t num_threads = 10; + MatchLoopCommunicator f1(num_threads, 10); + auto task = [&f1](Nexus& ctx) { + EXPECT_TRUE(selectBest(f1, hit_vec({}), ctx.thread_id()).empty()); + }; + Nexus::run(num_threads, task); } -TEST_MT_F("require that selectBest works with too many hits from all threads", 5, MatchLoopCommunicator(num_threads, 13)) { - if (thread_id < 3) { - TEST_DO(equal(3u, makeScores(thread_id), selectBest(f1, makeScores(thread_id), thread_id))); - } else { - TEST_DO(equal(2u, makeScores(thread_id), selectBest(f1, makeScores(thread_id), thread_id))); - } +TEST(MatchLoopCommunicatorTest, require_that_selectBest_works_with_too_many_hits_from_all_threads) +{ + constexpr size_t num_threads = 5; + MatchLoopCommunicator f1(num_threads, 13); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + if (thread_id < 3) { + EXPECT_EQ(make_first_scores(thread_id, 3), selectBest(f1, makeScores(thread_id), thread_id)); + } else { + EXPECT_EQ(make_first_scores(thread_id, 2), selectBest(f1, makeScores(thread_id), thread_id)); + } + }; + Nexus::run(num_threads, task); } -TEST_MT_F("require that selectBest works with some exhausted threads", 5, MatchLoopCommunicator(num_threads, 22)) { - if (thread_id < 2) { - TEST_DO(equal(5u, makeScores(thread_id), selectBest(f1, makeScores(thread_id), thread_id))); - } else { - TEST_DO(equal(4u, makeScores(thread_id), selectBest(f1, makeScores(thread_id), thread_id))); - } +TEST(MatchLoopCommunicatorTest, require_that_selectBest_works_with_some_exhausted_threads) +{ + constexpr size_t num_threads = 5; + MatchLoopCommunicator f1(num_threads, 22); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + if (thread_id < 2) { + EXPECT_EQ(makeScores(thread_id), selectBest(f1, makeScores(thread_id), thread_id)); + } else { + EXPECT_EQ(make_first_scores(thread_id, 4), selectBest(f1, makeScores(thread_id), thread_id)); + } + }; + Nexus::run(num_threads, task); } -TEST_MT_F("require that selectBest can select all hits from all threads", 5, MatchLoopCommunicator(num_threads, 100)) { - EXPECT_EQUAL(5u, selectBest(f1, makeScores(thread_id), thread_id).size()); +TEST(MatchLoopCommunicatorTest, require_that_selectBest_can_select_all_hits_from_all_threads) +{ + constexpr size_t num_threads = 5; + MatchLoopCommunicator f1(num_threads, 100); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + EXPECT_EQ(5u, selectBest(f1, makeScores(thread_id), thread_id).size()); + }; + Nexus::run(num_threads, task); } -TEST_MT_F("require that selectBest works with some empty threads", 10, MatchLoopCommunicator(num_threads, 7)) { - if (thread_id < 2) { - TEST_DO(equal(2u, makeScores(thread_id), selectBest(f1, makeScores(thread_id), thread_id))); - } else if (thread_id < 5) { - TEST_DO(equal(1u, makeScores(thread_id), selectBest(f1, makeScores(thread_id), thread_id))); - } else { - EXPECT_TRUE(selectBest(f1, makeScores(thread_id), thread_id).empty()); - } +TEST(MatchLoopCommunicatorTest, require_that_selectBest_works_with_some_empty_threads) +{ + constexpr size_t num_threads = 5; + MatchLoopCommunicator f1(num_threads, 7); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + if (thread_id < 2) { + EXPECT_EQ(make_first_scores(thread_id, 2), selectBest(f1, makeScores(thread_id), thread_id)); + } else if (thread_id < 5) { + EXPECT_EQ(make_first_scores(thread_id, 1), selectBest(f1, makeScores(thread_id), thread_id)); + } else { + EXPECT_TRUE(selectBest(f1, makeScores(thread_id), thread_id).empty()); + } + }; + Nexus::run(num_threads, task); } -TEST_F("require that rangeCover works with a single thread", MatchLoopCommunicator(num_threads, 5)) { +TEST(MatchLoopCommunicatorTest, require_that_rangeCover_works_with_a_single_thread) +{ + constexpr size_t num_threads = 1; + constexpr size_t thread_id = 0; + MatchLoopCommunicator f1(num_threads, 5); RangePair res = rangeCover(f1, hit_vec({{1, 7.5}, {2, 1.5}}), thread_id, 10); - TEST_DO(equal_ranges(RangePair({1.5, 7.5}, {11.5, 17.5}), res)); + EXPECT_EQ(RangePair({1.5, 7.5}, {11.5, 17.5}), res); } -TEST_MT_F("require that rangeCover works with multiple threads", 5, MatchLoopCommunicator(num_threads, 10)) { - RangePair res = rangeCover(f1, hit_vec({{thread_id * 100 + 1, 100.0 + thread_id}, {thread_id * 100 + 2, 100.0 - thread_id}}), thread_id, 10); - TEST_DO(equal_ranges(RangePair({96.0, 104.0}, {106.0, 114.0}), res)); +TEST(MatchLoopCommunicatorTest, require_that_rangeCover_works_with_multiple_threads) +{ + constexpr size_t num_threads = 5; + MatchLoopCommunicator f1(num_threads, 10); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + RangePair res = rangeCover(f1, hit_vec({{thread_id * 100 + 1, 100.0 + thread_id}, {thread_id * 100 + 2, 100.0 - thread_id}}), thread_id, 10); + EXPECT_EQ(RangePair({96.0, 104.0}, {106.0, 114.0}), res); + }; + Nexus::run(num_threads, task); } -TEST_MT_F("require that rangeCover works with no hits", 10, MatchLoopCommunicator(num_threads, 5)) { - RangePair res = rangeCover(f1, hit_vec({}), thread_id, 10); - TEST_DO(equal_ranges(RangePair({}, {}), res)); +TEST(MatchLoopCommunicatorTest, require_that_rangeCover_works_with_no_hits) +{ + constexpr size_t num_threads = 10; + MatchLoopCommunicator f1(num_threads, 5); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + RangePair res = rangeCover(f1, hit_vec({}), thread_id, 10); + EXPECT_EQ(RangePair({}, {}), res); + }; + Nexus::run(num_threads, task); } -TEST_FFF("require that hits dropped due to lack of diversity affects range cover result", - MatchLoopCommunicator(num_threads, 3), - MatchLoopCommunicator(num_threads, 3, std::make_unique<EveryOdd>()), - MatchLoopCommunicator(num_threads, 3, std::make_unique<None>())) +TEST(MatchLoopCommunicatorTest, require_that_hits_dropped_due_to_lack_of_diversity_affects_range_cover_result) { + constexpr size_t num_threads = 1; + constexpr size_t thread_id = 0; + MatchLoopCommunicator f1(num_threads, 3); + MatchLoopCommunicator f2(num_threads, 3, std::make_unique<EveryOdd>(), nullptr, do_nothing); + MatchLoopCommunicator f3(num_threads, 3, std::make_unique<None>(), nullptr, do_nothing); auto hits_in = hit_vec({{1, 5}, {2, 4}, {3, 3}, {4, 2}, {5, 1}}); auto [my_work1, hits1, ranges1] = second_phase(f1, hits_in, thread_id, 10); auto [my_work2, hits2, ranges2] = second_phase(f2, hits_in, thread_id, 10); auto [my_work3, hits3, ranges3] = second_phase(f3, hits_in, thread_id, 10); - EXPECT_EQUAL(my_work1, 3u); - EXPECT_EQUAL(my_work2, 3u); - EXPECT_EQUAL(my_work3, 0u); + EXPECT_EQ(my_work1, 3u); + EXPECT_EQ(my_work2, 3u); + EXPECT_EQ(my_work3, 0u); - TEST_DO(equal(3u, hit_vec({{1, 15}, {2, 14}, {3, 13}}), hits1)); - TEST_DO(equal(3u, hit_vec({{1, 15}, {3, 13}, {5, 11}}), hits2)); - TEST_DO(equal(0u, hit_vec({}), hits3)); + EXPECT_EQ(hit_vec({{1, 15}, {2, 14}, {3, 13}}), hits1); + EXPECT_EQ(hit_vec({{1, 15}, {3, 13}, {5, 11}}), hits2); + EXPECT_EQ(hit_vec({}), hits3); - TEST_DO(equal_ranges(RangePair({3,5},{13,15}), ranges1)); - TEST_DO(equal_ranges(RangePair({4,5},{11,15}), ranges2)); // best dropped: 4 + EXPECT_EQ(RangePair({3,5},{13,15}), ranges1); + EXPECT_EQ(RangePair({4,5},{11,15}), ranges2); // best dropped: 4 // note that the 'drops all hits due to diversity' case will // trigger much of the same code path as dropping second phase // ranking due to hard doom. - TEST_DO(equal_ranges(RangePair({},{}), ranges3)); + EXPECT_EQ(RangePair({},{}), ranges3); } -TEST_MT_F("require that estimate_match_frequency will count hits and docs across threads", 4, MatchLoopCommunicator(num_threads, 5)) { - double freq = (0.0/10.0 + 1.0/11.0 + 2.0/12.0 + 3.0/13.0) / 4.0; - EXPECT_APPROX(freq, f1.estimate_match_frequency(Matches(thread_id, thread_id + 10)), 0.00001); +TEST(MatchLoopCommunicatorTest, require_that_estimate_match_frequency_will_count_hits_and_docs_across_threads) +{ + constexpr size_t num_threads = 4; + MatchLoopCommunicator f1(num_threads, 5); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + double freq = (0.0/10.0 + 1.0/11.0 + 2.0/12.0 + 3.0/13.0) / 4.0; + EXPECT_NEAR(freq, f1.estimate_match_frequency(Matches(thread_id, thread_id + 10)), 0.00001); + }; + Nexus::run(num_threads, task); } -TEST_MT_F("require that second phase work is evenly distributed among search threads", 5, MatchLoopCommunicator(num_threads, 20)) { - size_t num_hits = thread_id * 5; - size_t docid = thread_id * 100; - double score = thread_id * 100.0; - Hits my_hits; - for(size_t i = 0; i < num_hits; ++i) { - my_hits.emplace_back(++docid, score); - score -= 1.0; - } - auto [my_work, best_hits, ranges] = second_phase(f1, my_hits, thread_id, 1000.0); - EXPECT_EQUAL(my_work, 4u); - TEST_DO(equal_ranges(RangePair({381,400},{1381,1400}), ranges)); - if (thread_id == 4) { - for (auto &hit: my_hits) { - hit.second += 1000.0; - } - TEST_DO(equal(num_hits, my_hits, best_hits)); - } else { - EXPECT_TRUE(best_hits.empty()); +TEST(MatchLoopCommunicatorTest, require_that_second_phase_work_is_evenly_distributed_among_search_threads) +{ + constexpr size_t num_threads = 5; + MatchLoopCommunicator f1(num_threads, 20); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + size_t num_hits = thread_id * 5; + size_t docid = thread_id * 100; + double score = thread_id * 100.0; + Hits my_hits; + for(size_t i = 0; i < num_hits; ++i) { + my_hits.emplace_back(++docid, score); + score -= 1.0; + } + auto [my_work, best_hits, ranges] = second_phase(f1, my_hits, thread_id, 1000.0); + EXPECT_EQ(my_work, 4u); + EXPECT_EQ(RangePair({381,400},{1381,1400}), ranges); + if (thread_id == 4) { + for (auto &hit: my_hits) { + hit.second += 1000.0; + } + EXPECT_EQ(my_hits, best_hits); + } else { + EXPECT_TRUE(best_hits.empty()); + } + }; + Nexus::run(num_threads, task); +} + +namespace { + +std::vector<double> extract_ranks(const FirstPhaseRankLookup& l) { + std::vector<double> result; + for (uint32_t docid = 21; docid < 26; ++docid) { + result.emplace_back(l.lookup(docid)); } + return result; +} + +search::feature_t unranked = std::numeric_limits<search::feature_t>::max(); + +using FeatureVec = std::vector<search::feature_t>; + +} + +TEST(MatchLoopCommunicatorTest, require_that_first_phase_rank_lookup_is_populated) +{ + constexpr size_t num_threads = 1; + constexpr size_t thread_id = 0; + FirstPhaseRankLookup l1; + FirstPhaseRankLookup l2; + MatchLoopCommunicator f1(num_threads, 3, {}, &l1, do_nothing); + MatchLoopCommunicator f2(num_threads, 3, std::make_unique<EveryOdd>(), &l2, do_nothing); + auto hits_in = hit_vec({{21, 5}, {22, 4}, {23, 3}, {24, 2}, {25, 1}}); + auto res1 = second_phase(f1, hits_in, thread_id, 10); + auto res2 = second_phase(f2, hits_in, thread_id, 10); + EXPECT_EQ(FeatureVec({1, 2, 3, unranked, unranked}), extract_ranks(l1)); + EXPECT_EQ(FeatureVec({1, unranked, 3, unranked, 5}), extract_ranks(l2)); +} + +TEST(MatchLoopCommunicatorTest, require_that_before_second_phase_is_called_once) +{ + constexpr size_t num_threads = 5; + std::atomic<int> cnt(0); + auto before_second_phase = [&cnt]() noexcept { ++cnt; }; + MatchLoopCommunicator f1(num_threads, 3, {}, nullptr, before_second_phase); + auto task = [&f1](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + auto hits_in = hit_vec({}); + (void) second_phase(f1, hits_in, thread_id, 1000.0); + }; + Nexus::run(num_threads, task); + EXPECT_EQ(1, cnt.load(std::memory_order_acquire)); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 6dd8a93bcbd..fc0e5acdaa0 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -31,9 +31,10 @@ #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_codec.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/featureset.h> +#include <vespa/vespalib/util/limited_thread_bundle_wrapper.h> #include <vespa/vespalib/util/simple_thread_bundle.h> #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -65,6 +66,97 @@ using vespalib::eval::TensorSpec; using vespalib::FeatureSet; using vespalib::nbostream; +constexpr uint32_t NUM_DOCS = 1000; + +class MatchingTestSharedState { + std::unique_ptr<vespalib::SimpleThreadBundle> _thread_bundle; + std::unique_ptr<MockAttributeContext> _attribute_context; + std::unique_ptr<DocumentMetaStore> _meta_store; +public: + static constexpr size_t max_threads = 75; + MatchingTestSharedState(); + ~MatchingTestSharedState(); + vespalib::ThreadBundle& thread_bundle(); + IAttributeContext& attribute_context(); + const proton::IDocumentMetaStore& meta_store(); +}; + +MatchingTestSharedState::MatchingTestSharedState() + : _thread_bundle(), + _attribute_context(), + _meta_store() +{ +} + +MatchingTestSharedState::~MatchingTestSharedState() = default; + +vespalib::ThreadBundle& +MatchingTestSharedState::thread_bundle() +{ + if (!_thread_bundle) { + _thread_bundle = std::make_unique<vespalib::SimpleThreadBundle>(max_threads); + } + return *_thread_bundle; +} + +IAttributeContext& +MatchingTestSharedState::attribute_context() +{ + if (!_attribute_context) { + _attribute_context = std::make_unique<MockAttributeContext>(); + // attribute context + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("a1"); + AttributeVector::DocId docid(0); + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i, docid); // value = docid + } + assert(docid + 1 == NUM_DOCS); + _attribute_context->add(attr); + } + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("a2"); + AttributeVector::DocId docid(0); + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i * 2, docid); // value = docid * 2 + } + assert(docid + 1 == NUM_DOCS); + _attribute_context->add(attr); + } + { + auto attr = std::make_shared<SingleInt32ExtAttribute>("a3"); + AttributeVector::DocId docid(0); + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + attr->addDoc(docid); + attr->add(i%10, docid); + } + assert(docid + 1 == NUM_DOCS); + _attribute_context->add(attr); + } + } + return *_attribute_context; +} + +const proton::IDocumentMetaStore& +MatchingTestSharedState::meta_store() +{ + if (!_meta_store) { + _meta_store = std::make_unique<DocumentMetaStore>(std::make_shared<bucketdb::BucketDBOwner>()); + // metaStore + for (uint32_t i = 0; i < NUM_DOCS; ++i) { + document::DocumentId docId(vespalib::make_string("id:ns:searchdocument::%u", i)); + const document::GlobalId &gid = docId.getGlobalId(); + document::BucketId bucketId(BucketFactory::getBucketId(docId)); + uint32_t docSize = 1; + _meta_store->put(gid, bucketId, Timestamp(0u), docSize, i, 0u); + _meta_store->setBucketState(bucketId, true); + } + } + return *_meta_store; +} + vespalib::ThreadBundle &ttb() { return vespalib::ThreadBundle::trivial(); } void inject_match_phase_limiting(Properties &setup, const vespalib::string &attribute, size_t max_hits, bool descending) @@ -106,8 +198,6 @@ vespalib::string make_same_element_stack_dump(const vespalib::string &a1_term, c //----------------------------------------------------------------------------- -const uint32_t NUM_DOCS = 1000; - struct EmptyRankingAssetsRepo : public search::fef::IRankingAssetsRepo { vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &) const override { return {}; @@ -125,18 +215,19 @@ struct EmptyRankingAssetsRepo : public search::fef::IRankingAssetsRepo { //----------------------------------------------------------------------------- struct MyWorld { + MatchingTestSharedState& shared_state; Schema schema; Properties config; FakeSearchContext searchContext; - MockAttributeContext attributeContext; + IAttributeContext& attributeContext; std::shared_ptr<SessionManager> sessionManager; - DocumentMetaStore metaStore; + const proton::IDocumentMetaStore& metaStore; MatchingStats matchingStats; vespalib::TestClock clock; QueryLimiter queryLimiter; EmptyRankingAssetsRepo constantValueRepo; - MyWorld(); + MyWorld(MatchingTestSharedState& shared_state); ~MyWorld(); void basicSetup(size_t heapSize=10, size_t arraySize=100) { @@ -172,50 +263,9 @@ struct MyWorld { // odd -> 1 } - // attribute context - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("a1"); - AttributeVector::DocId docid(0); - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i, docid); // value = docid - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); - } - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("a2"); - AttributeVector::DocId docid(0); - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i * 2, docid); // value = docid * 2 - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); - } - { - auto attr = std::make_shared<SingleInt32ExtAttribute>("a3"); - AttributeVector::DocId docid(0); - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - attr->addDoc(docid); - attr->add(i%10, docid); - } - assert(docid + 1 == NUM_DOCS); - attributeContext.add(attr); - } - // grouping sessionManager = std::make_shared<SessionManager>(100); - // metaStore - for (uint32_t i = 0; i < NUM_DOCS; ++i) { - document::DocumentId docId(vespalib::make_string("id:ns:searchdocument::%u", i)); - const document::GlobalId &gid = docId.getGlobalId(); - document::BucketId bucketId(BucketFactory::getBucketId(docId)); - uint32_t docSize = 1; - metaStore.put(gid, bucketId, Timestamp(0u), docSize, i, 0u); - metaStore.setBucketState(bucketId, true); - } } void set_property(const vespalib::string &name, const vespalib::string &value) { @@ -241,30 +291,30 @@ struct MyWorld { static void verify_match_features(SearchReply &reply, const vespalib::string &matched_field) { if (reply.hits.empty()) { - EXPECT_EQUAL(reply.match_features.names.size(), 0u); - EXPECT_EQUAL(reply.match_features.values.size(), 0u); + EXPECT_EQ(reply.match_features.names.size(), 0u); + EXPECT_EQ(reply.match_features.values.size(), 0u); } else { - ASSERT_EQUAL(reply.match_features.names.size(), 5u); - EXPECT_EQUAL(reply.match_features.names[0], "attribute(a1)"); - EXPECT_EQUAL(reply.match_features.names[1], "attribute(a2)"); - EXPECT_EQUAL(reply.match_features.names[2], "matches(a1)"); - EXPECT_EQUAL(reply.match_features.names[3], "matches(f1)"); - EXPECT_EQUAL(reply.match_features.names[4], "rankingExpression(\"tensor(x[3])(x)\")"); - ASSERT_EQUAL(reply.match_features.values.size(), 5 * reply.hits.size()); + ASSERT_EQ(reply.match_features.names.size(), 5u); + EXPECT_EQ(reply.match_features.names[0], "attribute(a1)"); + EXPECT_EQ(reply.match_features.names[1], "attribute(a2)"); + EXPECT_EQ(reply.match_features.names[2], "matches(a1)"); + EXPECT_EQ(reply.match_features.names[3], "matches(f1)"); + EXPECT_EQ(reply.match_features.names[4], "rankingExpression(\"tensor(x[3])(x)\")"); + ASSERT_EQ(reply.match_features.values.size(), 5 * reply.hits.size()); for (size_t i = 0; i < reply.hits.size(); ++i) { const auto *f = &reply.match_features.values[i * 5]; - EXPECT_GREATER(f[0].as_double(), 0.0); - EXPECT_GREATER(f[1].as_double(), 0.0); - EXPECT_EQUAL(f[0].as_double(), reply.hits[i].metric); - EXPECT_EQUAL(f[0].as_double() * 2, f[1].as_double()); - EXPECT_EQUAL(f[2].as_double(), double(matched_field == "a1")); - EXPECT_EQUAL(f[3].as_double(), double(matched_field == "f1")); + EXPECT_GT(f[0].as_double(), 0.0); + EXPECT_GT(f[1].as_double(), 0.0); + EXPECT_EQ(f[0].as_double(), reply.hits[i].metric); + EXPECT_EQ(f[0].as_double() * 2, f[1].as_double()); + EXPECT_EQ(f[2].as_double(), double(matched_field == "a1")); + EXPECT_EQ(f[3].as_double(), double(matched_field == "f1")); EXPECT_TRUE(f[4].is_data()); { nbostream buf(f[4].as_data().data, f[4].as_data().size); auto actual = spec_from_value(*SimpleValue::from_stream(buf)); auto expect = TensorSpec("tensor(x[3])").add({{"x", 0}}, 0).add({{"x", 1}}, 1).add({{"x", 2}}, 2); - EXPECT_EQUAL(actual, expect); + EXPECT_EQ(actual, expect); } } } @@ -272,16 +322,16 @@ struct MyWorld { static void verify_match_feature_renames(SearchReply &reply, const vespalib::string &matched_field) { if (reply.hits.empty()) { - EXPECT_EQUAL(reply.match_features.names.size(), 0u); - EXPECT_EQUAL(reply.match_features.values.size(), 0u); + EXPECT_EQ(reply.match_features.names.size(), 0u); + EXPECT_EQ(reply.match_features.values.size(), 0u); } else { - ASSERT_EQUAL(reply.match_features.names.size(), 5u); - EXPECT_EQUAL(reply.match_features.names[3], "foobar"); - EXPECT_EQUAL(reply.match_features.names[4], "tensor(x[3])(x)"); - ASSERT_EQUAL(reply.match_features.values.size(), 5 * reply.hits.size()); + ASSERT_EQ(reply.match_features.names.size(), 5u); + EXPECT_EQ(reply.match_features.names[3], "foobar"); + EXPECT_EQ(reply.match_features.names[4], "tensor(x[3])(x)"); + ASSERT_EQ(reply.match_features.values.size(), 5 * reply.hits.size()); for (size_t i = 0; i < reply.hits.size(); ++i) { const auto *f = &reply.match_features.values[i * 5]; - EXPECT_EQUAL(f[3].as_double(), double(matched_field == "f1")); + EXPECT_EQ(f[3].as_double(), double(matched_field == "f1")); EXPECT_TRUE(f[4].is_data()); } } @@ -378,7 +428,7 @@ struct MyWorld { auto mtf = matcher->create_match_tools_factory(req, searchContext, attributeContext, metaStore, overrides, ttb(), nullptr, searchContext.getDocIdLimit(), true); auto diversity = mtf->createDiversifier(HeapSize::lookup(config)); - EXPECT_EQUAL(expectDiverse, static_cast<bool>(diversity)); + EXPECT_EQ(expectDiverse, static_cast<bool>(diversity)); } double get_first_phase_termwise_limit() { @@ -397,7 +447,8 @@ struct MyWorld { SearchSession::OwnershipBundle owned_objects({std::make_unique<MockAttributeContext>(), std::make_unique<FakeSearchContext>()}, std::make_shared<MySearchHandler>(matcher)); - vespalib::SimpleThreadBundle threadBundle(threads); + assert(threads <= MatchingTestSharedState::max_threads); + vespalib::LimitedThreadBundleWrapper threadBundle(shared_state.thread_bundle(), threads); SearchReply::UP reply = matcher->match(req, threadBundle, searchContext, attributeContext, *sessionManager, metaStore, metaStore.getBucketDB(), std::move(owned_objects)); @@ -449,13 +500,14 @@ struct MyWorld { } }; -MyWorld::MyWorld() - : schema(), +MyWorld::MyWorld(MatchingTestSharedState& shared_state_in) + : shared_state(shared_state_in), + schema(), config(), searchContext(), - attributeContext(), + attributeContext(shared_state.attribute_context()), sessionManager(), - metaStore(std::make_shared<bucketdb::BucketDBOwner>()), + metaStore(shared_state.meta_store()), matchingStats(), clock(), queryLimiter() @@ -469,30 +521,66 @@ void verifyViewResolver(const ViewResolver &resolver) { std::vector<vespalib::string> fields; EXPECT_TRUE(resolver.resolve("foo", fields)); ASSERT_TRUE(fields.size() == 2u); - EXPECT_EQUAL("x", fields[0]); - EXPECT_EQUAL("y", fields[1]); + EXPECT_EQ("x", fields[0]); + EXPECT_EQ("y", fields[1]); } { std::vector<vespalib::string> fields; EXPECT_TRUE(resolver.resolve("bar", fields)); ASSERT_TRUE(fields.size() == 1u); - EXPECT_EQUAL("z", fields[0]); + EXPECT_EQ("z", fields[0]); } { std::vector<vespalib::string> fields; EXPECT_TRUE(!resolver.resolve("baz", fields)); ASSERT_TRUE(fields.size() == 1u); - EXPECT_EQUAL("baz", fields[0]); + EXPECT_EQ("baz", fields[0]); } } -TEST("require that view resolver can be set up directly") { +class MatchingTest : public ::testing::Test { + static std::unique_ptr<MatchingTestSharedState> _shared_state; +protected: + MatchingTest(); + ~MatchingTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); + static MatchingTestSharedState& shared_state(); +}; + +MatchingTest::MatchingTest() = default; + +MatchingTest::~MatchingTest() = default; + +void +MatchingTest::SetUpTestSuite() +{ + _shared_state = std::make_unique<MatchingTestSharedState>(); +} + +void +MatchingTest::TearDownTestSuite() +{ + _shared_state.reset(); +} + +MatchingTestSharedState& +MatchingTest::shared_state() +{ + return *_shared_state; +} + +std::unique_ptr<MatchingTestSharedState> MatchingTest::_shared_state; + +TEST_F(MatchingTest, require_that_view_resolver_can_be_set_up_directly) +{ ViewResolver resolver; resolver.add("foo", "x").add("foo", "y").add("bar", "z"); - TEST_DO(verifyViewResolver(resolver)); + verifyViewResolver(resolver); } -TEST("require that view resolver can be set up from schema") { +TEST_F(MatchingTest, require_that_view_resolver_can_be_set_up_from_schema) +{ Schema schema; Schema::FieldSet foo("foo"); foo.addField("x").addField("y"); @@ -501,124 +589,132 @@ TEST("require that view resolver can be set up from schema") { schema.addFieldSet(foo); schema.addFieldSet(bar); ViewResolver resolver = ViewResolver::createFromSchema(schema); - TEST_DO(verifyViewResolver(resolver)); + verifyViewResolver(resolver); } //----------------------------------------------------------------------------- -TEST("require that matching is performed (multi-threaded)") { +TEST_F(MatchingTest, require_that_matching_is_performed_with_multi_threaded_matcher) +{ for (size_t threads = 1; threads <= 16; ++threads) { - MyWorld world; + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "spread"); SearchReply::UP reply = world.performSearch(*request, threads); - EXPECT_EQUAL(9u, world.matchingStats.docsMatched()); - EXPECT_EQUAL(9u, reply->hits.size()); - EXPECT_GREATER(world.matchingStats.matchTimeAvg(), 0.0000001); + EXPECT_EQ(9u, world.matchingStats.docsMatched()); + EXPECT_EQ(9u, reply->hits.size()); + EXPECT_GT(world.matchingStats.matchTimeAvg(), 0.0000001); } } -TEST("require that match features are calculated (multi-threaded)") { +TEST_F(MatchingTest, require_that_match_features_are_calculated_with_multi_threaded_matcher) +{ for (size_t threads = 1; threads <= 16; ++threads) { - MyWorld world; + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); world.setup_match_features(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "spread"); SearchReply::UP reply = world.performSearch(*request, threads); - EXPECT_GREATER(reply->hits.size(), 0u); + EXPECT_GT(reply->hits.size(), 0u); MyWorld::verify_match_features(*reply, "f1"); } } -TEST("require that match features can be renamed") { - MyWorld world; +TEST_F(MatchingTest, require_that_match_features_can_be_renamed) +{ + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); world.setup_match_features(); world.setup_feature_renames(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "spread"); SearchReply::UP reply = world.performSearch(*request, 1); - EXPECT_GREATER(reply->hits.size(), 0u); + EXPECT_GT(reply->hits.size(), 0u); MyWorld::verify_match_feature_renames(*reply, "f1"); } -TEST("require that no hits gives no match feature names") { - MyWorld world; +TEST_F(MatchingTest, require_that_no_hits_gives_no_match_feature_names) + { + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); world.setup_match_features(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "not_found"); SearchReply::UP reply = world.performSearch(*request, 1); - EXPECT_EQUAL(reply->hits.size(), 0u); + EXPECT_EQ(reply->hits.size(), 0u); MyWorld::verify_match_features(*reply, "f1"); } -TEST("require that matching also returns hits when only bitvector is used (multi-threaded)") { +TEST_F(MatchingTest, require_that_matching_also_returns_hits_when_only_bitvector_is_used_with_multi_threaded_matcher) + { for (size_t threads = 1; threads <= 16; ++threads) { - MyWorld world; + MyWorld world(shared_state()); world.basicSetup(0, 0); world.verbose_a1_result("all"); SearchRequest::SP request = MyWorld::createSimpleRequest("a1", "all"); SearchReply::UP reply = world.performSearch(*request, threads); - EXPECT_EQUAL(985u, world.matchingStats.docsMatched()); - EXPECT_EQUAL(10u, reply->hits.size()); - EXPECT_GREATER(world.matchingStats.matchTimeAvg(), 0.0000001); + EXPECT_EQ(985u, world.matchingStats.docsMatched()); + EXPECT_EQ(10u, reply->hits.size()); + EXPECT_GT(world.matchingStats.matchTimeAvg(), 0.0000001); } } -TEST("require that ranking is performed (multi-threaded)") { +TEST_F(MatchingTest, require_that_ranking_is_performed_with_multi_threaded_matcher) + { for (size_t threads = 1; threads <= 16; ++threads) { - MyWorld world; + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "spread"); SearchReply::UP reply = world.performSearch(*request, threads); - EXPECT_EQUAL(9u, world.matchingStats.docsMatched()); - EXPECT_EQUAL(9u, world.matchingStats.docsRanked()); - EXPECT_EQUAL(0u, world.matchingStats.docsReRanked()); + EXPECT_EQ(9u, world.matchingStats.docsMatched()); + EXPECT_EQ(9u, world.matchingStats.docsRanked()); + EXPECT_EQ(0u, world.matchingStats.docsReRanked()); ASSERT_TRUE(reply->hits.size() == 9u); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::900").getGlobalId(), reply->hits[0].gid); - EXPECT_EQUAL(900.0, reply->hits[0].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::800").getGlobalId(), reply->hits[1].gid); - EXPECT_EQUAL(800.0, reply->hits[1].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::700").getGlobalId(), reply->hits[2].gid); - EXPECT_EQUAL(700.0, reply->hits[2].metric); - EXPECT_GREATER(world.matchingStats.matchTimeAvg(), 0.0000001); - EXPECT_EQUAL(0.0, world.matchingStats.rerankTimeAvg()); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::900").getGlobalId(), reply->hits[0].gid); + EXPECT_EQ(900.0, reply->hits[0].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::800").getGlobalId(), reply->hits[1].gid); + EXPECT_EQ(800.0, reply->hits[1].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::700").getGlobalId(), reply->hits[2].gid); + EXPECT_EQ(700.0, reply->hits[2].metric); + EXPECT_GT(world.matchingStats.matchTimeAvg(), 0.0000001); + EXPECT_EQ(0.0, world.matchingStats.rerankTimeAvg()); } } -TEST("require that re-ranking is performed (multi-threaded)") { +TEST_F(MatchingTest, require_that_reranking_is_performed_with_multi_threaded_matcher) + { for (size_t threads = 1; threads <= 16; ++threads) { - MyWorld world; + MyWorld world(shared_state()); world.basicSetup(); world.setupSecondPhaseRanking(); world.basicResults(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "spread"); SearchReply::UP reply = world.performSearch(*request, threads); - EXPECT_EQUAL(9u, world.matchingStats.docsMatched()); - EXPECT_EQUAL(9u, world.matchingStats.docsRanked()); - EXPECT_EQUAL(3u, world.matchingStats.docsReRanked()); + EXPECT_EQ(9u, world.matchingStats.docsMatched()); + EXPECT_EQ(9u, world.matchingStats.docsRanked()); + EXPECT_EQ(3u, world.matchingStats.docsReRanked()); ASSERT_TRUE(reply->hits.size() == 9u); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::900").getGlobalId(), reply->hits[0].gid); - EXPECT_EQUAL(1800.0, reply->hits[0].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::800").getGlobalId(), reply->hits[1].gid); - EXPECT_EQUAL(1600.0, reply->hits[1].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::700").getGlobalId(), reply->hits[2].gid); - EXPECT_EQUAL(1400.0, reply->hits[2].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::600").getGlobalId(), reply->hits[3].gid); - EXPECT_EQUAL(600.0, reply->hits[3].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::500").getGlobalId(), reply->hits[4].gid); - EXPECT_EQUAL(500.0, reply->hits[4].metric); - EXPECT_GREATER(world.matchingStats.matchTimeAvg(), 0.0000001); - EXPECT_GREATER(world.matchingStats.rerankTimeAvg(), 0.0000001); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::900").getGlobalId(), reply->hits[0].gid); + EXPECT_EQ(1800.0, reply->hits[0].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::800").getGlobalId(), reply->hits[1].gid); + EXPECT_EQ(1600.0, reply->hits[1].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::700").getGlobalId(), reply->hits[2].gid); + EXPECT_EQ(1400.0, reply->hits[2].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::600").getGlobalId(), reply->hits[3].gid); + EXPECT_EQ(600.0, reply->hits[3].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::500").getGlobalId(), reply->hits[4].gid); + EXPECT_EQ(500.0, reply->hits[4].metric); + EXPECT_GT(world.matchingStats.matchTimeAvg(), 0.0000001); + EXPECT_GT(world.matchingStats.rerankTimeAvg(), 0.0000001); } } -TEST("require that re-ranking is not diverse when not requested to be.") { - MyWorld world; +TEST_F(MatchingTest, require_that_reranking_is_not_diverse_when_not_requested_to_be) +{ + MyWorld world(shared_state()); world.basicSetup(); world.setupSecondPhaseRanking(); world.basicResults(); @@ -628,8 +724,9 @@ TEST("require that re-ranking is not diverse when not requested to be.") { using namespace search::fef::indexproperties::matchphase; -TEST("require that re-ranking is diverse when requested to be") { - MyWorld world; +TEST_F(MatchingTest, require_that_reranking_is_diverse_when_requested_to_be) +{ + MyWorld world(shared_state()); world.basicSetup(); world.setupSecondPhaseRanking(); world.basicResults(); @@ -641,8 +738,9 @@ TEST("require that re-ranking is diverse when requested to be") { world.verify_diversity_filter(*request, true); } -TEST("require that re-ranking is diverse with diversity = 1/1") { - MyWorld world; +TEST_F(MatchingTest, require_that_reranking_is_diverse_with_diversity_1_of_1) +{ + MyWorld world(shared_state()); world.basicSetup(); world.setupSecondPhaseRanking(); world.basicResults(); @@ -652,24 +750,25 @@ TEST("require that re-ranking is diverse with diversity = 1/1") { .add(DiversityMinGroups::NAME, "3") .add(DiversityCutoffStrategy::NAME, "strict"); SearchReply::UP reply = world.performSearch(*request, 1); - EXPECT_EQUAL(9u, world.matchingStats.docsMatched()); - EXPECT_EQUAL(9u, world.matchingStats.docsRanked()); - EXPECT_EQUAL(3u, world.matchingStats.docsReRanked()); + EXPECT_EQ(9u, world.matchingStats.docsMatched()); + EXPECT_EQ(9u, world.matchingStats.docsRanked()); + EXPECT_EQ(3u, world.matchingStats.docsReRanked()); ASSERT_TRUE(reply->hits.size() == 9u); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::900").getGlobalId(), reply->hits[0].gid); - EXPECT_EQUAL(1800.0, reply->hits[0].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::800").getGlobalId(), reply->hits[1].gid); - EXPECT_EQUAL(1600.0, reply->hits[1].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::700").getGlobalId(), reply->hits[2].gid); - EXPECT_EQUAL(1400.0, reply->hits[2].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::600").getGlobalId(), reply->hits[3].gid); - EXPECT_EQUAL(600.0, reply->hits[3].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::500").getGlobalId(), reply->hits[4].gid); - EXPECT_EQUAL(500.0, reply->hits[4].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::900").getGlobalId(), reply->hits[0].gid); + EXPECT_EQ(1800.0, reply->hits[0].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::800").getGlobalId(), reply->hits[1].gid); + EXPECT_EQ(1600.0, reply->hits[1].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::700").getGlobalId(), reply->hits[2].gid); + EXPECT_EQ(1400.0, reply->hits[2].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::600").getGlobalId(), reply->hits[3].gid); + EXPECT_EQ(600.0, reply->hits[3].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::500").getGlobalId(), reply->hits[4].gid); + EXPECT_EQ(500.0, reply->hits[4].metric); } -TEST("require that re-ranking is diverse with diversity = 1/10") { - MyWorld world; +TEST_F(MatchingTest, require_that_reranking_is_diverse_with_diversity_1_of_10) + { + MyWorld world(shared_state()); world.basicSetup(); world.setupSecondPhaseRanking(); world.basicResults(); @@ -679,47 +778,50 @@ TEST("require that re-ranking is diverse with diversity = 1/10") { .add(DiversityMinGroups::NAME, "3") .add(DiversityCutoffStrategy::NAME, "strict"); SearchReply::UP reply = world.performSearch(*request, 1); - EXPECT_EQUAL(9u, world.matchingStats.docsMatched()); - EXPECT_EQUAL(9u, world.matchingStats.docsRanked()); - EXPECT_EQUAL(1u, world.matchingStats.docsReRanked()); + EXPECT_EQ(9u, world.matchingStats.docsMatched()); + EXPECT_EQ(9u, world.matchingStats.docsRanked()); + EXPECT_EQ(1u, world.matchingStats.docsReRanked()); ASSERT_TRUE(reply->hits.size() == 9u); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::900").getGlobalId(), reply->hits[0].gid); - EXPECT_EQUAL(1800.0, reply->hits[0].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::900").getGlobalId(), reply->hits[0].gid); + EXPECT_EQ(1800.0, reply->hits[0].metric); //TODO This is of course incorrect until the selectBest method sees everything. - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::800").getGlobalId(), reply->hits[1].gid); - EXPECT_EQUAL(800.0, reply->hits[1].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::700").getGlobalId(), reply->hits[2].gid); - EXPECT_EQUAL(700.0, reply->hits[2].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::600").getGlobalId(), reply->hits[3].gid); - EXPECT_EQUAL(600.0, reply->hits[3].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::500").getGlobalId(), reply->hits[4].gid); - EXPECT_EQUAL(500.0, reply->hits[4].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::800").getGlobalId(), reply->hits[1].gid); + EXPECT_EQ(800.0, reply->hits[1].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::700").getGlobalId(), reply->hits[2].gid); + EXPECT_EQ(700.0, reply->hits[2].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::600").getGlobalId(), reply->hits[3].gid); + EXPECT_EQ(600.0, reply->hits[3].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::500").getGlobalId(), reply->hits[4].gid); + EXPECT_EQ(500.0, reply->hits[4].metric); } -TEST("require that sortspec can be used (multi-threaded)") { +TEST_F(MatchingTest, require_that_sortspec_can_be_used_with_multi_threaded_matcher) +{ for (size_t threads = 1; threads <= 16; ++threads) { - MyWorld world; + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "spread"); request->sortSpec = "+a1"; SearchReply::UP reply = world.performSearch(*request, threads); - ASSERT_EQUAL(9u, reply->hits.size()); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::100").getGlobalId(), reply->hits[0].gid); - EXPECT_EQUAL(zero_rank_value, reply->hits[0].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::200").getGlobalId(), reply->hits[1].gid); - EXPECT_EQUAL(zero_rank_value, reply->hits[1].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::300").getGlobalId(), reply->hits[2].gid); - EXPECT_EQUAL(zero_rank_value, reply->hits[2].metric); + ASSERT_EQ(9u, reply->hits.size()); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::100").getGlobalId(), reply->hits[0].gid); + EXPECT_EQ(zero_rank_value, reply->hits[0].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::200").getGlobalId(), reply->hits[1].gid); + EXPECT_EQ(zero_rank_value, reply->hits[1].metric); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::300").getGlobalId(), reply->hits[2].gid); + EXPECT_EQ(zero_rank_value, reply->hits[2].metric); EXPECT_FALSE(reply->sortIndex.empty()); EXPECT_FALSE(reply->sortData.empty()); } } ExpressionNode::UP createAttr() { return std::make_unique<AttributeNode>("a1"); } -TEST("require that grouping is performed (multi-threaded)") { + +TEST_F(MatchingTest, require_that_grouping_is_performed_with_multi_threaded_matcher) + { for (size_t threads = 1; threads <= 16; ++threads) { - MyWorld world; + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "spread"); @@ -739,120 +841,125 @@ TEST("require that grouping is performed (multi-threaded)") { vespalib::NBOSerializer is(buf); uint32_t n; is >> n; - EXPECT_EQUAL(1u, n); + EXPECT_EQ(1u, n); Grouping gresult; gresult.deserialize(is); Grouping gexpect; gexpect.setRoot(Group().addResult(SumAggregationResult() .setExpression(createAttr()) .setResult(Int64ResultNode(4500)))); - EXPECT_EQUAL(gexpect.root().asString(), gresult.root().asString()); + EXPECT_EQ(gexpect.root().asString(), gresult.root().asString()); } - EXPECT_GREATER(world.matchingStats.groupingTimeAvg(), 0.0000001); + EXPECT_GT(world.matchingStats.groupingTimeAvg(), 0.0000001); } } -TEST("require that summary features are filled") { - MyWorld world; +TEST_F(MatchingTest, require_that_summary_features_are_filled) +{ + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); DocsumRequest::SP req = MyWorld::createSimpleDocsumRequest("f1", "foo"); FeatureSet::SP fs = world.getSummaryFeatures(*req); const FeatureSet::Value * f = nullptr; - EXPECT_EQUAL(5u, fs->numFeatures()); - EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]); - EXPECT_EQUAL("matches(f1)", fs->getNames()[1]); - EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[2]); - EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[3]); - EXPECT_EQUAL("value(100)", fs->getNames()[4]); - EXPECT_EQUAL(3u, fs->numDocs()); + EXPECT_EQ(5u, fs->numFeatures()); + EXPECT_EQ("attribute(a1)", fs->getNames()[0]); + EXPECT_EQ("matches(f1)", fs->getNames()[1]); + EXPECT_EQ("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[2]); + EXPECT_EQ("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[3]); + EXPECT_EQ("value(100)", fs->getNames()[4]); + EXPECT_EQ(3u, fs->numDocs()); f = fs->getFeaturesByDocId(10); EXPECT_TRUE(f != nullptr); - EXPECT_EQUAL(10, f[0].as_double()); - EXPECT_EQUAL(1, f[1].as_double()); - EXPECT_EQUAL(100, f[4].as_double()); + EXPECT_EQ(10, f[0].as_double()); + EXPECT_EQ(1, f[1].as_double()); + EXPECT_EQ(100, f[4].as_double()); f = fs->getFeaturesByDocId(15); EXPECT_TRUE(f != nullptr); - EXPECT_EQUAL(15, f[0].as_double()); - EXPECT_EQUAL(0, f[1].as_double()); - EXPECT_EQUAL(100, f[4].as_double()); + EXPECT_EQ(15, f[0].as_double()); + EXPECT_EQ(0, f[1].as_double()); + EXPECT_EQ(100, f[4].as_double()); f = fs->getFeaturesByDocId(30); EXPECT_TRUE(f != nullptr); - EXPECT_EQUAL(30, f[0].as_double()); - EXPECT_EQUAL(1, f[1].as_double()); + EXPECT_EQ(30, f[0].as_double()); + EXPECT_EQ(1, f[1].as_double()); EXPECT_TRUE(f[2].is_double()); EXPECT_TRUE(!f[2].is_data()); - EXPECT_EQUAL(f[2].as_double(), 3.0); // 0 + 1 + 2 + EXPECT_EQ(f[2].as_double(), 3.0); // 0 + 1 + 2 EXPECT_TRUE(!f[3].is_double()); EXPECT_TRUE(f[3].is_data()); - EXPECT_EQUAL(100, f[4].as_double()); + EXPECT_EQ(100, f[4].as_double()); { nbostream buf(f[3].as_data().data, f[3].as_data().size); auto actual = spec_from_value(*SimpleValue::from_stream(buf)); auto expect = TensorSpec("tensor(x[3])").add({{"x", 0}}, 0).add({{"x", 1}}, 1).add({{"x", 2}}, 2); - EXPECT_EQUAL(actual, expect); + EXPECT_EQ(actual, expect); } } -TEST("require that rank features are filled") { - MyWorld world; +TEST_F(MatchingTest, require_that_rank_features_are_filled) +{ + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); DocsumRequest::SP req = MyWorld::createSimpleDocsumRequest("f1", "foo"); FeatureSet::SP fs = world.getRankFeatures(*req); const FeatureSet::Value * f = nullptr; - EXPECT_EQUAL(1u, fs->numFeatures()); - EXPECT_EQUAL("attribute(a2)", fs->getNames()[0]); - EXPECT_EQUAL(3u, fs->numDocs()); + EXPECT_EQ(1u, fs->numFeatures()); + EXPECT_EQ("attribute(a2)", fs->getNames()[0]); + EXPECT_EQ(3u, fs->numDocs()); f = fs->getFeaturesByDocId(10); EXPECT_TRUE(f != nullptr); - EXPECT_EQUAL(20, f[0].as_double()); + EXPECT_EQ(20, f[0].as_double()); f = fs->getFeaturesByDocId(15); EXPECT_TRUE(f != nullptr); - EXPECT_EQUAL(30, f[0].as_double()); + EXPECT_EQ(30, f[0].as_double()); f = fs->getFeaturesByDocId(30); EXPECT_TRUE(f != nullptr); - EXPECT_EQUAL(60, f[0].as_double()); + EXPECT_EQ(60, f[0].as_double()); } -TEST("require that search session can be cached") { - MyWorld world; +TEST_F(MatchingTest, require_that_search_session_can_be_cached) +{ + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "foo"); request->propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true"); request->sessionId.push_back('a'); - EXPECT_EQUAL(0u, world.sessionManager->getSearchStats().numInsert); + EXPECT_EQ(0u, world.sessionManager->getSearchStats().numInsert); SearchReply::UP reply = world.performSearch(*request, 1); - EXPECT_EQUAL(1u, world.sessionManager->getSearchStats().numInsert); + EXPECT_EQ(1u, world.sessionManager->getSearchStats().numInsert); SearchSession::SP session = world.sessionManager->pickSearch("a"); ASSERT_TRUE(session.get()); - EXPECT_EQUAL(request->getTimeOfDoom(), session->getTimeOfDoom()); - EXPECT_EQUAL("a", session->getSessionId()); + EXPECT_EQ(request->getTimeOfDoom(), session->getTimeOfDoom()); + EXPECT_EQ("a", session->getSessionId()); } -TEST("require that summary features can be renamed") { - MyWorld world; +TEST_F(MatchingTest, require_that_summary_features_can_be_renamed) +{ + MyWorld world(shared_state()); world.basicSetup(); world.setup_feature_renames(); world.basicResults(); DocsumRequest::SP req = MyWorld::createSimpleDocsumRequest("f1", "foo"); FeatureSet::SP fs = world.getSummaryFeatures(*req); const FeatureSet::Value * f = nullptr; - EXPECT_EQUAL(5u, fs->numFeatures()); - EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]); - EXPECT_EQUAL("foobar", fs->getNames()[1]); - EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[2]); - EXPECT_EQUAL("tensor(x[3])(x)", fs->getNames()[3]); - EXPECT_EQUAL(3u, fs->numDocs()); + EXPECT_EQ(5u, fs->numFeatures()); + EXPECT_EQ("attribute(a1)", fs->getNames()[0]); + EXPECT_EQ("foobar", fs->getNames()[1]); + EXPECT_EQ("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[2]); + EXPECT_EQ("tensor(x[3])(x)", fs->getNames()[3]); + EXPECT_EQ(3u, fs->numDocs()); f = fs->getFeaturesByDocId(30); EXPECT_TRUE(f != nullptr); EXPECT_TRUE(f[2].is_double()); EXPECT_TRUE(f[3].is_data()); } -TEST("require that getSummaryFeatures can use cached query setup") { - MyWorld world; +TEST_F(MatchingTest, require_that_getSummaryFeatures_can_use_cached_query_setup) +{ + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "foo"); @@ -867,46 +974,46 @@ TEST("require that getSummaryFeatures can use cached query setup") { docsum_request->hits.back().docid = 30; FeatureSet::SP fs = world.getSummaryFeatures(*docsum_request); - ASSERT_EQUAL(5u, fs->numFeatures()); - EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]); - EXPECT_EQUAL("matches(f1)", fs->getNames()[1]); - EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[2]); - EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[3]); - EXPECT_EQUAL("value(100)", fs->getNames()[4]); - ASSERT_EQUAL(1u, fs->numDocs()); + ASSERT_EQ(5u, fs->numFeatures()); + EXPECT_EQ("attribute(a1)", fs->getNames()[0]); + EXPECT_EQ("matches(f1)", fs->getNames()[1]); + EXPECT_EQ("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[2]); + EXPECT_EQ("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[3]); + EXPECT_EQ("value(100)", fs->getNames()[4]); + ASSERT_EQ(1u, fs->numDocs()); const auto *f = fs->getFeaturesByDocId(30); ASSERT_TRUE(f); - EXPECT_EQUAL(30, f[0].as_double()); - EXPECT_EQUAL(100, f[4].as_double()); + EXPECT_EQ(30, f[0].as_double()); + EXPECT_EQ(100, f[4].as_double()); // getSummaryFeatures can be called multiple times. fs = world.getSummaryFeatures(*docsum_request); - ASSERT_EQUAL(5u, fs->numFeatures()); - EXPECT_EQUAL("attribute(a1)", fs->getNames()[0]); - EXPECT_EQUAL("matches(f1)", fs->getNames()[1]); - EXPECT_EQUAL("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[2]); - EXPECT_EQUAL("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[3]); - EXPECT_EQUAL("value(100)", fs->getNames()[4]); - ASSERT_EQUAL(1u, fs->numDocs()); + ASSERT_EQ(5u, fs->numFeatures()); + EXPECT_EQ("attribute(a1)", fs->getNames()[0]); + EXPECT_EQ("matches(f1)", fs->getNames()[1]); + EXPECT_EQ("rankingExpression(\"reduce(tensor(x[3])(x),sum)\")", fs->getNames()[2]); + EXPECT_EQ("rankingExpression(\"tensor(x[3])(x)\")", fs->getNames()[3]); + EXPECT_EQ("value(100)", fs->getNames()[4]); + ASSERT_EQ(1u, fs->numDocs()); f = fs->getFeaturesByDocId(30); ASSERT_TRUE(f); - EXPECT_EQUAL(30, f[0].as_double()); - EXPECT_EQUAL(100, f[4].as_double()); + EXPECT_EQ(30, f[0].as_double()); + EXPECT_EQ(100, f[4].as_double()); } -double count_f1_matches(FeatureSet &fs) { +void count_f1_matches(FeatureSet &fs, double& sum) { ASSERT_TRUE(fs.getNames().size() > 1); - ASSERT_EQUAL(fs.getNames()[1], "matches(f1)"); - double sum = 0.0; + ASSERT_EQ(fs.getNames()[1], "matches(f1)"); + sum = 0.0; for (size_t i = 0; i < fs.numDocs(); ++i) { auto *f = fs.getFeaturesByIndex(i); sum += f[1].as_double(); } - return sum; } -TEST("require that getSummaryFeatures prefers cached query setup") { - MyWorld world; +TEST_F(MatchingTest, require_that_getSummaryFeatures_prefers_cached_query_setup) +{ + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); SearchRequest::SP request = MyWorld::createSimpleRequest("f1", "spread"); @@ -918,94 +1025,106 @@ TEST("require that getSummaryFeatures prefers cached query setup") { req->sessionId = request->sessionId; req->propertiesMap.lookupCreate(search::MapNames::CACHES).add("query", "true"); FeatureSet::SP fs = world.getSummaryFeatures(*req); - EXPECT_EQUAL(5u, fs->numFeatures()); - EXPECT_EQUAL(3u, fs->numDocs()); - EXPECT_EQUAL(0.0, count_f1_matches(*fs)); // "spread" has no hits + EXPECT_EQ(5u, fs->numFeatures()); + EXPECT_EQ(3u, fs->numDocs()); + double sum = 0.0; + ASSERT_NO_FATAL_FAILURE(count_f1_matches(*fs, sum)); + EXPECT_EQ(0.0, sum); // "spread" has no hits // Empty cache auto pruneTime = vespalib::steady_clock::now() + 600s; world.sessionManager->pruneTimedOutSessions(pruneTime); fs = world.getSummaryFeatures(*req); - EXPECT_EQUAL(5u, fs->numFeatures()); - EXPECT_EQUAL(3u, fs->numDocs()); - EXPECT_EQUAL(2.0, count_f1_matches(*fs)); // "foo" has two hits + EXPECT_EQ(5u, fs->numFeatures()); + EXPECT_EQ(3u, fs->numDocs()); + ASSERT_NO_FATAL_FAILURE(count_f1_matches(*fs, sum)); + EXPECT_EQ(2.0, sum); // "foo" has two hits } -TEST("require that match params are set up straight with ranking on") { - MatchParams p(10, 2, 4, 0.7, 0, 1, true, true); - ASSERT_EQUAL(10u, p.numDocs); - ASSERT_EQUAL(2u, p.heapSize); - ASSERT_EQUAL(4u, p.arraySize); - ASSERT_EQUAL(0.7, p.rankDropLimit); - ASSERT_EQUAL(0u, p.offset); - ASSERT_EQUAL(1u, p.hits); - ASSERT_TRUE(p.has_rank_drop_limit()); +TEST_F(MatchingTest, require_that_match_params_are_set_up_straight_with_ranking_on) +{ + MatchParams p(10, 2, 4, 0.7, 0.75, 0, 1, true, true); + ASSERT_EQ(10u, p.numDocs); + ASSERT_EQ(2u, p.heapSize); + ASSERT_EQ(4u, p.arraySize); + ASSERT_EQ(0.7, p.first_phase_rank_score_drop_limit.value()); + ASSERT_EQ(0.75, p.second_phase_rank_score_drop_limit.value()); + ASSERT_EQ(0u, p.offset); + ASSERT_EQ(1u, p.hits); } -TEST("require that match params can turn off rank-drop-limit") { - MatchParams p(10, 2, 4, -std::numeric_limits<feature_t>::quiet_NaN(), 0, 1, true, true); - ASSERT_EQUAL(10u, p.numDocs); - ASSERT_EQUAL(2u, p.heapSize); - ASSERT_EQUAL(4u, p.arraySize); - ASSERT_TRUE(std::isnan(p.rankDropLimit)); - ASSERT_EQUAL(0u, p.offset); - ASSERT_EQUAL(1u, p.hits); - ASSERT_FALSE(p.has_rank_drop_limit()); +TEST_F(MatchingTest, require_that_match_params_can_turn_off_rank_score_drop_limits) +{ + MatchParams p(10, 2, 4, std::nullopt, std::nullopt, 0, 1, true, true); + ASSERT_EQ(10u, p.numDocs); + ASSERT_EQ(2u, p.heapSize); + ASSERT_EQ(4u, p.arraySize); + ASSERT_FALSE(p.first_phase_rank_score_drop_limit.has_value()); + ASSERT_FALSE(p.second_phase_rank_score_drop_limit.has_value()); + ASSERT_EQ(0u, p.offset); + ASSERT_EQ(1u, p.hits); } -TEST("require that match params are set up straight with ranking on arraySize is atleast the size of heapSize") { - MatchParams p(10, 6, 4, 0.7, 1, 1, true, true); - ASSERT_EQUAL(10u, p.numDocs); - ASSERT_EQUAL(6u, p.heapSize); - ASSERT_EQUAL(6u, p.arraySize); - ASSERT_EQUAL(0.7, p.rankDropLimit); - ASSERT_EQUAL(1u, p.offset); - ASSERT_EQUAL(1u, p.hits); +TEST_F(MatchingTest, require_that_match_params_are_set_up_straight_with_ranking_on_arraySize_is_atleast_the_size_of_heapSize) +{ + MatchParams p(10, 6, 4, 0.7, std::nullopt, 1, 1, true, true); + ASSERT_EQ(10u, p.numDocs); + ASSERT_EQ(6u, p.heapSize); + ASSERT_EQ(6u, p.arraySize); + ASSERT_EQ(0.7, p.first_phase_rank_score_drop_limit.value()); + ASSERT_FALSE(p.second_phase_rank_score_drop_limit.has_value()); + ASSERT_EQ(1u, p.offset); + ASSERT_EQ(1u, p.hits); } -TEST("require that match params are set up straight with ranking on arraySize is atleast the size of hits+offset") { - MatchParams p(10, 6, 4, 0.7, 4, 4, true, true); - ASSERT_EQUAL(10u, p.numDocs); - ASSERT_EQUAL(6u, p.heapSize); - ASSERT_EQUAL(8u, p.arraySize); - ASSERT_EQUAL(0.7, p.rankDropLimit); - ASSERT_EQUAL(4u, p.offset); - ASSERT_EQUAL(4u, p.hits); +TEST_F(MatchingTest, require_that_match_params_are_set_up_straight_with_ranking_on_arraySize_is_atleast_the_size_of_hits_plus_offset) +{ + MatchParams p(10, 6, 4, 0.7, std::nullopt, 4, 4, true, true); + ASSERT_EQ(10u, p.numDocs); + ASSERT_EQ(6u, p.heapSize); + ASSERT_EQ(8u, p.arraySize); + ASSERT_EQ(0.7, p.first_phase_rank_score_drop_limit.value()); + ASSERT_EQ(4u, p.offset); + ASSERT_EQ(4u, p.hits); } -TEST("require that match params are capped by numDocs") { - MatchParams p(1, 6, 4, 0.7, 4, 4, true, true); - ASSERT_EQUAL(1u, p.numDocs); - ASSERT_EQUAL(1u, p.heapSize); - ASSERT_EQUAL(1u, p.arraySize); - ASSERT_EQUAL(0.7, p.rankDropLimit); - ASSERT_EQUAL(1u, p.offset); - ASSERT_EQUAL(0u, p.hits); +TEST_F(MatchingTest, require_that_match_params_are_capped_by_numDocs) +{ + MatchParams p(1, 6, 4, 0.7, std::nullopt, 4, 4, true, true); + ASSERT_EQ(1u, p.numDocs); + ASSERT_EQ(1u, p.heapSize); + ASSERT_EQ(1u, p.arraySize); + ASSERT_EQ(0.7, p.first_phase_rank_score_drop_limit.value()); + ASSERT_EQ(1u, p.offset); + ASSERT_EQ(0u, p.hits); } -TEST("require that match params are capped by numDocs and hits adjusted down") { - MatchParams p(5, 6, 4, 0.7, 4, 4, true, true); - ASSERT_EQUAL(5u, p.numDocs); - ASSERT_EQUAL(5u, p.heapSize); - ASSERT_EQUAL(5u, p.arraySize); - ASSERT_EQUAL(0.7, p.rankDropLimit); - ASSERT_EQUAL(4u, p.offset); - ASSERT_EQUAL(1u, p.hits); +TEST_F(MatchingTest, require_that_match_params_are_capped_by_numDocs_and_hits_adjusted_down) +{ + MatchParams p(5, 6, 4, 0.7, std::nullopt, 4, 4, true, true); + ASSERT_EQ(5u, p.numDocs); + ASSERT_EQ(5u, p.heapSize); + ASSERT_EQ(5u, p.arraySize); + ASSERT_EQ(0.7, p.first_phase_rank_score_drop_limit.value()); + ASSERT_EQ(4u, p.offset); + ASSERT_EQ(1u, p.hits); } -TEST("require that match params are set up straight with ranking off array and heap size is 0") { - MatchParams p(10, 6, 4, 0.7, 4, 4, true, false); - ASSERT_EQUAL(10u, p.numDocs); - ASSERT_EQUAL(0u, p.heapSize); - ASSERT_EQUAL(0u, p.arraySize); - ASSERT_EQUAL(0.7, p.rankDropLimit); - ASSERT_EQUAL(4u, p.offset); - ASSERT_EQUAL(4u, p.hits); +TEST_F(MatchingTest, require_that_match_params_are_set_up_straight_with_ranking_off_array_and_heap_size_is_0) +{ + MatchParams p(10, 6, 4, 0.7, std::nullopt, 4, 4, true, false); + ASSERT_EQ(10u, p.numDocs); + ASSERT_EQ(0u, p.heapSize); + ASSERT_EQ(0u, p.arraySize); + ASSERT_EQ(0.7, p.first_phase_rank_score_drop_limit.value()); + ASSERT_EQ(4u, p.offset); + ASSERT_EQ(4u, p.hits); } -TEST("require that match phase limiting works") { +TEST_F(MatchingTest, require_that_match_phase_limiting_works) +{ for (int s = 0; s <= 1; ++s) { for (int i = 0; i <= 6; ++i) { bool enable = (i != 0); @@ -1014,7 +1133,7 @@ TEST("require that match phase limiting works") { bool descending = (i == 2) || (i == 4) || (i == 6); bool use_sorting = (s == 1); size_t want_threads = 75; - MyWorld world; + MyWorld world(shared_state()); world.basicSetup(); world.verbose_a1_result("all"); if (enable) { @@ -1036,51 +1155,54 @@ TEST("require that match phase limiting works") { request->sortSpec = "-a1"; } SearchReply::UP reply = world.performSearch(*request, want_threads); - ASSERT_EQUAL(10u, reply->hits.size()); + ASSERT_EQ(10u, reply->hits.size()); if (enable) { - EXPECT_EQUAL(79u, reply->totalHitCount); + EXPECT_EQ(79u, reply->totalHitCount); if (!use_sorting) { - EXPECT_EQUAL(997.0, reply->hits[0].metric); - EXPECT_EQUAL(994.0, reply->hits[1].metric); - EXPECT_EQUAL(991.0, reply->hits[2].metric); - EXPECT_EQUAL(987.0, reply->hits[3].metric); - EXPECT_EQUAL(974.0, reply->hits[4].metric); - EXPECT_EQUAL(963.0, reply->hits[5].metric); - EXPECT_EQUAL(961.0, reply->hits[6].metric); - EXPECT_EQUAL(951.0, reply->hits[7].metric); - EXPECT_EQUAL(948.0, reply->hits[8].metric); - EXPECT_EQUAL(935.0, reply->hits[9].metric); + EXPECT_EQ(997.0, reply->hits[0].metric); + EXPECT_EQ(994.0, reply->hits[1].metric); + EXPECT_EQ(991.0, reply->hits[2].metric); + EXPECT_EQ(987.0, reply->hits[3].metric); + EXPECT_EQ(974.0, reply->hits[4].metric); + EXPECT_EQ(963.0, reply->hits[5].metric); + EXPECT_EQ(961.0, reply->hits[6].metric); + EXPECT_EQ(951.0, reply->hits[7].metric); + EXPECT_EQ(948.0, reply->hits[8].metric); + EXPECT_EQ(935.0, reply->hits[9].metric); } } else { - EXPECT_EQUAL(985u, reply->totalHitCount); + EXPECT_EQ(985u, reply->totalHitCount); if (!use_sorting) { - EXPECT_EQUAL(999.0, reply->hits[0].metric); - EXPECT_EQUAL(998.0, reply->hits[1].metric); - EXPECT_EQUAL(997.0, reply->hits[2].metric); - EXPECT_EQUAL(996.0, reply->hits[3].metric); + EXPECT_EQ(999.0, reply->hits[0].metric); + EXPECT_EQ(998.0, reply->hits[1].metric); + EXPECT_EQ(997.0, reply->hits[2].metric); + EXPECT_EQ(996.0, reply->hits[3].metric); } } } } } -TEST("require that arithmetic used for rank drop limit works") { +TEST_F(MatchingTest, require_that_arithmetic_used_for_rank_drop_limit_works) +{ double small = -HUGE_VAL; double limit = -std::numeric_limits<feature_t>::quiet_NaN(); EXPECT_TRUE(!(small <= limit)); } -TEST("require that termwise limit is set correctly for first phase ranking program") { - MyWorld world; +TEST_F(MatchingTest, require_that_termwise_limit_is_set_correctly_for_first_phase_ranking_program) +{ + MyWorld world(shared_state()); world.basicSetup(); world.basicResults(); - EXPECT_EQUAL(1.0, world.get_first_phase_termwise_limit()); + EXPECT_EQ(1.0, world.get_first_phase_termwise_limit()); world.set_property(indexproperties::matching::TermwiseLimit::NAME, "0.02"); - EXPECT_EQUAL(0.02, world.get_first_phase_termwise_limit()); + EXPECT_EQ(0.02, world.get_first_phase_termwise_limit()); } -TEST("require that fields are tagged with data type") { - MyWorld world; +TEST_F(MatchingTest, require_that_fields_are_tagged_with_data_type) +{ + MyWorld world(shared_state()); world.basicSetup(); auto int32_field = world.get_field_info("a1"); auto string_field = world.get_field_info("f1"); @@ -1090,24 +1212,26 @@ TEST("require that fields are tagged with data type") { ASSERT_TRUE(bool(string_field)); ASSERT_TRUE(bool(tensor_field)); ASSERT_TRUE(bool(predicate_field)); - EXPECT_EQUAL(int32_field->get_data_type(), FieldInfo::DataType::INT32); - EXPECT_EQUAL(string_field->get_data_type(), FieldInfo::DataType::STRING); - EXPECT_EQUAL(tensor_field->get_data_type(), FieldInfo::DataType::TENSOR); - EXPECT_EQUAL(predicate_field->get_data_type(), FieldInfo::DataType::BOOLEANTREE); + EXPECT_EQ(int32_field->get_data_type(), FieldInfo::DataType::INT32); + EXPECT_EQ(string_field->get_data_type(), FieldInfo::DataType::STRING); + EXPECT_EQ(tensor_field->get_data_type(), FieldInfo::DataType::TENSOR); + EXPECT_EQ(predicate_field->get_data_type(), FieldInfo::DataType::BOOLEANTREE); } -TEST("require that same element search works") { - MyWorld world; +TEST_F(MatchingTest, require_that_same_element_search_works) +{ + MyWorld world(shared_state()); world.basicSetup(); world.add_same_element_results("foo", "bar"); SearchRequest::SP request = MyWorld::createSameElementRequest("foo", "bar"); SearchReply::UP reply = world.performSearch(*request, 1); - ASSERT_EQUAL(1u, reply->hits.size()); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::20").getGlobalId(), reply->hits[0].gid); + ASSERT_EQ(1u, reply->hits.size()); + EXPECT_EQ(document::DocumentId("id:ns:searchdocument::20").getGlobalId(), reply->hits[0].gid); } -TEST("require that docsum matcher can extract matching elements from same element blueprint") { - MyWorld world; +TEST_F(MatchingTest, require_that_docsum_matcher_can_extract_matching_elements_from_same_element_blueprint) +{ + MyWorld world(shared_state()); world.basicSetup(); world.add_same_element_results("foo", "bar"); auto request = MyWorld::create_docsum_request(make_same_element_stack_dump("foo", "bar"), {20}); @@ -1116,12 +1240,13 @@ TEST("require that docsum matcher can extract matching elements from same elemen fields.add_mapping("my", "my.f1"); auto result = world.get_matching_elements(*request, fields); const auto &list = result->get_matching_elements(20, "my"); - ASSERT_EQUAL(list.size(), 1u); - EXPECT_EQUAL(list[0], 2u); + ASSERT_EQ(list.size(), 1u); + EXPECT_EQ(list[0], 2u); } -TEST("require that docsum matcher can extract matching elements from single attribute term") { - MyWorld world; +TEST_F(MatchingTest, require_that_docsum_matcher_can_extract_matching_elements_from_single_attribute_term) +{ + MyWorld world(shared_state()); world.basicSetup(); world.add_same_element_results("foo", "bar"); auto request = MyWorld::create_docsum_request(make_simple_stack_dump("my.a1", "foo"), {20}); @@ -1130,9 +1255,9 @@ TEST("require that docsum matcher can extract matching elements from single attr fields.add_mapping("my", "my.f1"); auto result = world.get_matching_elements(*request, fields); const auto &list = result->get_matching_elements(20, "my"); - ASSERT_EQUAL(list.size(), 2u); - EXPECT_EQUAL(list[0], 2u); - EXPECT_EQUAL(list[1], 3u); + ASSERT_EQ(list.size(), 2u); + EXPECT_EQ(list[0], 2u); + EXPECT_EQ(list[1], 3u); } using FMA = vespalib::FuzzyMatchingAlgorithm; @@ -1162,35 +1287,41 @@ struct AttributeBlueprintParamsFixture { rank_properties.add(TargetHitsMaxAdjustmentFactor::NAME, target_hits_max_adjustment_factor); rank_properties.add(FuzzyAlgorithm::NAME, fuzzy_matching_algorithm); } + ~AttributeBlueprintParamsFixture(); AttributeBlueprintParams extract(uint32_t active_docids = 9, uint32_t docid_limit = 10) const { return MatchToolsFactory::extract_attribute_blueprint_params(rank_setup, rank_properties, active_docids, docid_limit); } }; -TEST_F("attribute blueprint params are extracted from rank profile", AttributeBlueprintParamsFixture(0.2, 0.8, 5.0, FMA::DfaTable)) +AttributeBlueprintParamsFixture::~AttributeBlueprintParamsFixture() = default; + +TEST_F(MatchingTest, attribute_blueprint_params_are_extracted_from_rank_profile) { + AttributeBlueprintParamsFixture f(0.2, 0.8, 5.0, FMA::DfaTable); auto params = f.extract(); - EXPECT_EQUAL(0.2, params.global_filter_lower_limit); - EXPECT_EQUAL(0.8, params.global_filter_upper_limit); - EXPECT_EQUAL(5.0, params.target_hits_max_adjustment_factor); - EXPECT_EQUAL(FMA::DfaTable, params.fuzzy_matching_algorithm); + EXPECT_EQ(0.2, params.global_filter_lower_limit); + EXPECT_EQ(0.8, params.global_filter_upper_limit); + EXPECT_EQ(5.0, params.target_hits_max_adjustment_factor); + EXPECT_EQ(FMA::DfaTable, params.fuzzy_matching_algorithm); } -TEST_F("attribute blueprint params are extracted from query", AttributeBlueprintParamsFixture(0.2, 0.8, 5.0, FMA::DfaTable)) +TEST_F(MatchingTest, attribute_blueprint_params_are_extracted_from_query) { + AttributeBlueprintParamsFixture f(0.2, 0.8, 5.0, FMA::DfaTable); f.set_query_properties("0.15", "0.75", "3.0", "dfa_explicit"); auto params = f.extract(); - EXPECT_EQUAL(0.15, params.global_filter_lower_limit); - EXPECT_EQUAL(0.75, params.global_filter_upper_limit); - EXPECT_EQUAL(3.0, params.target_hits_max_adjustment_factor); - EXPECT_EQUAL(FMA::DfaExplicit, params.fuzzy_matching_algorithm); + EXPECT_EQ(0.15, params.global_filter_lower_limit); + EXPECT_EQ(0.75, params.global_filter_upper_limit); + EXPECT_EQ(3.0, params.target_hits_max_adjustment_factor); + EXPECT_EQ(FMA::DfaExplicit, params.fuzzy_matching_algorithm); } -TEST_F("global filter params are scaled with active hit ratio", AttributeBlueprintParamsFixture(0.2, 0.8, 5.0, FMA::DfaTable)) +TEST_F(MatchingTest, global_filter_params_are_scaled_with_active_hit_ratio) { + AttributeBlueprintParamsFixture f(0.2, 0.8, 5.0, FMA::DfaTable); auto params = f.extract(5, 10); - EXPECT_EQUAL(0.12, params.global_filter_lower_limit); - EXPECT_EQUAL(0.48, params.global_filter_upper_limit); + EXPECT_EQ(0.12, params.global_filter_lower_limit); + EXPECT_EQ(0.48, params.global_filter_upper_limit); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 83b7e10c7a8..b917e69ce7b 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -30,7 +30,7 @@ #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/thread_bundle.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/query/tree/querytreecreator.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index 64c6870499c..a1c73d0fa76 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -29,7 +29,7 @@ #include <vespa/searchlib/queryeval/sourceblendersearch.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("querynodes_test"); diff --git a/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp b/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp index e85bef44f3e..d73038ddcb5 100644 --- a/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp +++ b/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp @@ -10,7 +10,7 @@ LOG_SETUP("resolveviewvisitor_test"); #include <vespa/searchcore/proton/matching/viewresolver.h> #include <vespa/searchlib/query/tree/node.h> #include <vespa/searchlib/query/tree/querybuilder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <string> namespace fef_test = search::fef::test; diff --git a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp index 8bfbcacbf23..3153bcccb42 100644 --- a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp +++ b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchcore/proton/matching/match_tools.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp index 43d2c54fd03..f440d573859 100644 --- a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp +++ b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp @@ -15,7 +15,7 @@ LOG_SETUP("termdataextractor_test"); #include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/querybuilder.h> #include <vespa/searchlib/query/weight.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <string> #include <vector> diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore deleted file mode 100644 index 84c97c63aca..00000000000 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_documentdb_job_trackers_test_app diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt deleted file mode 100644 index 81d054e2242..00000000000 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_documentdb_job_trackers_test_app TEST - SOURCES - documentdb_job_trackers_test.cpp - DEPENDS - searchcore_proton_metrics - searchcore_test -) -vespa_add_test(NAME searchcore_documentdb_job_trackers_test_app COMMAND searchcore_documentdb_job_trackers_test_app) diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore b/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore deleted file mode 100644 index 2e02ec8191b..00000000000 --- a/searchcore/src/tests/proton/metrics/job_load_sampler/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_job_load_sampler_test_app diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt deleted file mode 100644 index 955b1e14028..00000000000 --- a/searchcore/src/tests/proton/metrics/job_load_sampler/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_job_load_sampler_test_app TEST - SOURCES - job_load_sampler_test.cpp - DEPENDS - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_job_load_sampler_test_app COMMAND searchcore_job_load_sampler_test_app) diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore b/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore deleted file mode 100644 index 85e6097878b..00000000000 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_job_tracked_flush_test_app diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt deleted file mode 100644 index a4f0ff0bcec..00000000000 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_job_tracked_flush_test_app TEST - SOURCES - job_tracked_flush_test.cpp - DEPENDS - searchcore_proton_metrics - searchcore_test -) -vespa_add_test(NAME searchcore_job_tracked_flush_test_app COMMAND searchcore_job_tracked_flush_test_app) diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp index 523ffccd2f0..dd622a4cac1 100644 --- a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp +++ b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp @@ -3,7 +3,7 @@ #include <vespa/metrics/metricset.h> #include <vespa/searchcore/proton/metrics/attribute_metrics.h> #include <vespa/searchcore/proton/metrics/metrics_engine.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("metrics_engine_test"); diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp index 528067aeeb1..f13b9e9a9db 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -5,7 +5,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using namespace proton; diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 21ac6893356..0f1e2e9487a 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -17,7 +17,7 @@ #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <set> using document::BucketId; diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 06264e3e642..1dfc6c481b6 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -24,7 +24,7 @@ #include <vespa/fileacquirer/config-filedistributorrpc.h> #include <vespa/vespalib/util/hw_info.h> #include <vespa/vespalib/util/varholder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config.h> #include <map> #include <thread> diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp index 411dd88f995..881c547df1e 100644 --- a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp +++ b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/stringfmt.h> #include <filesystem> diff --git a/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp index a2f85ce991b..dee0b6a2448 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <cassert> #include <vespa/log/log.h> LOG_SETUP("document_db_reference_test"); diff --git a/searchcore/src/tests/proton/reference/document_db_reference_registry/document_db_reference_registry_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_registry/document_db_reference_registry_test.cpp index 0aa0e31442e..5ec785496e2 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_registry/document_db_reference_registry_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_registry/document_db_reference_registry_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchcore/proton/reference/document_db_reference_registry.h> #include <vespa/searchcore/proton/test/mock_document_db_reference.h> diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp index 80f4f826c4a..57bae7cfc7c 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp @@ -20,7 +20,7 @@ #include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config-imported-fields.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp index 48df346317a..22c6c936465 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/base/documentid.h> #include <vespa/vespalib/util/threadstackexecutor.h> diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp index da3449b91aa..4c70c1830db 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h> diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp index 2c218534545..6e08c8397d0 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/base/globalid.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp index 36429c79a26..70211909296 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_mapper/gid_to_lid_mapper_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp index 97a319ecfd4..97c5b458867 100644 --- a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp @@ -4,7 +4,7 @@ LOG_SETUP("document_reprocessing_handler_test"); #include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h> #include <vespa/searchlib/test/doc_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace document; using namespace proton; diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp index bb62b239cd2..5e7b11e09a7 100644 --- a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp @@ -4,7 +4,7 @@ LOG_SETUP("reprocessing_runner_test"); #include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h> #include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace proton; diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index 142a2b8693f..b05964fe7e0 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -40,7 +40,7 @@ #include <vespa/document/test/fieldvalue_helpers.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> diff --git a/searchcore/src/tests/proton/server/feeddebugger_test.cpp b/searchcore/src/tests/proton/server/feeddebugger_test.cpp index cedaf2be12e..4d1bbdb2f03 100644 --- a/searchcore/src/tests/proton/server/feeddebugger_test.cpp +++ b/searchcore/src/tests/proton/server/feeddebugger_test.cpp @@ -6,7 +6,7 @@ LOG_SETUP("feeddebugger_test"); #include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/common/feeddebugger.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::DocumentId; using std::string; diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index 8ecdfba63f5..c46510c5181 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -16,7 +16,7 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/foreground_thread_executor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp index 79b4b4a3627..0dc8390d7d9 100644 --- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp +++ b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/server/memory_flush_config_updater.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp index e07886f9576..c0245cc1a9d 100644 --- a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp +++ b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("memoryconfigstore_test"); #include <vespa/searchcommon/common/schema.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using search::SerialNum; using search::index::Schema; diff --git a/searchcore/src/tests/proton/statusreport/.gitignore b/searchcore/src/tests/proton/statusreport/.gitignore deleted file mode 100644 index 68753df292a..00000000000 --- a/searchcore/src/tests/proton/statusreport/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_statusreport_test_app diff --git a/searchcore/src/tests/proton/statusreport/CMakeLists.txt b/searchcore/src/tests/proton/statusreport/CMakeLists.txt deleted file mode 100644 index 1f2a3cfbec1..00000000000 --- a/searchcore/src/tests/proton/statusreport/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_statusreport_test_app TEST - SOURCES - statusreport_test.cpp - DEPENDS - searchcore_pcommon -) -vespa_add_test(NAME searchcore_statusreport_test_app COMMAND searchcore_statusreport_test_app) diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp index bb1d06acb4c..2ab81058e43 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> #include <vespa/searchlib/engine/searchreply.h> diff --git a/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt b/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt index b27a7ee53c2..eb330be1446 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt +++ b/searchcore/src/tests/proton/verify_ranksetup/CMakeLists.txt @@ -5,4 +5,4 @@ vespa_add_executable(searchcore_verify_ranksetup_test_app TEST DEPENDS ) vespa_add_test(NAME searchcore_verify_ranksetup_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/verify_ranksetup_test.sh - DEPENDS searchcore_verify_ranksetup_test_app searchcore_verify_ranksetup_app) + DEPENDS searchcore_verify_ranksetup_test_app searchcore_verify_ranksetup_app COST 50) diff --git a/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt b/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt index 1c0fc93d24c..324e0b89d26 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt @@ -50,6 +50,6 @@ vespa_add_library(searchcore_bmcluster STATIC searchcore_proton_metrics searchcorespi storageserver_storageapp - messagebus_messagebus-test - messagebus + vespa_messagebus-test + vespa_messagebus ) diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp index e21cb4c1282..88939958aa0 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp @@ -53,11 +53,7 @@ GroupingContext::setDistributionKey(uint32_t distributionKey) GroupingContext::GroupingContext(const BitVector & validLids, const std::atomic<steady_time> & now_ref, vespalib::steady_time timeOfDoom, const char *groupSpec, uint32_t groupSpecLen) - : _validLids(validLids), - _now_ref(now_ref), - _timeOfDoom(timeOfDoom), - _os(), - _groupingList() + : GroupingContext(validLids, now_ref, timeOfDoom) { deserialize(groupSpec, groupSpecLen); } @@ -71,11 +67,7 @@ GroupingContext::GroupingContext(const BitVector & validLids, const std::atomic< { } GroupingContext::GroupingContext(const GroupingContext & rhs) - : _validLids(rhs._validLids), - _now_ref(rhs._now_ref), - _timeOfDoom(rhs._timeOfDoom), - _os(), - _groupingList() + : GroupingContext(rhs._validLids, rhs._now_ref, rhs._timeOfDoom) { } void diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp index 9ece37665cd..9b393b6e6d8 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp @@ -4,6 +4,7 @@ #include "groupingcontext.h" #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/expression/attributenode.h> +#include <vespa/searchlib/aggregation/modifiers.h> #include <vespa/vespalib/util/issue.h> #include <vespa/vespalib/util/stringfmt.h> @@ -48,6 +49,7 @@ GroupingManager::init(const IAttributeContext &attrCtx) an.enableEnumOptimization(true); } } + aggregation::NonAttribute2DocumentAccessor nonAttributes2DocumentAccess(attrCtx); ConfigureStaticParams stuff(&attrCtx, nullptr); grouping.configureStaticStuff(stuff); list.push_back(groupingList[i]); diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp index ef45b36b551..e3f8919e747 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp @@ -27,18 +27,11 @@ GroupingSession::GroupingSession(const SessionId &sessionId, GroupingSession::~GroupingSession() = default; -using search::expression::ExpressionNode; -using search::expression::AttributeNode; -using search::expression::ConfigureStaticParams; -using search::aggregation::Grouping; -using search::aggregation::GroupingLevel; - void GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext &attrCtx) { GroupingList & sessionList(groupingContext.getGroupingList()); - for (size_t i = 0; i < sessionList.size(); ++i) { - GroupingPtr g(sessionList[i]); + for (auto g : sessionList) { // Make internal copy of those we want to keep for another pass if (!_sessionId.empty() && g->getLastLevel() < g->levels().size()) { auto gp = std::make_shared<Grouping>(*g); diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index 919309c5dae..f8b6666afc4 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -62,12 +62,17 @@ private: Blueprint::UP _result; void buildChildren(IntermediateBlueprint &parent, const std::vector<Node *> &children); + bool is_search_multi_threaded() const noexcept { + return _requestContext.thread_bundle().size() > 1; + } template <typename NodeType> void buildIntermediate(IntermediateBlueprint *b, NodeType &n) __attribute__((noinline)); void buildWeakAnd(ProtonWeakAnd &n) { - auto *wand = new WeakAndBlueprint(n.getTargetNumHits(), _requestContext.get_attribute_blueprint_params().weakand_range); + auto *wand = new WeakAndBlueprint(n.getTargetNumHits(), + _requestContext.get_attribute_blueprint_params().weakand_range, + is_search_multi_threaded()); Blueprint::UP result(wand); for (auto node : n.getChildren()) { uint32_t weight = getWeightFromNode(*node).percent(); diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp index 4d4b136aba5..c7f86903e05 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp @@ -24,6 +24,7 @@ using search::queryeval::AndNotBlueprint; using search::queryeval::Blueprint; using search::queryeval::IntermediateBlueprint; using search::queryeval::MatchingElementsSearch; +using search::queryeval::MatchingPhase; using search::queryeval::SameElementBlueprint; using search::queryeval::SearchIterator; using vespalib::FeatureSet; @@ -41,8 +42,10 @@ get_feature_set(const MatchToolsFactory &mtf, { MatchTools::UP matchTools = mtf.createMatchTools(); if (summaryFeatures) { + mtf.query().set_matching_phase(MatchingPhase::SUMMARY_FEATURES); matchTools->setup_summary(); } else { + mtf.query().set_matching_phase(MatchingPhase::DUMP_FEATURES); matchTools->setup_dump(); } auto retval = ExtractFeatures::get_feature_set(matchTools->search(), matchTools->rank_program(), docs, diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp index 87b2fa8c1cb..ee255ab41ba 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp @@ -19,6 +19,7 @@ using vespalib::Runnable; using vespalib::ThreadBundle; using search::fef::FeatureResolver; using search::fef::RankProgram; +using search::queryeval::MatchingPhase; using search::queryeval::SearchIterator; namespace proton::matching { @@ -112,6 +113,7 @@ FeatureValues ExtractFeatures::get_match_features(const MatchToolsFactory &mtf, const OrderedDocs &docs, ThreadBundle &thread_bundle) { FeatureValues result; + mtf.query().set_matching_phase(MatchingPhase::MATCH_FEATURES); auto tools = mtf.createMatchTools(); tools->setup_match_features(); FeatureResolver resolver(tools->rank_program().get_seeds(false)); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp index 01a9508220d..affa2bdc554 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp @@ -1,18 +1,21 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "match_loop_communicator.h" +#include <vespa/searchlib/features/first_phase_rank_lookup.h> #include <vespa/vespalib/util/priority_queue.h> +using search::features::FirstPhaseRankLookup; + namespace proton:: matching { MatchLoopCommunicator::MatchLoopCommunicator(size_t threads, size_t topN) - : MatchLoopCommunicator(threads, topN, std::unique_ptr<IDiversifier>()) + : MatchLoopCommunicator(threads, topN, {}, nullptr, []() noexcept {}) {} -MatchLoopCommunicator::MatchLoopCommunicator(size_t threads, size_t topN, std::unique_ptr<IDiversifier> diversifier) +MatchLoopCommunicator::MatchLoopCommunicator(size_t threads, size_t topN, std::unique_ptr<IDiversifier> diversifier, FirstPhaseRankLookup* first_phase_rank_lookup, std::function<void()> before_second_phase) : _best_scores(), _best_dropped(), _estimate_match_frequency(threads), - _get_second_phase_work(threads, topN, _best_scores, _best_dropped, std::move(diversifier)), + _get_second_phase_work(threads, topN, _best_scores, _best_dropped, std::move(diversifier), first_phase_rank_lookup, std::move(before_second_phase)), _complete_second_phase(threads, topN, _best_scores, _best_dropped) {} MatchLoopCommunicator::~MatchLoopCommunicator() = default; @@ -34,18 +37,44 @@ MatchLoopCommunicator::EstimateMatchFrequency::mingle() } } -MatchLoopCommunicator::GetSecondPhaseWork::GetSecondPhaseWork(size_t n, size_t topN_in, Range &best_scores_in, BestDropped &best_dropped_in, std::unique_ptr<IDiversifier> diversifier) +namespace { + +class NoRegisterFirstPhaseRank { +public: + static void pick(uint32_t) noexcept { }; + static void drop() noexcept { } +}; + +class RegisterFirstPhaseRank { + FirstPhaseRankLookup& _first_phase_rank_lookup; + uint32_t _rank; +public: + RegisterFirstPhaseRank(FirstPhaseRankLookup& first_phase_rank_lookup) + : _first_phase_rank_lookup(first_phase_rank_lookup), + _rank(0) + { + } + void pick(uint32_t docid) noexcept { _first_phase_rank_lookup.add(docid, ++_rank); } + void drop() noexcept { ++_rank; } +}; + +} + +MatchLoopCommunicator::GetSecondPhaseWork::GetSecondPhaseWork(size_t n, size_t topN_in, Range &best_scores_in, BestDropped &best_dropped_in, std::unique_ptr<IDiversifier> diversifier, FirstPhaseRankLookup* first_phase_rank_lookup, std::function<void()> before_second_phase) : vespalib::Rendezvous<SortedHitSequence, TaggedHits, true>(n), topN(topN_in), best_scores(best_scores_in), best_dropped(best_dropped_in), - _diversifier(std::move(diversifier)) + _diversifier(std::move(diversifier)), + _first_phase_rank_lookup(first_phase_rank_lookup), + _before_second_phase(std::move(before_second_phase)) {} + MatchLoopCommunicator::GetSecondPhaseWork::~GetSecondPhaseWork() = default; -template<typename Q, typename F> +template<typename Q, typename F, typename R> void -MatchLoopCommunicator::GetSecondPhaseWork::mingle(Q &queue, F &&accept) +MatchLoopCommunicator::GetSecondPhaseWork::mingle(Q &queue, F &&accept, R register_first_phase_rank) { size_t picked = 0; search::feature_t last_score = 0.0; @@ -53,14 +82,18 @@ MatchLoopCommunicator::GetSecondPhaseWork::mingle(Q &queue, F &&accept) uint32_t i = queue.front(); const Hit & hit = in(i).get(); if (accept(hit.first)) { + register_first_phase_rank.pick(hit.first); out(picked % size()).emplace_back(hit, i); last_score = hit.second; if (++picked == 1) { best_scores.high = hit.second; } - } else if (!best_dropped.valid) { - best_dropped.valid = true; - best_dropped.score = hit.second; + } else { + if (!best_dropped.valid) { + best_dropped.valid = true; + best_dropped.score = hit.second; + } + register_first_phase_rank.drop(); } in(i).next(); if (in(i).valid()) { @@ -74,9 +107,21 @@ MatchLoopCommunicator::GetSecondPhaseWork::mingle(Q &queue, F &&accept) } } +template<typename Q, typename R> +void +MatchLoopCommunicator::GetSecondPhaseWork::mingle(Q &queue, R register_first_phase_rank) +{ + if (_diversifier) { + mingle(queue, [diversifier=_diversifier.get()](uint32_t docId) { return diversifier->accepted(docId);}, register_first_phase_rank); + } else { + mingle(queue, [](uint32_t) { return true;}, register_first_phase_rank); + } +} + void MatchLoopCommunicator::GetSecondPhaseWork::mingle() { + _before_second_phase(); best_scores = Range(); best_dropped.valid = false; size_t est_out = (topN / size()) + 1; @@ -87,10 +132,10 @@ MatchLoopCommunicator::GetSecondPhaseWork::mingle() queue.push(i); } } - if (_diversifier) { - mingle(queue, [diversifier=_diversifier.get()](uint32_t docId) { return diversifier->accepted(docId);}); + if (_first_phase_rank_lookup != nullptr) { + mingle(queue, RegisterFirstPhaseRank(*_first_phase_rank_lookup)); } else { - mingle(queue, [](uint32_t) { return true;}); + mingle(queue, NoRegisterFirstPhaseRank()); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h index eb93bdb68d5..a0a8376254b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h @@ -5,6 +5,9 @@ #include "i_match_loop_communicator.h" #include <vespa/searchlib/queryeval/idiversifier.h> #include <vespa/vespalib/util/rendezvous.h> +#include <functional> + +namespace search::features { class FirstPhaseRankLookup; } namespace proton::matching { @@ -12,6 +15,7 @@ class MatchLoopCommunicator final : public IMatchLoopCommunicator { private: using IDiversifier = search::queryeval::IDiversifier; + using FirstPhaseRankLookup = search::features::FirstPhaseRankLookup; struct BestDropped { bool valid = false; search::feature_t score = 0.0; @@ -25,11 +29,15 @@ private: Range &best_scores; BestDropped &best_dropped; std::unique_ptr<IDiversifier> _diversifier; - GetSecondPhaseWork(size_t n, size_t topN_in, Range &best_scores_in, BestDropped &best_dropped_in, std::unique_ptr<IDiversifier>); + FirstPhaseRankLookup* _first_phase_rank_lookup; + std::function<void()> _before_second_phase; + GetSecondPhaseWork(size_t n, size_t topN_in, Range &best_scores_in, BestDropped &best_dropped_in, std::unique_ptr<IDiversifier> diversifier, FirstPhaseRankLookup* first_phase_rank_lookup, std::function<void()> before_second_phase); ~GetSecondPhaseWork() override; void mingle() override; - template<typename Q, typename F> - void mingle(Q &queue, F &&accept); + template<typename Q, typename R> + void mingle(Q &queue, R register_first_phase_rank); + template<typename Q, typename F, typename R> + void mingle(Q &queue, F &&accept, R register_first_phase_rank); bool cmp(uint32_t a, uint32_t b) { return (in(a).get().second > in(b).get().second); } @@ -59,7 +67,7 @@ private: public: MatchLoopCommunicator(size_t threads, size_t topN); - MatchLoopCommunicator(size_t threads, size_t topN, std::unique_ptr<IDiversifier>); + MatchLoopCommunicator(size_t threads, size_t topN, std::unique_ptr<IDiversifier>, FirstPhaseRankLookup* first_phase_rank_lookup, std::function<void()> before_second_phsae); ~MatchLoopCommunicator(); double estimate_match_frequency(const Matches &matches) override { diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 89cc97767bf..152ba978cd1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -17,6 +17,7 @@ namespace proton::matching { using namespace search::fef; +using search::queryeval::MatchingPhase; using search::queryeval::SearchIterator; using vespalib::FeatureSet; using vespalib::ThreadBundle; @@ -85,7 +86,13 @@ MatchMaster::match(search::engine::Trace & trace, { vespalib::Timer query_latency_time; vespalib::DualMergeDirector mergeDirector(threadBundle.size()); - MatchLoopCommunicator communicator(threadBundle.size(), params.heapSize, mtf.createDiversifier(params.heapSize)); + /* + * We need a non-const first phase rank lookup since it will be populated + * later on when selecting documents for second phase ranking. + */ + MatchLoopCommunicator communicator(threadBundle.size(), params.heapSize, mtf.createDiversifier(params.heapSize), + mtf.get_first_phase_rank_lookup(), + [&mtf]() noexcept { mtf.query().set_matching_phase(MatchingPhase::SECOND_PHASE); }); TimedMatchLoopCommunicator timedCommunicator(communicator); DocidRangeScheduler::UP scheduler = createScheduler(threadBundle.size(), numSearchPartitions, params.numDocs); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp index 5cd97d314b5..316ef003a28 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp @@ -19,7 +19,8 @@ computeArraySize(uint32_t hitsPlussOffset, uint32_t heapSize, uint32_t arraySize MatchParams::MatchParams(uint32_t numDocs_in, uint32_t heapSize_in, uint32_t arraySize_in, - search::feature_t rankDropLimit_in, + std::optional<search::feature_t> first_phase_rank_score_drop_limit_in, + std::optional<search::feature_t> second_phase_rank_score_drop_limit_in, uint32_t offset_in, uint32_t hits_in, bool hasFinalRank, @@ -31,12 +32,8 @@ MatchParams::MatchParams(uint32_t numDocs_in, : 0), offset(std::min(numDocs_in, offset_in)), hits(std::min(numDocs_in - offset, hits_in)), - rankDropLimit(rankDropLimit_in) + first_phase_rank_score_drop_limit(first_phase_rank_score_drop_limit_in), + second_phase_rank_score_drop_limit(second_phase_rank_score_drop_limit_in) { } -bool -MatchParams::has_rank_drop_limit() const { - return ! std::isnan(rankDropLimit); -} - } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_params.h b/searchcore/src/vespa/searchcore/proton/matching/match_params.h index 5b58c11b7e1..19abcd8e449 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_params.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_params.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/common/feature.h> #include <cstdint> +#include <optional> namespace proton::matching { @@ -17,18 +18,19 @@ struct MatchParams { const uint32_t arraySize; const uint32_t offset; const uint32_t hits; - const search::feature_t rankDropLimit; + const std::optional<search::feature_t> first_phase_rank_score_drop_limit; + const std::optional<search::feature_t> second_phase_rank_score_drop_limit; MatchParams(uint32_t numDocs_in, uint32_t heapSize_in, uint32_t arraySize_in, - search::feature_t rankDropLimit_in, + std::optional<search::feature_t> first_phase_rank_drop_limit_in, + std::optional<search::feature_t> second_phase_rank_score_drop_limit_in, uint32_t offset_in, uint32_t hits_in, bool hasFinalRank, - bool needRanking=true); - bool save_rank_scores() const { return ((heapSize + arraySize) != 0); } - bool has_rank_drop_limit() const; + bool needRanking); + bool save_rank_scores() const noexcept { return (arraySize != 0); } }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index 211e67f1e2b..7ccc120d047 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -15,6 +15,7 @@ #include <vespa/searchlib/queryeval/profiled_iterator.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> +#include <limits> #include <vespa/log/log.h> LOG_SETUP(".proton.matching.match_thread"); @@ -99,11 +100,11 @@ fillPartialResult(ResultProcessor::Context & context, size_t totalHits, size_t n //----------------------------------------------------------------------------- -MatchThread::Context::Context(double rankDropLimit, MatchTools &tools, HitCollector &hits, uint32_t num_threads) +MatchThread::Context::Context(std::optional<double> first_phase_rank_score_drop_limit, MatchTools &tools, HitCollector &hits, uint32_t num_threads) : matches(0), _matches_limit(tools.match_limiter().sample_hits_per_thread(num_threads)), _score_feature(get_score_feature(tools.rank_program())), - _rankDropLimit(rankDropLimit), + _first_phase_rank_score_drop_limit(first_phase_rank_score_drop_limit.value_or(0.0 /* ignored */)), _hits(hits), _doom(tools.getDoom()), dropped() @@ -119,7 +120,7 @@ MatchThread::Context::rankHit(uint32_t docId) { score = -HUGE_VAL; } if (use_rank_drop_limit != RankDropLimitE::no) { - if (__builtin_expect(score > _rankDropLimit, true)) { + if (__builtin_expect(score > _first_phase_rank_score_drop_limit, true)) { _hits.addHit(docId, score); } else if (use_rank_drop_limit == RankDropLimitE::track) { dropped.template emplace_back(docId); @@ -217,7 +218,7 @@ MatchThread::match_loop(MatchTools &tools, HitCollector &hits) bool softDoomed = false; uint32_t docsCovered = 0; vespalib::duration overtime(vespalib::duration::zero()); - Context context(matchParams.rankDropLimit, tools, hits, num_threads); + Context context(matchParams.first_phase_rank_score_drop_limit, tools, hits, num_threads); for (DocidRange docid_range = scheduler.first_range(thread_id); !docid_range.empty(); docid_range = scheduler.next_range(thread_id)) @@ -270,7 +271,7 @@ template <bool do_rank, bool do_limit, bool do_share> void MatchThread::match_loop_helper_rank_limit_share(MatchTools &tools, HitCollector &hits) { - if (matchParams.has_rank_drop_limit()) { + if (matchParams.first_phase_rank_score_drop_limit.has_value()) { if (matchToolsFactory.hasOnMatchTask()) { match_loop_helper_rank_limit_share_drop<do_rank, do_limit, do_share, RankDropLimitE::track>(tools, hits); } else { @@ -367,7 +368,7 @@ MatchThread::findMatches(MatchTools &tools) tools.give_back_search(ProfiledIterator::profile(*match_profiler, tools.borrow_search())); tools.tag_search_as_changed(); } - HitCollector hits(matchParams.numDocs, matchParams.arraySize); + HitCollector hits(matchParams.numDocs, match_with_ranking ? matchParams.arraySize : 0); trace->addEvent(4, "Start match and first phase rank"); /** * All, or none of the threads in the bundle must execute the match loop. @@ -380,7 +381,32 @@ MatchThread::findMatches(MatchTools &tools) secondPhase(tools, hits); } trace->addEvent(4, "Create result set"); - return hits.getResultSet(fallback_rank_value()); + if (tools.has_second_phase_rank() && matchParams.second_phase_rank_score_drop_limit.has_value()) { + return get_matches_after_second_phase_rank_score_drop(hits); + } else { + return hits.getResultSet(); + } +} + +std::unique_ptr<search::ResultSet> +MatchThread::get_matches_after_second_phase_rank_score_drop(HitCollector& hits) +{ + std::vector<uint32_t> dropped; + auto result = hits.get_result_set(matchParams.second_phase_rank_score_drop_limit, &dropped); + if (!dropped.empty()) { + /* + * Hits dropped due to second phase rank score drop limit are + * not present in the result. Schedule extra tasks to update + * mutable attributes for earlier match phases. + */ + if (auto task = matchToolsFactory.createOnMatchTask()) { + task->run(dropped); + } + if (auto task = matchToolsFactory.createOnFirstPhaseTask()) { + task->run(std::move(dropped)); + } + } + return result; } void diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index c6b233f2fcd..e017dc53c5c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -73,7 +73,7 @@ private: class Context { public: - Context(double rankDropLimit, MatchTools &tools, HitCollector &hits, + Context(std::optional<double> first_phase_rank_score_drop_limit, MatchTools &tools, HitCollector &hits, uint32_t num_threads) __attribute__((noinline)); template <RankDropLimitE use_rank_drop_limit> void rankHit(uint32_t docId); @@ -86,7 +86,7 @@ private: private: uint32_t _matches_limit; LazyValue _score_feature; - double _rankDropLimit; + double _first_phase_rank_score_drop_limit; HitCollector &_hits; const Doom _doom; public: @@ -113,6 +113,7 @@ private: void match_loop_helper(MatchTools &tools, HitCollector &hits); search::ResultSet::UP findMatches(MatchTools &tools); + std::unique_ptr<search::ResultSet> get_matches_after_second_phase_rank_score_drop(HitCollector& hits); void secondPhase(MatchTools & tools, HitCollector & hits); void processResult(const Doom & doom, search::ResultSet::UP result, ResultProcessor::Context &context); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 06290386a31..73a812f936f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/attribute/diversity.h> #include <vespa/searchlib/queryeval/flow.h> #include <vespa/searchlib/engine/trace.h> +#include <vespa/searchlib/features/first_phase_rank_lookup.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/ranksetup.h> #include <vespa/vespalib/util/issue.h> @@ -190,7 +191,8 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _rankSetup(rankSetup), _featureOverrides(featureOverrides), _diversityParams(), - _valid(false) + _valid(false), + _first_phase_rank_lookup(nullptr) { if (doom.soft_doom()) return; auto trace = root_trace.make_trace(); @@ -204,6 +206,9 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _query.extractLocations(_queryEnv.locations()); trace.addEvent(5, "Build query execution plan"); _query.reserveHandles(_requestContext, searchContext, _mdl); + if (trace.getLevel() >= 6) { // will dump blueprint later + _query.enumerate_blueprint_nodes(); + } trace.addEvent(5, "Optimize query execution plan"); bool sort_by_cost = SortBlueprintsByCost::check(_queryEnv.getProperties(), rankSetup.sort_blueprints_by_cost()); double hitRate = std::min(1.0, double(maxNumHits)/double(searchContext.getDocIdLimit())); @@ -219,6 +224,7 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _query.freeze(); trace.addEvent(5, "Prepare shared state for multi-threaded rank executors"); _rankSetup.prepareSharedState(_queryEnv, _queryEnv.getObjectStore()); + _first_phase_rank_lookup = FirstPhaseRankLookup::get_mutable_shared_state(_queryEnv.getObjectStore()); _diversityParams = extractDiversityParams(_rankSetup, rankProperties); vespalib::string attribute = DegradationAttribute::lookup(rankProperties, _rankSetup.getDegradationAttribute()); DegradationParams degradationParams = extractDegradationParams(_rankSetup, attribute, rankProperties); @@ -272,11 +278,13 @@ MatchToolsFactory::createTask(vespalib::stringref attribute, vespalib::stringref ? std::make_unique<AttributeOperationTask>(_requestContext, attribute, operation) : std::unique_ptr<AttributeOperationTask>(); } + std::unique_ptr<AttributeOperationTask> MatchToolsFactory::createOnMatchTask() const { const auto & op = _rankSetup.getMutateOnMatch(); return createTask(op._attribute, op._operation); } + std::unique_ptr<AttributeOperationTask> MatchToolsFactory::createOnFirstPhaseTask() const { const auto & op = _rankSetup.getMutateOnFirstPhase(); @@ -289,6 +297,7 @@ MatchToolsFactory::createOnFirstPhaseTask() const { return createTask(op._attribute, op._operation); } } + std::unique_ptr<AttributeOperationTask> MatchToolsFactory::createOnSecondPhaseTask() const { const auto & op = _rankSetup.getMutateOnSecondPhase(); @@ -299,6 +308,7 @@ MatchToolsFactory::createOnSecondPhaseTask() const { return createTask(op._attribute, op._operation); } } + std::unique_ptr<AttributeOperationTask> MatchToolsFactory::createOnSummaryTask() const { const auto & op = _rankSetup.getMutateOnSummary(); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h index 759fe68eea2..da18a8b0a2f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h @@ -21,6 +21,7 @@ namespace vespalib { class ExecutionProfiler; } namespace vespalib { struct ThreadBundle; } namespace search::engine { class Trace; } +namespace search::features { class FirstPhaseRankLookup; } namespace search::fef { class RankProgram; @@ -119,6 +120,7 @@ private: using RankSetup = search::fef::RankSetup; using IIndexEnvironment = search::fef::IIndexEnvironment; using IDiversifier = search::queryeval::IDiversifier; + using FirstPhaseRankLookup = search::features::FirstPhaseRankLookup; QueryLimiter & _queryLimiter; AttributeBlueprintParams _attribute_blueprint_params; Query _query; @@ -131,6 +133,7 @@ private: const Properties & _featureOverrides; DiversityParams _diversityParams; bool _valid; + FirstPhaseRankLookup* _first_phase_rank_lookup; std::unique_ptr<AttributeOperationTask> createTask(vespalib::stringref attribute, vespalib::stringref operation) const; @@ -186,6 +189,7 @@ public: static AttributeBlueprintParams extract_attribute_blueprint_params(const RankSetup& rank_setup, const Properties& rank_properties, uint32_t active_docids, uint32_t docid_limit); + FirstPhaseRankLookup* get_first_phase_rank_lookup() const noexcept { return _first_phase_rank_lookup; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 4a9156770f0..a5ace1676ef 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -18,6 +18,7 @@ #include <vespa/searchlib/fef/test/plugin/setup.h> #include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/vespalib/data/slime/inserter.h> +#include <vespa/vespalib/util/limited_thread_bundle_wrapper.h> #include <cinttypes> #include <vespa/log/log.h> @@ -38,7 +39,8 @@ using search::fef::MatchData; using search::fef::RankSetup; using search::fef::indexproperties::hitcollector::HeapSize; using search::fef::indexproperties::hitcollector::ArraySize; -using search::fef::indexproperties::hitcollector::RankScoreDropLimit; +using search::fef::indexproperties::hitcollector::FirstPhaseRankScoreDropLimit; +using search::fef::indexproperties::hitcollector::SecondPhaseRankScoreDropLimit; using search::queryeval::Blueprint; using search::queryeval::SearchIterator; using vespalib::Doom; @@ -76,30 +78,14 @@ numThreads(size_t hits, size_t minHits) { return static_cast<size_t>(std::ceil(double(hits) / double(minHits))); } -class LimitedThreadBundleWrapper final : public vespalib::ThreadBundle -{ -public: - LimitedThreadBundleWrapper(vespalib::ThreadBundle &threadBundle, uint32_t maxThreads) - : _threadBundle(threadBundle), - _maxThreads(std::min(maxThreads, static_cast<uint32_t>(threadBundle.size()))) - { } - size_t size() const override { return _maxThreads; } - void run(vespalib::Runnable* const* targets, size_t cnt) override { - _threadBundle.run(targets, cnt); - } -private: - vespalib::ThreadBundle &_threadBundle; - const uint32_t _maxThreads; -}; - bool willNeedRanking(const SearchRequest & request, const GroupingContext & groupingContext, - search::feature_t rank_score_drop_limit) + std::optional<search::feature_t> first_phase_rank_score_drop_limit) { return (groupingContext.needRanking() || (request.maxhits != 0)) && (request.sortSpec.empty() || (request.sortSpec.find("[rank]") != vespalib::string::npos) || - !std::isnan(rank_score_drop_limit)); + first_phase_rank_score_drop_limit.has_value()); } SearchReply::UP @@ -289,17 +275,19 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl const Properties & rankProperties = request.propertiesMap.rankProperties(); uint32_t heapSize = HeapSize::lookup(rankProperties, _rankSetup->getHeapSize()); uint32_t arraySize = ArraySize::lookup(rankProperties, _rankSetup->getArraySize()); - search::feature_t rank_score_drop_limit = RankScoreDropLimit::lookup(rankProperties, _rankSetup->getRankScoreDropLimit()); + auto first_phase_rank_score_drop_limit = FirstPhaseRankScoreDropLimit::lookup(rankProperties, _rankSetup->get_first_phase_rank_score_drop_limit()); + auto second_phase_rank_score_drop_limit = SecondPhaseRankScoreDropLimit::lookup(rankProperties, _rankSetup->get_second_phase_rank_score_drop_limit()); - MatchParams params(searchContext.getDocIdLimit(), heapSize, arraySize, rank_score_drop_limit, + MatchParams params(searchContext.getDocIdLimit(), heapSize, arraySize, first_phase_rank_score_drop_limit, + second_phase_rank_score_drop_limit, request.offset, request.maxhits, !_rankSetup->getSecondPhaseRank().empty(), - willNeedRanking(request, groupingContext, rank_score_drop_limit)); + willNeedRanking(request, groupingContext, first_phase_rank_score_drop_limit)); ResultProcessor rp(attrContext, metaStore, sessionMgr, groupingContext, sessionId, request.sortSpec, params.offset, params.hits); size_t numThreadsPerSearch = computeNumThreadsPerSearch(mtf->estimate(), rankProperties); - LimitedThreadBundleWrapper limitedThreadBundle(threadBundle, numThreadsPerSearch); + vespalib::LimitedThreadBundleWrapper limitedThreadBundle(threadBundle, numThreadsPerSearch); MatchMaster master; uint32_t numParts = NumSearchPartitions::lookup(rankProperties, _rankSetup->getNumSearchPartitions()); if (limitedThreadBundle.size() > 1) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 70f60ff1c2d..e0fe8f5beb9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -39,6 +39,7 @@ using search::queryeval::Blueprint; using search::queryeval::GlobalFilter; using search::queryeval::IRequestContext; using search::queryeval::IntermediateBlueprint; +using search::queryeval::MatchingPhase; using search::queryeval::RankBlueprint; using search::queryeval::SearchIterator; using vespalib::Issue; @@ -198,6 +199,12 @@ Query::reserveHandles(const IRequestContext & requestContext, ISearchContext &co } void +Query::enumerate_blueprint_nodes() noexcept +{ + _blueprint->enumerate(1); +} + +void Query::optimize(InFlow in_flow, bool sort_by_cost) { _in_flow = in_flow; @@ -280,6 +287,12 @@ Query::freeze() _blueprint->freeze(); } +void +Query::set_matching_phase(MatchingPhase matching_phase) const noexcept +{ + _blueprint->set_matching_phase(matching_phase); +} + Blueprint::HitEstimate Query::estimate() const { diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index 03aea5a0d2d..32c7b2a91b6 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -97,6 +97,8 @@ public: ISearchContext &context, search::fef::MatchDataLayout &mdl); + void enumerate_blueprint_nodes() noexcept; + /** * Optimize the query to be executed. This function should be * called after the reserveHandles function and before the @@ -130,6 +132,7 @@ public: vespalib::ThreadBundle &thread_bundle, search::engine::Trace* trace); void freeze(); + void set_matching_phase(search::queryeval::MatchingPhase matching_phase) const noexcept; /** * Create the actual search iterator tree used to find matches. diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index f3a764bc4c9..e3857b2d8a9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -126,5 +126,5 @@ vespa_add_library(searchcore_server STATIC searchcore_reprocessing searchcore_summaryengine searchcore_reference - configdefinitions + vespa_configdefinitions ) diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index c72a4eaf352..785ffcd9663 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -90,11 +90,6 @@ RPCHooksBase::initRPC() rb.ReturnDesc("message", "Array of status messages"); rb.RequestAccessFilter(make_proton_admin_api_capability_filter()); //------------------------------------------------------------------------- - rb.DefineMethod("pandora.rtc.die", "", "", - FRT_METHOD(RPCHooksBase::rpc_die), this); - rb.MethodDesc("Exit the rtc application without cleanup"); - rb.RequestAccessFilter(make_proton_admin_api_capability_filter()); - //------------------------------------------------------------------------- rb.DefineMethod("proton.triggerFlush", "", "b", FRT_METHOD(RPCHooksBase::rpc_triggerFlush), this); rb.MethodDesc("Tell the node to trigger flush ASAP"); @@ -242,19 +237,6 @@ RPCHooksBase::getProtonStatus(FRT_RPCRequest *req) } void -RPCHooksBase::rpc_die(FRT_RPCRequest * req) -{ - LOG(debug, "RPCHooksBase::rpc_die"); - req->Detach(); - letProtonDo(makeLambdaTask([req]() { - LOG(debug, "Nap for 10ms and then quickly exit."); - req->Return(); - std::this_thread::sleep_for(10ms); - std::_Exit(0); - })); -} - -void RPCHooksBase::rpc_triggerFlush(FRT_RPCRequest *req) { LOG(info, "RPCHooksBase::rpc_triggerFlush started"); diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h index 0b9329551f5..7f863bc1fc3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h @@ -55,7 +55,6 @@ public: void rpc_GetState(FRT_RPCRequest *req); void rpc_GetProtonStatus(FRT_RPCRequest *req); - void rpc_die(FRT_RPCRequest *req); void rpc_triggerFlush(FRT_RPCRequest *req); void rpc_prepareRestart(FRT_RPCRequest *req); protected: diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h index ce67099dc3e..288f6ebfebd 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> #include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/document/base/globalid.h> diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index a5453ac5273..310152244dc 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -4,11 +4,11 @@ vespa_define_module( vespalog vespalib vespaeval - fnet - configdefinitions - metrics - document - config_cloudconfig + vespa_fnet + vespa_configdefinitions + vespa_metrics + vespa_document + vespa_config EXTERNAL_DEPENDS ${VESPA_GLIBC_RT_LIB} @@ -60,6 +60,7 @@ vespa_define_module( src/apps/vespa-attribute-inspect src/apps/vespa-fileheader-inspect src/apps/vespa-index-inspect + src/apps/vespa-query-analyzer src/apps/vespa-ranking-expression-analyzer TESTS @@ -140,6 +141,7 @@ vespa_define_module( src/tests/features/element_completeness src/tests/features/element_similarity_feature src/tests/features/euclidean_distance + src/tests/features/first_phase_rank src/tests/features/imported_dot_product src/tests/features/internal_max_reduce_prod_join_feature src/tests/features/item_raw_score @@ -197,6 +199,7 @@ vespa_define_module( src/tests/queryeval/blueprint src/tests/queryeval/dot_product src/tests/queryeval/equiv + src/tests/queryeval/exact_nearest_neighbor src/tests/queryeval/fake_searchable src/tests/queryeval/filter_search src/tests/queryeval/flow @@ -206,7 +209,6 @@ vespa_define_module( src/tests/queryeval/matching_elements_search src/tests/queryeval/monitoring_search_iterator src/tests/queryeval/multibitvectoriterator - src/tests/queryeval/nearest_neighbor src/tests/queryeval/or_speed src/tests/queryeval/parallel_weak_and src/tests/queryeval/predicate diff --git a/searchlib/src/apps/docstore/CMakeLists.txt b/searchlib/src/apps/docstore/CMakeLists.txt index f03010f2825..628c1527d0b 100644 --- a/searchlib/src/apps/docstore/CMakeLists.txt +++ b/searchlib/src/apps/docstore/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_executable(searchlib_vespa-verify-logdatastore_app OUTPUT_NAME vespa-verify-logdatastore INSTALL bin DEPENDS - searchlib + vespa_searchlib ) vespa_add_executable(searchlib_vespa-documentstore-inspect_app SOURCES @@ -13,7 +13,7 @@ vespa_add_executable(searchlib_vespa-documentstore-inspect_app OUTPUT_NAME vespa-documentstore-inspect INSTALL bin DEPENDS - searchlib + vespa_searchlib ) vespa_add_executable(searchlib_vespa-documentstore-benchmark_app SOURCES @@ -21,7 +21,7 @@ vespa_add_executable(searchlib_vespa-documentstore-benchmark_app OUTPUT_NAME vespa-documentstore-benchmark INSTALL bin DEPENDS - searchlib + vespa_searchlib ) vespa_add_executable(searchlib_vespa-create-idx-from-dat_app SOURCES @@ -29,5 +29,5 @@ vespa_add_executable(searchlib_vespa-create-idx-from-dat_app OUTPUT_NAME vespa-create-idx-from-dat INSTALL bin DEPENDS - searchlib + vespa_searchlib ) diff --git a/searchlib/src/apps/tests/CMakeLists.txt b/searchlib/src/apps/tests/CMakeLists.txt index af48f379f37..2fb8c666cdf 100644 --- a/searchlib/src/apps/tests/CMakeLists.txt +++ b/searchlib/src/apps/tests/CMakeLists.txt @@ -3,21 +3,22 @@ vespa_add_executable(searchlib_biglog_test_app SOURCES biglogtest.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_biglog_test_app COMMAND searchlib_biglog_test_app BENCHMARK) vespa_add_executable(searchlib_memoryindexstress_test_app SOURCES memoryindexstress_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_memoryindexstress_test_app COMMAND searchlib_memoryindexstress_test_app BENCHMARK) vespa_add_executable(searchlib_document_weight_attribute_lookup_stress_test_app SOURCES document_weight_attribute_lookup_stress_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_document_weight_attribute_lookup_stress_test_app COMMAND searchlib_document_weight_attribute_lookup_stress_test_app BENCHMARK) diff --git a/searchlib/src/apps/tests/biglogtest.cpp b/searchlib/src/apps/tests/biglogtest.cpp index 6106a0b7585..9f5fa581549 100644 --- a/searchlib/src/apps/tests/biglogtest.cpp +++ b/searchlib/src/apps/tests/biglogtest.cpp @@ -1,62 +1,70 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/random.h> #include <vespa/searchlib/docstore/logdatastore.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/nosyncproxy.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/data/databuffer.h> #include <filesystem> +#include <memory> using namespace search; using search::index::DummyFileHeaderContext; -class Test : public vespalib::TestApp { +class BigLogTest : public ::testing::Test { private: struct Blob { ssize_t sz; - char *buf; - Blob(size_t s) : sz(s), buf(s == 0 ? 0 : new char[s]) {} + std::unique_ptr<char[]> buf; + Blob(size_t s) : sz(s), buf(s == 0 ? nullptr : new char[s]) {} }; using Map = std::map<uint32_t, uint32_t>; - void makeBlobs(); - void cleanBlobs(); + static void makeBlobs(); + static void cleanBlobs(); void checkBlobs(const IDataStore &datastore, const Map &lidToBlobMap); + std::string _dir; + static std::vector<Blob> _blobs; + static vespalib::RandomGen _randomgenerator; + +protected: template <typename DS> void testDIO(); - std::string _dir; - std::vector<Blob> _blobs; - vespalib::RandomGen _randomgenerator; - public: - int Main() override { - TEST_INIT("big_logdatastore_test"); - - if (_argc > 0) { - DummyFileHeaderContext::setCreator(_argv[0]); - } - makeBlobs(); + BigLogTest(); + ~BigLogTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; - _dir = "logged"; - TEST_DO(testDIO<LogDataStore>()); +std::vector<BigLogTest::Blob> BigLogTest::_blobs; +vespalib::RandomGen BigLogTest::_randomgenerator(42); - cleanBlobs(); +BigLogTest::BigLogTest() + : _dir("logged") +{ +} - TEST_DONE(); - } +BigLogTest::~BigLogTest() = default; - Test() : _dir(""), _blobs(), _randomgenerator(42) {} - ~Test() {} -}; +void +BigLogTest::SetUpTestSuite() +{ + makeBlobs(); +} -TEST_APPHOOK(Test); +void +BigLogTest::TearDownTestSuite() +{ + cleanBlobs(); +} void -Test::makeBlobs() +BigLogTest::makeBlobs() { _randomgenerator.setSeed(42); _blobs.push_back(Blob(0)); @@ -66,7 +74,7 @@ Test::makeBlobs() size_t blobsize = _randomgenerator.nextUint32() % (1<<sizeclass); if (blobsize > usemem) blobsize = usemem; _blobs.push_back(Blob(blobsize)); - char *p = _blobs.back().buf; + char *p = _blobs.back().buf.get(); for (size_t j=0; j < blobsize; ++j) { *p++ = _randomgenerator.nextUint32(); } @@ -75,22 +83,19 @@ Test::makeBlobs() } void -Test::cleanBlobs() +BigLogTest::cleanBlobs() { printf("count %lu blobs sizes:", _blobs.size()); while (_blobs.size() > 0) { - char *p = _blobs.back().buf; printf(" %lu", _blobs.back().sz); - delete[] p; _blobs.pop_back(); } printf("\n"); } - void -Test::checkBlobs(const IDataStore &datastore, - const Map &lidToBlobMap) +BigLogTest::checkBlobs(const IDataStore &datastore, + const Map &lidToBlobMap) { for (Map::const_iterator it = lidToBlobMap.begin(); it != lidToBlobMap.end(); @@ -99,8 +104,8 @@ Test::checkBlobs(const IDataStore &datastore, uint32_t lid = it->first; uint32_t bno = it->second; vespalib::DataBuffer got; - EXPECT_EQUAL(datastore.read(lid, got), _blobs[bno].sz); - EXPECT_TRUE(memcmp(got.getData(), _blobs[bno].buf, _blobs[bno].sz) == 0); + EXPECT_EQ(_blobs[bno].sz, datastore.read(lid, got)); + EXPECT_EQ(0, memcmp(got.getData(), _blobs[bno].buf.get(), _blobs[bno].sz)); } } @@ -144,7 +149,7 @@ factory<LogDataStore>::~factory() {} template <typename DS> void -Test::testDIO() +BigLogTest::testDIO() { uint64_t serial = 0; @@ -158,14 +163,14 @@ Test::testDIO() for (uint32_t lid=0; lid<15; ++lid) { uint32_t blobno = _randomgenerator.nextUint32() % _blobs.size(); lidToBlobMap[lid] = blobno; - ds().write(++serial, lid, _blobs[blobno].buf, _blobs[blobno].sz); + ds().write(++serial, lid, _blobs[blobno].buf.get(), _blobs[blobno].sz); } uint64_t flushToken = ds().initFlush(serial); ds().flush(flushToken); for (uint32_t lid=10; lid<30; ++lid) { uint32_t blobno = _randomgenerator.nextUint32() % _blobs.size(); lidToBlobMap[lid] = blobno; - ds().write(++serial, lid, _blobs[blobno].buf, _blobs[blobno].sz); + ds().write(++serial, lid, _blobs[blobno].buf.get(), _blobs[blobno].sz); } checkBlobs(ds(), lidToBlobMap); flushToken = ds().initFlush(serial); @@ -179,17 +184,17 @@ Test::testDIO() for (uint32_t lid=3; lid<8; ++lid) { uint32_t blobno = _randomgenerator.nextUint32() % _blobs.size(); lidToBlobMap[lid] = blobno; - ds().write(++serial, lid, _blobs[blobno].buf, _blobs[blobno].sz); + ds().write(++serial, lid, _blobs[blobno].buf.get(), _blobs[blobno].sz); } for (uint32_t lid=23; lid<28; ++lid) { uint32_t blobno = _randomgenerator.nextUint32() % _blobs.size(); lidToBlobMap[lid] = blobno; - ds().write(++serial, lid, _blobs[blobno].buf, _blobs[blobno].sz); + ds().write(++serial, lid, _blobs[blobno].buf.get(), _blobs[blobno].sz); } for (uint32_t lid=100033; lid<100088; ++lid) { uint32_t blobno = _randomgenerator.nextUint32() % _blobs.size(); lidToBlobMap[lid] = blobno; - ds().write(++serial, lid, _blobs[blobno].buf, _blobs[blobno].sz); + ds().write(++serial, lid, _blobs[blobno].buf.get(), _blobs[blobno].sz); } checkBlobs(ds(), lidToBlobMap); @@ -226,7 +231,7 @@ Test::testDIO() for (uint32_t lid=1234567; lid < 1234999; ++lid) { uint32_t blobno = _randomgenerator.nextUint32() % _blobs.size(); lidToBlobMap[lid] = blobno; - ds().write(++serial, lid, _blobs[blobno].buf, _blobs[blobno].sz); + ds().write(++serial, lid, _blobs[blobno].buf.get(), _blobs[blobno].sz); } checkBlobs(ds(), lidToBlobMap); uint64_t flushToken = ds().initFlush(22); @@ -238,5 +243,19 @@ Test::testDIO() checkBlobs(ds(), lidToBlobMap); } std::filesystem::remove_all(std::filesystem::path(_dir)); - TEST_FLUSH(); +} + +TEST_F(BigLogTest, logdatastore_dio) +{ + testDIO<LogDataStore>(); +} + +int +main(int argc, char* argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + if (argc > 0) { + DummyFileHeaderContext::setCreator(argv[0]); + } + return RUN_ALL_TESTS(); } diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 763ba860fe6..b9c80a3c04d 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -21,7 +21,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/vespalib/util/rand48.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchlib/src/apps/uniform/CMakeLists.txt b/searchlib/src/apps/uniform/CMakeLists.txt index 8cdb17c4265..4fc3115ae3f 100644 --- a/searchlib/src/apps/uniform/CMakeLists.txt +++ b/searchlib/src/apps/uniform/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_executable(searchlib_uniform_app SOURCES uniform.cpp DEPENDS - searchlib + vespa_searchlib ) diff --git a/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt b/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt index dcc8350acfc..110babec765 100644 --- a/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt +++ b/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt @@ -5,5 +5,5 @@ vespa_add_executable(searchlib_vespa-attribute-inspect_app OUTPUT_NAME vespa-attribute-inspect INSTALL bin DEPENDS - searchlib + vespa_searchlib ) diff --git a/searchlib/src/apps/vespa-fileheader-inspect/CMakeLists.txt b/searchlib/src/apps/vespa-fileheader-inspect/CMakeLists.txt index 629f20e6d5b..f105bdba31b 100644 --- a/searchlib/src/apps/vespa-fileheader-inspect/CMakeLists.txt +++ b/searchlib/src/apps/vespa-fileheader-inspect/CMakeLists.txt @@ -5,5 +5,5 @@ vespa_add_executable(searchlib_vespa-fileheader-inspect_app OUTPUT_NAME vespa-fileheader-inspect INSTALL bin DEPENDS - searchlib + vespa_searchlib ) diff --git a/searchlib/src/apps/vespa-index-inspect/CMakeLists.txt b/searchlib/src/apps/vespa-index-inspect/CMakeLists.txt index 91a277e8489..4dbe08df902 100644 --- a/searchlib/src/apps/vespa-index-inspect/CMakeLists.txt +++ b/searchlib/src/apps/vespa-index-inspect/CMakeLists.txt @@ -5,5 +5,5 @@ vespa_add_executable(searchlib_vespa-index-inspect_app OUTPUT_NAME vespa-index-inspect INSTALL bin DEPENDS - searchlib + vespa_searchlib ) diff --git a/searchlib/src/apps/vespa-query-analyzer/.gitignore b/searchlib/src/apps/vespa-query-analyzer/.gitignore new file mode 100644 index 00000000000..e5a31caab09 --- /dev/null +++ b/searchlib/src/apps/vespa-query-analyzer/.gitignore @@ -0,0 +1,3 @@ +/.depend +/Makefile +/vespa-query-analyzer diff --git a/searchlib/src/apps/vespa-query-analyzer/CMakeLists.txt b/searchlib/src/apps/vespa-query-analyzer/CMakeLists.txt new file mode 100644 index 00000000000..cf6b6f3e56f --- /dev/null +++ b/searchlib/src/apps/vespa-query-analyzer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_vespa-query-analyzer_app + SOURCES + vespa-query-analyzer.cpp + OUTPUT_NAME vespa-query-analyzer + INSTALL bin + DEPENDS + vespa_searchlib +) diff --git a/searchlib/src/apps/vespa-query-analyzer/vespa-query-analyzer.cpp b/searchlib/src/apps/vespa-query-analyzer/vespa-query-analyzer.cpp new file mode 100644 index 00000000000..6a746d59daa --- /dev/null +++ b/searchlib/src/apps/vespa-query-analyzer/vespa-query-analyzer.cpp @@ -0,0 +1,675 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/io/mapped_file_input.h> +#include <vespa/vespalib/util/overload.h> +#include <vespa/vespalib/util/signalhandler.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/searchlib/queryeval/flow.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> +#include <optional> +#include <variant> +#include <vector> +#include <map> + +using namespace vespalib::slime::convenience; +using vespalib::make_string_short::fmt; +using namespace search::queryeval; +using namespace vespalib::slime; +using namespace vespalib; + +//----------------------------------------------------------------------------- + +int rel_diff(double a, double b, double e, double m) { + int res = 0; + if (a < e && b < e) { + return res; + } + double x = std::abs(b - a) / std::max(std::min(a, b), e); + while (x > m && res < 10) { + x /= 10.0; + ++res; + } + return res; +} + +void apply_diff(vespalib::string &str, int diff, char small, char big, int len) { + for (int i = 0; i < diff && i < len; ++i) { + if (diff + i >= len * 2) { + str.append(big); + } else { + str.append(small); + } + } +} + +using Path = std::vector<std::variant<size_t,vespalib::stringref>>; +using Paths = std::vector<Path>; + +template <typename F> +struct Matcher : vespalib::slime::ObjectTraverser { + Path path; + Paths result; + F match; + ~Matcher(); + Matcher(F match_in) noexcept : path(), result(), match(match_in) {} + void search(const Inspector &node) { + if (path.empty() && match(path, node)) { + result.push_back(path); + } + if (node.type() == OBJECT()) { + node.traverse(*this); + } + if (node.type() == ARRAY()) { + size_t size = node.entries(); + for (size_t i = 0; i < size; ++i) { + path.emplace_back(i); + if (match(path, node[i])) { + result.push_back(path); + } + search(node[i]); + path.pop_back(); + } + } + } + void field(const Memory &symbol, const Inspector &inspector) final { + path.emplace_back(symbol.make_stringref()); + if (match(path, inspector)) { + result.push_back(path); + } + search(inspector); + path.pop_back(); + } +}; +template <typename F> Matcher<F>::~Matcher() = default; + +std::vector<Path> find_field(const Inspector &root, const vespalib::string &name) { + auto matcher = Matcher([&](const Path &path, const Inspector &){ + return ((path.size() > 0) && + (std::holds_alternative<vespalib::stringref>(path.back())) && + (std::get<vespalib::stringref>(path.back()) == name)); + }); + matcher.search(root); + return matcher.result; +} + +std::vector<Path> find_tag(const Inspector &root, const vespalib::string &name) { + auto matcher = Matcher([&](const Path &path, const Inspector &value){ + return ((path.size() > 0) && + (std::holds_alternative<vespalib::stringref>(path.back())) && + (std::get<vespalib::stringref>(path.back()) == "tag") && + (value.asString().make_stringref() == name)); + }); + matcher.search(root); + return matcher.result; +} + +vespalib::string path_to_str(const Path &path) { + size_t cnt = 0; + vespalib::string str("["); + for (const auto &item: path) { + if (cnt++ > 0) { + str.append(","); + } + std::visit(vespalib::overload{ + [&str](size_t value)noexcept{ str.append(fmt("%zu", value)); }, + [&str](vespalib::stringref value)noexcept{ str.append(value); }}, item); + } + str.append("]"); + return str; +} + +vespalib::string strip_name(vespalib::stringref name) { + auto end = name.find("<"); + auto ns = name.rfind("::", end); + size_t begin = (ns > name.size()) ? 0 : ns + 2; + return name.substr(begin, end - begin); +} + +const Inspector &apply_path(const Inspector &node, const Path &path, size_t max = -1) { + size_t cnt = 0; + const Inspector *ptr = &node; + for (const auto &elem: path) { + if (cnt++ >= max) { + return *ptr; + } + if (std::holds_alternative<size_t>(elem)) { + ptr = &((*ptr)[std::get<size_t>(elem)]); + } + if (std::holds_alternative<vespalib::stringref>(elem)) { + auto ref = std::get<vespalib::stringref>(elem); + ptr = &((*ptr)[Memory(ref.data(), ref.size())]); + } + } + return *ptr; +} + +void extract(vespalib::string &value, const Inspector &data) { + if (data.valid() && data.type() == STRING()) { + value = data.asString().make_stringref(); + } +} + +struct Sample { + enum class Type { INVALID, INIT, SEEK, UNPACK, TERMWISE }; + Type type = Type::INVALID; + std::vector<size_t> path; + double self_time_ms = 0.0; + double total_time_ms = 0.0; + size_t count = 0; + Sample(const Inspector &sample) { + auto name = sample["name"].asString().make_stringref(); + if (ends_with(name, "/init")) { + type = Type::INIT; + } + if (ends_with(name, "/seek")) { + type = Type::SEEK; + } + if (ends_with(name, "/unpack")) { + type = Type::UNPACK; + } + if (ends_with(name, "/termwise")) { + type = Type::TERMWISE; + } + if (starts_with(name, "/")) { + size_t child = 0; + for (size_t pos = 1; pos < name.size(); ++pos) { + char c = name[pos]; + if (c == '/') { + path.push_back(child); + child = 0; + } else { + if (c < '0' || c > '9') { + break; + } + child = child * 10 + (c - '0'); + } + } + } + count = sample["count"].asLong(); + total_time_ms = sample["total_time_ms"].asDouble(); + const Inspector &self = sample["self_time_ms"]; + if (self.valid()) { + self_time_ms = self.asDouble(); + } else { + // Self time is not reported for leaf nodes. Make sure + // profile depth is high enough to not clip the tree + // before reaching actual leafs. + self_time_ms = total_time_ms; + } + } + static vespalib::string type_to_str(Type type) { + switch(type) { + case Type::INVALID: return "<invalid>"; + case Type::INIT: return "init"; + case Type::SEEK: return "seek"; + case Type::UNPACK: return "unpack"; + case Type::TERMWISE: return "termwise"; + } + abort(); + } + static vespalib::string path_to_str(const std::vector<size_t> &path) { + vespalib::string result("/"); + for (size_t elem: path) { + result += fmt("%zu/", elem); + } + return result; + } + vespalib::string to_string() const { + return fmt("type: %s, path: %s, count: %zu, total_time_ms: %g\n", + type_to_str(type).c_str(), path_to_str(path).c_str(), count, total_time_ms); + } +}; + +struct BlueprintMeta { + static AnyFlow no_flow(InFlow) { abort(); } + static double no_self_cost(double, size_t) { return 0.0; } + struct MetaEntry { + std::function<AnyFlow(InFlow)> make_flow; + std::function<double(double, size_t)> self_cost_strict; + std::function<double(double, size_t)> self_cost_non_strict; + MetaEntry() + : make_flow(no_flow), + self_cost_strict(no_self_cost), + self_cost_non_strict(no_self_cost) {} + MetaEntry(std::function<AnyFlow(InFlow)> make_flow_in) + : make_flow(make_flow_in), + self_cost_strict(no_self_cost), + self_cost_non_strict(no_self_cost) {} + MetaEntry(std::function<AnyFlow(InFlow)> make_flow_in, + std::function<double(double, size_t)> self_cost_strict_in) + : make_flow(make_flow_in), + self_cost_strict(self_cost_strict_in), + self_cost_non_strict(no_self_cost) {} + MetaEntry(std::function<AnyFlow(InFlow)> make_flow_in, + std::function<double(double, size_t)> self_cost_strict_in, + std::function<double(double, size_t)> self_cost_non_strict_in) + : make_flow(make_flow_in), + self_cost_strict(self_cost_strict_in), + self_cost_non_strict(self_cost_non_strict_in) {} + ~MetaEntry(); + }; + std::map<vespalib::string,MetaEntry> map; + BlueprintMeta() { + map["AndNotBlueprint"] = MetaEntry{ + [](InFlow in_flow)noexcept{ return AnyFlow::create<AndNotFlow>(in_flow); } + }; + map["AndBlueprint"] = MetaEntry{ + [](InFlow in_flow)noexcept{ return AnyFlow::create<AndFlow>(in_flow); } + }; + map["OrBlueprint"] = MetaEntry{ + [](InFlow in_flow)noexcept{ return AnyFlow::create<OrFlow>(in_flow); }, + [](double est, size_t n)noexcept{ return flow::heap_cost(est, n); } + }; + map["WeakAndBlueprint"] = MetaEntry{ + [](InFlow in_flow)noexcept{ return AnyFlow::create<OrFlow>(in_flow); }, + [](double est, size_t n)noexcept{ return flow::heap_cost(est, n); } + }; + map["NearBlueprint"] = MetaEntry{ + [](InFlow in_flow)noexcept{ return AnyFlow::create<AndFlow>(in_flow); }, + [](double est, size_t n)noexcept{ return est * n; }, + [](double est, size_t n)noexcept{ return est * n; } + }; + map["ONearBlueprint"] = MetaEntry{ + [](InFlow in_flow)noexcept{ return AnyFlow::create<AndFlow>(in_flow); }, + [](double est, size_t n)noexcept{ return est * n; }, + [](double est, size_t n)noexcept{ return est * n; } + }; + map["RankBlueprint"] = MetaEntry{ + [](InFlow in_flow)noexcept{ return AnyFlow::create<RankFlow>(in_flow); } + }; + map["SourceBlenderBlueprint"] = MetaEntry{ + [](InFlow in_flow)noexcept{ return AnyFlow::create<BlenderFlow>(in_flow); }, + [](double est, size_t)noexcept{ return est; }, + [](double, size_t)noexcept{ return 1.0; } + }; + } + bool is_known(const vespalib::string &type) { + return map.find(type) != map.end(); + } + const MetaEntry &lookup(const vespalib::string &type) { + return map.find(type)->second; + } +}; +BlueprintMeta::MetaEntry::~MetaEntry() = default; +BlueprintMeta blueprint_meta; + +struct Node { + vespalib::string type = "unknown"; + uint32_t id = 0; + uint32_t docid_limit = 0; + vespalib::string field_name; + vespalib::string query_term; + bool strict = false; + FlowStats flow_stats = FlowStats(0.0, 0.0, 0.0); + size_t count = 0; + double self_time_ms = 0.0; + double total_time_ms = 0.0; + double est_seek = 0.0; + double est_cost = 0.0; + char seek_type = '?'; + double ms_per_cost = 0.0; + double ms_self_limit = 0.0; + double ms_limit = 0.0; + std::vector<Node> children; + Node(const Inspector &obj) { + extract(type, obj["[type]"]); + type = strip_name(type); + id = obj["id"].asLong(); + docid_limit = obj["docid_limit"].asLong(); + query_term = obj["query_term"].asString().make_stringref(); + if (query_term.size() > 0) { + const Inspector &attr = obj["attribute"]; + if (attr.valid()) { + field_name = attr["name"].asString().make_stringref(); + if (type == "AttributeFieldBlueprint") { + type = fmt("Attribute{%s,%s}", + attr["type"].asString().make_string().c_str(), + attr["fast_search"].asBool() ? "fs" : "lookup"); + } + } else { + field_name = obj["field_name"].asString().make_stringref(); + if (type == "DiskTermBlueprint") { + type = "DiskTerm"; + } + if (type == "MemoryTermBlueprint") { + type = "MemoryTerm"; + } + } + } + strict = obj["strict"].asBool(); + flow_stats.estimate = obj["relative_estimate"].asDouble(); + flow_stats.cost = obj["cost"].asDouble(); + flow_stats.strict_cost = obj["strict_cost"].asDouble(); + const Inspector &list = obj["children"]; + for (size_t i = 0; true; ++i) { + const Inspector &child = list[fmt("[%zu]", i)]; + if (child.valid()) { + children.emplace_back(child); + } else { + break; + } + } + } + ~Node(); + vespalib::string name() const { + vespalib::string res = type; + if (id > 0) { + res.append(fmt("[%u]", id)); + } + if (query_term.size() > 0) { + if (field_name.size() > 0) { + res.append(fmt(" %s:%s", field_name.c_str(), query_term.c_str())); + } else { + res.append(fmt(" %s", query_term.c_str())); + } + } + return res; + } + double rel_count() const { + return double(count) / docid_limit; + } + size_t abs_est_seek() const { + return double(docid_limit) * est_seek; + } + void add_sample(const Sample &sample) { + Node *node = this; + for (size_t child: sample.path) { + if (child < node->children.size()) { + node = &node->children[child]; + } else { + fprintf(stderr, "... ignoring bad sample: %s\n", sample.to_string().c_str()); + return; + } + } + node->count += sample.count; + node->self_time_ms += sample.self_time_ms; + node->total_time_ms += sample.total_time_ms; + } + void each_node(auto f) { + f(*this); + for (auto &child: children) { + child.each_node(f); + } + } + void calc_cost(InFlow in_flow) { + if (!children.empty() && !blueprint_meta.is_known(type)) { + fprintf(stderr, "... blueprint meta-data not found for intermediate node: %s (treating as leaf)\n", name().c_str()); + } + if (children.empty() || !blueprint_meta.is_known(type)) { + if (in_flow.strict()) { + if (!strict) { + fprintf(stderr, "... invalid strictness for node: %s\n", name().c_str()); + } + est_seek = flow_stats.estimate; + est_cost = flow_stats.strict_cost; + seek_type = 'S'; + } else if (strict) { + est_seek = in_flow.rate(); + est_cost = flow::forced_strict_cost(flow_stats, est_seek); + seek_type = 'F'; + } else { + est_seek = in_flow.rate(); + est_cost = est_seek * flow_stats.cost; + seek_type = 'N'; + } + } else { + double cost_diff = 0.0; + double seek_diff = 0.0; + const auto &meta = blueprint_meta.lookup(type); + if (in_flow.strict()) { + if (!strict) { + fprintf(stderr, "... invalid strictness for node: %s\n", name().c_str()); + } + est_seek = flow_stats.estimate; + seek_type = 'S'; + } else if (strict) { + cost_diff = flow::strict_cost_diff(flow_stats.estimate, in_flow.rate()); + seek_diff = in_flow.rate() - flow_stats.estimate; + est_seek = in_flow.rate(); + in_flow.force_strict(); + seek_type = 'F'; + } else { + est_seek = in_flow.rate(); + seek_type = 'N'; + } + double flow_cost = 0.0; + auto flow = meta.make_flow(in_flow); + for (auto &child: children) { + child.calc_cost(InFlow(flow.strict(), flow.flow())); + flow.update_cost(flow_cost, child.est_cost); + flow.add(child.flow_stats.estimate); + } + est_cost = flow_cost + cost_diff; + if (in_flow.strict()) { + est_cost += meta.self_cost_strict(flow_stats.estimate, children.size()); + } else { + est_cost += est_seek * meta.self_cost_non_strict(flow_stats.estimate, children.size()); + } + if (seek_diff < 0.0) { + // adjust est_seek for sub-tree + each_node([factor = est_seek / (est_seek - seek_diff)](Node &node)noexcept{ + node.est_seek *= factor; + }); + } + if (cost_diff < 0.0) { + // adjust est_cost for sub-tree + each_node([factor = est_cost / (est_cost - cost_diff)](Node &node)noexcept{ + node.est_cost *= factor; + }); + } + } + } + void normalize() { + size_t num_nodes = 0; + double cost_limit = est_cost * 0.01; + double time_limit = total_time_ms * 0.01; + std::vector<double> samples; + each_node([&](Node &node){ + ++num_nodes; + if (node.est_cost >= cost_limit) { + samples.push_back(node.total_time_ms / node.est_cost); + } + }); + double self_time_limit = total_time_ms * 10.0 / num_nodes; + double norm_ms_per_cost = samples[samples.size()/2]; + each_node([&](Node &node)noexcept{ + node.ms_per_cost = norm_ms_per_cost; + node.ms_self_limit = self_time_limit; + node.ms_limit = time_limit; + }); + } + vespalib::string tingle() const { + vespalib::string res; + if (total_time_ms > ms_limit) { + apply_diff(res, rel_diff(est_seek, rel_count(), 1e-6, 0.50), 's', 'S', 3); + apply_diff(res, rel_diff(ms_per_cost * est_cost, total_time_ms, 1e-3, 0.50), 't', 'T', 3); + if (self_time_ms > ms_self_limit) { + apply_diff(res, rel_diff(self_time_ms, ms_self_limit, 1e-3, 0.01), '+', '*', 1); + } + } + return res; + } + void print_header() const { + fprintf(stdout, "|%10s ", "seeks"); + fprintf(stdout, "|%10s ", "est_seeks"); + fprintf(stdout, "|%11s ", "time_ms"); + fprintf(stdout, "|%11s ", "est_time"); + fprintf(stdout, "|%10s ", "self_ms"); + fprintf(stdout, "|%8s ", "tingle"); + fprintf(stdout, "|%5s ", "step"); + fprintf(stdout, "|\n"); + } + void print_separator() const { + const char *fill = "-------------------------------------------"; + fprintf(stdout, "+%.10s-", fill); + fprintf(stdout, "+%.10s-", fill); + fprintf(stdout, "+%.11s-", fill); + fprintf(stdout, "+%.11s-", fill); + fprintf(stdout, "+%.10s-", fill); + fprintf(stdout, "+%.8s-", fill); + fprintf(stdout, "+%.5s-", fill); + fprintf(stdout, "+\n"); + } + void print_stats() const { + fprintf(stdout, "|%10zu ", count); + fprintf(stdout, "|%10zu ", abs_est_seek()); + fprintf(stdout, "|%11.3f ", total_time_ms); + fprintf(stdout, "|%11.3f ", ms_per_cost * est_cost); + fprintf(stdout, "|%10.3f ", self_time_ms); + fprintf(stdout, "|%8s ", tingle().c_str()); + fprintf(stdout, "|%5c ", seek_type); + fprintf(stdout, "| "); + } + static constexpr const char *pads[4] = {" ├─ "," │ "," └─ "," "}; + void print_line(const vespalib::string &prefix, const char *pad_self, const char *pad_child) const { + print_stats(); + fprintf(stdout, "%s%s%s\n", prefix.c_str(), pad_self, name().c_str()); + for (size_t i = 0; i < children.size(); ++i) { + auto *my_pads = ((i + 1) < children.size()) ? pads : pads + 2; + children[i].print_line(prefix + pad_child, my_pads[0], my_pads[1]); + } + } + void print() const { + print_separator(); + print_header(); + print_separator(); + print_line("", "", ""); + print_separator(); + } +}; +Node::~Node() = default; + +void each_sample_list(const Inspector &list, auto f) { + for (size_t i = 0; i < list.entries(); ++i) { + f(Sample(list[i])); + each_sample_list(list[i]["children"], f); + } +} + +void each_sample(const Inspector &prof, auto f) { + each_sample_list(prof["roots"], f); +} + +struct Analyzer { + void analyze(const Inspector &root) { + auto bp_list = find_field(root, "optimized"); + for (const Path &path: bp_list) { + const Inspector &node = apply_path(root, path, path.size()-3); + const Inspector &key_field = node["distribution-key"]; + if (key_field.valid()) { + int key = key_field.asLong(); + Node data(apply_path(root, path)); + auto prof_list = find_tag(node, "match_profiling"); + double total_ms = 0.0; + std::map<Sample::Type,double> time_map; + for (const Path &prof_path: prof_list) { + const Inspector &prof = apply_path(node, prof_path, prof_path.size()-1); + if (prof["profiler"].asString().make_stringref() == "tree") { + total_ms += prof["total_time_ms"].asDouble(); + each_sample(prof, [&](const Sample &sample) { + if (sample.type == Sample::Type::SEEK) { + data.add_sample(sample); + } + if (sample.path.empty()) { + time_map[sample.type] += sample.total_time_ms; + } + }); + } + } + data.calc_cost(true); + data.normalize(); + data.print(); + fprintf(stdout, "distribution key: %d, total_time_ms: %g, estimated ms_per_cost: %g\n", key, total_ms, data.ms_per_cost); + for (auto [type, time]: time_map) { + fprintf(stdout, "sample type %s used %g ms total\n", Sample::type_to_str(type).c_str(), time); + } + } + } + } +}; + +//----------------------------------------------------------------------------- + +void usage(const char *self) { + fprintf(stderr, "usage: %s <json query result file>\n", self); + fprintf(stderr, " analyze query cost (planning vs profiling)\n"); + fprintf(stderr, " query result must contain optimized blueprint dump\n"); + fprintf(stderr, " query result must contain match phase tree profiling\n\n"); +} + +struct MyApp { + Analyzer analyzer; + vespalib::string file_name; + bool parse_params(int argc, char **argv); + int main(); +}; + +bool +MyApp::parse_params(int argc, char **argv) { + if (argc != 2) { + return false; + } + file_name = argv[1]; + return true; +} + +class StdIn : public Input { +private: + bool _eof = false; + SimpleBuffer _input; +public: + Memory obtain() override { + if ((_input.get().size == 0) && !_eof) { + WritableMemory buf = _input.reserve(4096); + ssize_t res = read(STDIN_FILENO, buf.data, buf.size); + _eof = (res == 0); + assert(res >= 0); // fail on stdio read errors + _input.commit(res); + } + return _input.obtain(); + } + Input &evict(size_t bytes) override { + _input.evict(bytes); + return *this; + } +}; + +int +MyApp::main() +{ + Slime slime; + std::unique_ptr<Input> input; + if (file_name == "-") { + input = std::make_unique<StdIn>(); + } else { + auto file = std::make_unique<MappedFileInput>(file_name); + if (!file->valid()) { + fprintf(stderr, "could not read input file: '%s'\n", + file_name.c_str()); + return 1; + } + input = std::move(file); + } + if(JsonFormat::decode(*input, slime) == 0) { + fprintf(stderr, "input contains invalid json (%s)\n", + file_name.c_str()); + return 1; + } + analyzer.analyze(slime.get()); + return 0; +} + +int main(int argc, char **argv) { + MyApp my_app; + vespalib::SignalHandler::PIPE.ignore(); + if (!my_app.parse_params(argc, argv)) { + usage(argv[0]); + return 1; + } + return my_app.main(); +} + +//----------------------------------------------------------------------------- diff --git a/searchlib/src/apps/vespa-ranking-expression-analyzer/CMakeLists.txt b/searchlib/src/apps/vespa-ranking-expression-analyzer/CMakeLists.txt index a8ffd8af277..39e0b3a447a 100644 --- a/searchlib/src/apps/vespa-ranking-expression-analyzer/CMakeLists.txt +++ b/searchlib/src/apps/vespa-ranking-expression-analyzer/CMakeLists.txt @@ -5,5 +5,5 @@ vespa_add_executable(searchlib_vespa-ranking-expression-analyzer_app OUTPUT_NAME vespa-ranking-expression-analyzer INSTALL bin DEPENDS - searchlib + vespa_searchlib ) diff --git a/searchlib/src/tests/aggregator/CMakeLists.txt b/searchlib/src/tests/aggregator/CMakeLists.txt index 9e65c82fe43..c21b072f27e 100644 --- a/searchlib/src/tests/aggregator/CMakeLists.txt +++ b/searchlib/src/tests/aggregator/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(searchlib_perdocexpr_test_app TEST SOURCES perdocexpr_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_perdocexpr_test_app COMMAND searchlib_perdocexpr_test_app) vespa_add_executable(searchlib_attr_test_app TEST SOURCES attr_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_attr_test_app COMMAND searchlib_attr_test_app) diff --git a/searchlib/src/tests/aggregator/perdocexpr_test.cpp b/searchlib/src/tests/aggregator/perdocexpr_test.cpp index e9f0981739c..61c5a4f8de9 100644 --- a/searchlib/src/tests/aggregator/perdocexpr_test.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/objects/objectdumper.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> diff --git a/searchlib/src/tests/alignment/CMakeLists.txt b/searchlib/src/tests/alignment/CMakeLists.txt index 49b1eb1adc5..6bc48a7398d 100644 --- a/searchlib/src/tests/alignment/CMakeLists.txt +++ b/searchlib/src/tests/alignment/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_alignment_test_app TEST SOURCES alignment_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_alignment_test_app COMMAND searchlib_alignment_test_app) diff --git a/searchlib/src/tests/alignment/alignment_test.cpp b/searchlib/src/tests/alignment/alignment_test.cpp index f155c12ba39..e1c6c217de5 100644 --- a/searchlib/src/tests/alignment/alignment_test.cpp +++ b/searchlib/src/tests/alignment/alignment_test.cpp @@ -4,7 +4,7 @@ LOG_SETUP("alignment_test"); #include <sys/resource.h> #include <sys/time.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/memory.h> @@ -26,8 +26,6 @@ struct Timer { } }; -TEST_SETUP(Test); - double timeAccess(void *bufp, uint32_t len, double &sum) { @@ -43,10 +41,7 @@ timeAccess(void *bufp, uint32_t len, double &sum) return ret; } -int -Test::Main() -{ - TEST_INIT("alignment_test"); +TEST("alignment_test") { uint32_t buf[129]; for (uint32_t i = 0; i < 129; ++i) { @@ -66,6 +61,6 @@ Test::Main() printf("warmup time = %.2f\n", timeAccess(reinterpret_cast<void*>(&buf[1]), 64, foo)); printf("real time = %.2f\n", timeAccess(reinterpret_cast<void*>(&buf[1]), 64, bar)); EXPECT_EQUAL(foo, bar); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/CMakeLists.txt b/searchlib/src/tests/attribute/CMakeLists.txt index cdda9fea8d0..cff19430e6b 100644 --- a/searchlib/src/tests/attribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attribute_test_app TEST SOURCES attribute_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_attribute_test_app COMMAND searchlib_attribute_test_app COST 250) diff --git a/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt b/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt index bde76137b1f..c5c14efa994 100644 --- a/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt +++ b/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attribute_header_test_app TEST SOURCES attribute_header_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_attribute_header_test_app COMMAND searchlib_attribute_header_test_app) diff --git a/searchlib/src/tests/attribute/attribute_operation/CMakeLists.txt b/searchlib/src/tests/attribute/attribute_operation/CMakeLists.txt index fcf7f420092..48216b20460 100644 --- a/searchlib/src/tests/attribute/attribute_operation/CMakeLists.txt +++ b/searchlib/src/tests/attribute/attribute_operation/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_attribute_operation_test_app TEST SOURCES attribute_operation_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_attribute_operation_test_app COMMAND searchlib_attribute_operation_test_app) diff --git a/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp b/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp index 733c9e1a574..7589663073f 100644 --- a/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp +++ b/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("attribute_operation_test"); diff --git a/searchlib/src/tests/attribute/attributefilewriter/CMakeLists.txt b/searchlib/src/tests/attribute/attributefilewriter/CMakeLists.txt index 419d62de213..6b5c69cb612 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/CMakeLists.txt +++ b/searchlib/src/tests/attribute/attributefilewriter/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_attributefilewriter_test_app TEST SOURCES attributefilewriter_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_attributefilewriter_test_app COMMAND searchlib_attributefilewriter_test_app) diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index e8d95eab7e2..56117fdd17f 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/attributefilewriter.h> #include <vespa/searchlib/attribute/attributefilebufferwriter.h> #include <vespa/searchlib/attribute/attribute_header.h> diff --git a/searchlib/src/tests/attribute/attributemanager/CMakeLists.txt b/searchlib/src/tests/attribute/attributemanager/CMakeLists.txt index 904706e0eb0..d3f9b010c93 100644 --- a/searchlib/src/tests/attribute/attributemanager/CMakeLists.txt +++ b/searchlib/src/tests/attribute/attributemanager/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_attributemanager_test_app TEST SOURCES attributemanager_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_attributemanager_test_app COMMAND searchlib_attributemanager_test_app) diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index c3254cf930d..2494b514c56 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/attribute/configconverter.h> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multinumericattribute.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("attribute_test"); diff --git a/searchlib/src/tests/attribute/benchmark/CMakeLists.txt b/searchlib/src/tests/attribute/benchmark/CMakeLists.txt index 90be19ba963..f3cad958edd 100644 --- a/searchlib/src/tests/attribute/benchmark/CMakeLists.txt +++ b/searchlib/src/tests/attribute/benchmark/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_executable(searchlib_attributebenchmark_app SOURCES attributebenchmark.cpp DEPENDS - searchlib + vespa_searchlib ) diff --git a/searchlib/src/tests/attribute/bitvector/CMakeLists.txt b/searchlib/src/tests/attribute/bitvector/CMakeLists.txt index 887f18d6c6c..7efbf9ab45b 100644 --- a/searchlib/src/tests/attribute/bitvector/CMakeLists.txt +++ b/searchlib/src/tests/attribute/bitvector/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_bitvector_test_app TEST SOURCES bitvector_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_bitvector_test_app COMMAND searchlib_bitvector_test_app COST 200) diff --git a/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt b/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt index 14be6913495..03afd39e759 100644 --- a/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt +++ b/searchlib/src/tests/attribute/bitvector_search_cache/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_bitvector_search_cache_test_app TEST SOURCES bitvector_search_cache_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_bitvector_search_cache_test_app COMMAND searchlib_bitvector_search_cache_test_app) diff --git a/searchlib/src/tests/attribute/changevector/CMakeLists.txt b/searchlib/src/tests/attribute/changevector/CMakeLists.txt index 6e65f8d8ae9..abb58a29409 100644 --- a/searchlib/src/tests/attribute/changevector/CMakeLists.txt +++ b/searchlib/src/tests/attribute/changevector/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_changevector_test_app TEST SOURCES changevector_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_changevector_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/changevector_test.sh DEPENDS searchlib_changevector_test_app) diff --git a/searchlib/src/tests/attribute/changevector/changevector_test.cpp b/searchlib/src/tests/attribute/changevector/changevector_test.cpp index ff0b358ea2e..9dd96b1b72c 100644 --- a/searchlib/src/tests/attribute/changevector/changevector_test.cpp +++ b/searchlib/src/tests/attribute/changevector/changevector_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/changevector.hpp> #include <vespa/vespalib/stllike/hash_set.h> diff --git a/searchlib/src/tests/attribute/compaction/CMakeLists.txt b/searchlib/src/tests/attribute/compaction/CMakeLists.txt index 56611e0a5d5..5cb5a9193bf 100644 --- a/searchlib/src/tests/attribute/compaction/CMakeLists.txt +++ b/searchlib/src/tests/attribute/compaction/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_attribute_compaction_test_app TEST SOURCES attribute_compaction_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_attribute_compaction_test_app COMMAND searchlib_attribute_compaction_test_app) diff --git a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/CMakeLists.txt b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/CMakeLists.txt index 6ef463e066e..8dece596e12 100644 --- a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/CMakeLists.txt +++ b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attribute_dfa_fuzzy_matcher_test_app TEST SOURCES dfa_fuzzy_matcher_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_attribute_dfa_fuzzy_matcher_test_app COMMAND searchlib_attribute_dfa_fuzzy_matcher_test_app) diff --git a/searchlib/src/tests/attribute/direct_multi_term_blueprint/CMakeLists.txt b/searchlib/src/tests/attribute/direct_multi_term_blueprint/CMakeLists.txt index 473d977ac7a..20a82b4af6e 100644 --- a/searchlib/src/tests/attribute/direct_multi_term_blueprint/CMakeLists.txt +++ b/searchlib/src/tests/attribute/direct_multi_term_blueprint/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_direct_multi_term_blueprint_test_app TEST SOURCES direct_multi_term_blueprint_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/attribute/direct_posting_store/CMakeLists.txt b/searchlib/src/tests/attribute/direct_posting_store/CMakeLists.txt index 3c8e76bc9b2..f0f1ca3b3e3 100644 --- a/searchlib/src/tests/attribute/direct_posting_store/CMakeLists.txt +++ b/searchlib/src/tests/attribute/direct_posting_store/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_direct_posting_store_test_app TEST SOURCES direct_posting_store_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/attribute/enum_attribute_compaction/CMakeLists.txt b/searchlib/src/tests/attribute/enum_attribute_compaction/CMakeLists.txt index 4938f203dd8..a8a26394335 100644 --- a/searchlib/src/tests/attribute/enum_attribute_compaction/CMakeLists.txt +++ b/searchlib/src/tests/attribute/enum_attribute_compaction/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_enum_attribute_compaction_test_app TEST SOURCES enum_attribute_compaction_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_enum_attribute_compaction_test_app COMMAND searchlib_enum_attribute_compaction_test_app COST 100) diff --git a/searchlib/src/tests/attribute/enum_comparator/CMakeLists.txt b/searchlib/src/tests/attribute/enum_comparator/CMakeLists.txt index fde35ce4fc0..0a619a46602 100644 --- a/searchlib/src/tests/attribute/enum_comparator/CMakeLists.txt +++ b/searchlib/src/tests/attribute/enum_comparator/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_enum_comparator_test_app TEST SOURCES enum_comparator_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_enum_comparator_test_app COMMAND searchlib_enum_comparator_test_app) diff --git a/searchlib/src/tests/attribute/enumeratedsave/CMakeLists.txt b/searchlib/src/tests/attribute/enumeratedsave/CMakeLists.txt index dbb7f640914..9cd0fa291f5 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/CMakeLists.txt +++ b/searchlib/src/tests/attribute/enumeratedsave/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_enumeratedsave_test_app TEST SOURCES enumeratedsave_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_enumeratedsave_test_app COMMAND searchlib_enumeratedsave_test_app COST 100) diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 0795d85e4a2..431d09a8665 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -19,7 +19,7 @@ #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/searchlib/src/tests/attribute/enumstore/CMakeLists.txt b/searchlib/src/tests/attribute/enumstore/CMakeLists.txt index 1d2b0e2c8da..477cea927b5 100644 --- a/searchlib/src/tests/attribute/enumstore/CMakeLists.txt +++ b/searchlib/src/tests/attribute/enumstore/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_enumstore_test_app TEST SOURCES enumstore_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_enumstore_test_app COMMAND searchlib_enumstore_test_app) diff --git a/searchlib/src/tests/attribute/extendattributes/CMakeLists.txt b/searchlib/src/tests/attribute/extendattributes/CMakeLists.txt index c83baf5b74c..782a53f3ffc 100644 --- a/searchlib/src/tests/attribute/extendattributes/CMakeLists.txt +++ b/searchlib/src/tests/attribute/extendattributes/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_extendattribute_test_app TEST SOURCES extendattribute_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_extendattribute_test_app COMMAND searchlib_extendattribute_test_app) diff --git a/searchlib/src/tests/attribute/guard/CMakeLists.txt b/searchlib/src/tests/attribute/guard/CMakeLists.txt index f5570377814..6549af11c0f 100644 --- a/searchlib/src/tests/attribute/guard/CMakeLists.txt +++ b/searchlib/src/tests/attribute/guard/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attributeguard_test_app TEST SOURCES attributeguard_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_attributeguard_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/attributeguard_test.sh diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/CMakeLists.txt b/searchlib/src/tests/attribute/imported_attribute_vector/CMakeLists.txt index db19b94b3ab..17b0e36530a 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/CMakeLists.txt +++ b/searchlib/src/tests/attribute/imported_attribute_vector/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_imported_attribute_vector_test_app TEST SOURCES imported_attribute_vector_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_imported_attribute_vector_test_app COMMAND searchlib_imported_attribute_vector_test_app) diff --git a/searchlib/src/tests/attribute/imported_search_context/CMakeLists.txt b/searchlib/src/tests/attribute/imported_search_context/CMakeLists.txt index 1102c75995d..2289c335bb2 100644 --- a/searchlib/src/tests/attribute/imported_search_context/CMakeLists.txt +++ b/searchlib/src/tests/attribute/imported_search_context/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_imported_search_context_test_app TEST SOURCES imported_search_context_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_imported_search_context_test_app COMMAND searchlib_imported_search_context_test_app) diff --git a/searchlib/src/tests/attribute/multi_term_or_filter_search/CMakeLists.txt b/searchlib/src/tests/attribute/multi_term_or_filter_search/CMakeLists.txt index 4ec5d849ad3..10e8725105c 100644 --- a/searchlib/src/tests/attribute/multi_term_or_filter_search/CMakeLists.txt +++ b/searchlib/src/tests/attribute/multi_term_or_filter_search/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_multi_term_or_filter_search_test_app TEST SOURCES multi_term_or_filter_search_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/attribute/multi_value_mapping/CMakeLists.txt b/searchlib/src/tests/attribute/multi_value_mapping/CMakeLists.txt index c537886626c..a172c21ca4e 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/CMakeLists.txt +++ b/searchlib/src/tests/attribute/multi_value_mapping/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_multi_value_mapping_test_app TEST SOURCES multi_value_mapping_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_multi_value_mapping_test_app COMMAND searchlib_multi_value_mapping_test_app) diff --git a/searchlib/src/tests/attribute/multi_value_read_view/CMakeLists.txt b/searchlib/src/tests/attribute/multi_value_read_view/CMakeLists.txt index fbe0e71fc4b..e4209fe34b8 100644 --- a/searchlib/src/tests/attribute/multi_value_read_view/CMakeLists.txt +++ b/searchlib/src/tests/attribute/multi_value_read_view/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attribute_multi_value_read_view_test_app TEST SOURCES multi_value_read_view_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_attribute_multi_value_read_view_test_app COMMAND searchlib_attribute_multi_value_read_view_test_app) diff --git a/searchlib/src/tests/attribute/posting_list_merger/CMakeLists.txt b/searchlib/src/tests/attribute/posting_list_merger/CMakeLists.txt index 2d6b328b84b..636fe01f1c2 100644 --- a/searchlib/src/tests/attribute/posting_list_merger/CMakeLists.txt +++ b/searchlib/src/tests/attribute/posting_list_merger/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_posting_list_merger_test_app TEST SOURCES posting_list_merger_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_posting_list_merger_test_app COMMAND searchlib_posting_list_merger_test_app) diff --git a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp index fb733db5f71..868c85c7962 100644 --- a/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp +++ b/searchlib/src/tests/attribute/posting_list_merger/posting_list_merger_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/posting_list_merger.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/searchlib/src/tests/attribute/posting_store/CMakeLists.txt b/searchlib/src/tests/attribute/posting_store/CMakeLists.txt index 75a6f2fcded..7f0e116fd86 100644 --- a/searchlib/src/tests/attribute/posting_store/CMakeLists.txt +++ b/searchlib/src/tests/attribute/posting_store/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_posting_store_test_app TEST SOURCES posting_store_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_posting_store_test_app COMMAND searchlib_posting_store_test_app COST 30) diff --git a/searchlib/src/tests/attribute/postinglist/CMakeLists.txt b/searchlib/src/tests/attribute/postinglist/CMakeLists.txt index e88b37696fc..f907abbfe53 100644 --- a/searchlib/src/tests/attribute/postinglist/CMakeLists.txt +++ b/searchlib/src/tests/attribute/postinglist/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_postinglist_test_app TEST SOURCES postinglist_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_postinglist_test_app COMMAND searchlib_postinglist_test_app) diff --git a/searchlib/src/tests/attribute/postinglistattribute/CMakeLists.txt b/searchlib/src/tests/attribute/postinglistattribute/CMakeLists.txt index a6741899066..c0394a1aac4 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/postinglistattribute/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_postinglistattribute_test_app TEST SOURCES postinglistattribute_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_postinglistattribute_test_app COMMAND searchlib_postinglistattribute_test_app) diff --git a/searchlib/src/tests/attribute/predicate_attribute/CMakeLists.txt b/searchlib/src/tests/attribute/predicate_attribute/CMakeLists.txt index eb2c99a5d81..122bdab2714 100644 --- a/searchlib/src/tests/attribute/predicate_attribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/predicate_attribute/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_predicate_attribute_test_app TEST SOURCES predicate_attribute_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_predicate_attribute_test_app COMMAND searchlib_predicate_attribute_test_app) diff --git a/searchlib/src/tests/attribute/raw_attribute/CMakeLists.txt b/searchlib/src/tests/attribute/raw_attribute/CMakeLists.txt index 3bfe5a8e91c..17264adc59b 100644 --- a/searchlib/src/tests/attribute/raw_attribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/raw_attribute/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_raw_attribute_test_app TEST SOURCES raw_attribute_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_raw_attribute_test_app COMMAND searchlib_raw_attribute_test_app) diff --git a/searchlib/src/tests/attribute/reference_attribute/CMakeLists.txt b/searchlib/src/tests/attribute/reference_attribute/CMakeLists.txt index 7b18cffcb8e..165a27d2576 100644 --- a/searchlib/src/tests/attribute/reference_attribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/reference_attribute/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_reference_attribute_test_app TEST SOURCES reference_attribute_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_reference_attribute_test_app COMMAND searchlib_reference_attribute_test_app) diff --git a/searchlib/src/tests/attribute/save_target/CMakeLists.txt b/searchlib/src/tests/attribute/save_target/CMakeLists.txt index 1227b545c8d..4d9b5f0816e 100644 --- a/searchlib/src/tests/attribute/save_target/CMakeLists.txt +++ b/searchlib/src/tests/attribute/save_target/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attribute_save_target_test_app TEST SOURCES attribute_save_target_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_attribute_save_target_test_app COMMAND searchlib_attribute_save_target_test_app) diff --git a/searchlib/src/tests/attribute/searchable/CMakeLists.txt b/searchlib/src/tests/attribute/searchable/CMakeLists.txt index 7932450e2db..6fbf0f72c0b 100644 --- a/searchlib/src/tests/attribute/searchable/CMakeLists.txt +++ b/searchlib/src/tests/attribute/searchable/CMakeLists.txt @@ -3,14 +3,14 @@ vespa_add_executable(searchlib_attribute_searchable_adapter_test_app TEST SOURCES attribute_searchable_adapter_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_attribute_searchable_adapter_test_app COMMAND searchlib_attribute_searchable_adapter_test_app) vespa_add_executable(searchlib_attribute_weighted_set_blueprint_test_app TEST SOURCES attribute_weighted_set_blueprint_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_attribute_weighted_set_blueprint_test_app COMMAND searchlib_attribute_weighted_set_blueprint_test_app) @@ -18,7 +18,7 @@ vespa_add_executable(searchlib_attribute_blueprint_test_app TEST SOURCES attributeblueprint_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index 7a794795cce..0278c4f32ef 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/attribute_blueprint_factory.h> #include <vespa/searchlib/attribute/attribute_weighted_set_blueprint.h> #include <vespa/searchlib/attribute/attributecontext.h> diff --git a/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt b/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt index d9043aaceba..daa729c90f5 100644 --- a/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt +++ b/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_searchcontext_test_app TEST SOURCES searchcontext_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_searchcontext_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/searchcontext_test.sh diff --git a/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt b/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt index f80b28358f4..63121352e1a 100644 --- a/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt +++ b/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attribute_searchcontextelementiterator_test_app T SOURCES searchcontextelementiterator_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_attribute_searchcontextelementiterator_test_app COMMAND searchlib_attribute_searchcontextelementiterator_test_app) diff --git a/searchlib/src/tests/attribute/sort_blob_writers/CMakeLists.txt b/searchlib/src/tests/attribute/sort_blob_writers/CMakeLists.txt index 1055dbb1271..880b81ef518 100644 --- a/searchlib/src/tests/attribute/sort_blob_writers/CMakeLists.txt +++ b/searchlib/src/tests/attribute/sort_blob_writers/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_sort_blob_writers_test_app TEST SOURCES sort_blob_writers_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_sort_blob_writers_test_app COMMAND searchlib_sort_blob_writers_test_app) diff --git a/searchlib/src/tests/attribute/sourceselector/CMakeLists.txt b/searchlib/src/tests/attribute/sourceselector/CMakeLists.txt index c70470852ba..50cd465371a 100644 --- a/searchlib/src/tests/attribute/sourceselector/CMakeLists.txt +++ b/searchlib/src/tests/attribute/sourceselector/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_sourceselector_test_app TEST SOURCES sourceselector_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_sourceselector_test_app COMMAND searchlib_sourceselector_test_app) diff --git a/searchlib/src/tests/attribute/stringattribute/CMakeLists.txt b/searchlib/src/tests/attribute/stringattribute/CMakeLists.txt index 1f5805ee19e..9aeb8daf236 100644 --- a/searchlib/src/tests/attribute/stringattribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/stringattribute/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_stringattribute_test_app TEST SOURCES stringattribute_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_stringattribute_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/stringattribute_test.sh DEPENDS searchlib_stringattribute_test_app) diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index 1bfb9fb41f9..b558c50488a 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/enumstore.h> #include <vespa/searchlib/attribute/singlestringattribute.h> diff --git a/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt b/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt index 0eb667b4d6d..10f9f5bcc0d 100644 --- a/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_tensorattribute_test_app TEST SOURCES tensorattribute_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_tensorattribute_test_app COMMAND searchlib_tensorattribute_test_app) diff --git a/searchlib/src/tests/bitcompression/expgolomb/CMakeLists.txt b/searchlib/src/tests/bitcompression/expgolomb/CMakeLists.txt index 5bddda57946..671e14d3ba5 100644 --- a/searchlib/src/tests/bitcompression/expgolomb/CMakeLists.txt +++ b/searchlib/src/tests/bitcompression/expgolomb/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_expgolomb_test_app TEST SOURCES expgolomb_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_expgolomb_test_app NO_VALGRIND COMMAND searchlib_expgolomb_test_app) diff --git a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp index 03cef8c8079..f76476dfda2 100644 --- a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp +++ b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vector> #include <algorithm> #include <cinttypes> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP("expglomb_test"); diff --git a/searchlib/src/tests/bitvector/CMakeLists.txt b/searchlib/src/tests/bitvector/CMakeLists.txt index f748e82c328..1d715834cf0 100644 --- a/searchlib/src/tests/bitvector/CMakeLists.txt +++ b/searchlib/src/tests/bitvector/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_bitvectorbenchmark_test_app SOURCES bitvectorbenchmark.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_bitvectorbenchmark_test_app COMMAND searchlib_bitvectorbenchmark_test_app BENCHMARK) diff --git a/searchlib/src/tests/common/bitvector/CMakeLists.txt b/searchlib/src/tests/common/bitvector/CMakeLists.txt index 42cbfced278..09b4fdea8b8 100644 --- a/searchlib/src/tests/common/bitvector/CMakeLists.txt +++ b/searchlib/src/tests/common/bitvector/CMakeLists.txt @@ -3,20 +3,20 @@ vespa_add_executable(searchlib_bitvector_test-common_app TEST SOURCES bitvector_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_bitvector_test-common_app COMMAND searchlib_bitvector_test-common_app) vespa_add_executable(searchlib_bitvector_benchmark_app SOURCES bitvector_benchmark.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_bitvector_benchmark_app COMMAND searchlib_bitvector_benchmark_app BENCHMARK) vespa_add_executable(searchlib_condensedbitvector_test_app TEST SOURCES condensedbitvector_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_condensedbitvector_test_app COMMAND searchlib_condensedbitvector_test_app) diff --git a/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp b/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp index d111a392270..2600ebedbee 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("bitvector_benchmark"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/bitvector.h> using namespace search; diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 758f44cdba2..ef9e801cda3 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/searchlib/common/growablebitvector.h> #include <vespa/searchlib/common/partialbitvector.h> diff --git a/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp b/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp index f4bbb0fe7ca..4f7a0dfa736 100644 --- a/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/condensedbitvectors.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/common/geogcd/CMakeLists.txt b/searchlib/src/tests/common/geogcd/CMakeLists.txt index ae0c1fa0dc4..d39a7f05ca2 100644 --- a/searchlib/src/tests/common/geogcd/CMakeLists.txt +++ b/searchlib/src/tests/common/geogcd/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_geo_gcd_test_app TEST SOURCES geo_gcd_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_geo_gcd_test_app COMMAND searchlib_geo_gcd_test_app) diff --git a/searchlib/src/tests/common/location/CMakeLists.txt b/searchlib/src/tests/common/location/CMakeLists.txt index 3167ab485df..201b4074e8c 100644 --- a/searchlib/src/tests/common/location/CMakeLists.txt +++ b/searchlib/src/tests/common/location/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_geo_location_test_app TEST SOURCES geo_location_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_geo_location_test_app COMMAND searchlib_geo_location_test_app) diff --git a/searchlib/src/tests/common/location_iterator/CMakeLists.txt b/searchlib/src/tests/common/location_iterator/CMakeLists.txt index 55639bdc283..1eb0ec5c8ba 100644 --- a/searchlib/src/tests/common/location_iterator/CMakeLists.txt +++ b/searchlib/src/tests/common/location_iterator/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_location_iterator_test_app TEST SOURCES location_iterator_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/common/matching_elements/CMakeLists.txt b/searchlib/src/tests/common/matching_elements/CMakeLists.txt index f8c21b311a3..56b69ab1b05 100644 --- a/searchlib/src/tests/common/matching_elements/CMakeLists.txt +++ b/searchlib/src/tests/common/matching_elements/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_common_matching_elements_test_app TEST SOURCES matching_elements_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_common_matching_elements_test_app COMMAND searchlib_common_matching_elements_test_app) diff --git a/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt b/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt index 0efd434e508..2d3c8fec060 100644 --- a/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt +++ b/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_common_matching_elements_fields_test_app TEST SOURCES matching_elements_fields_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_common_matching_elements_fields_test_app COMMAND searchlib_common_matching_elements_fields_test_app) diff --git a/searchlib/src/tests/common/resultset/CMakeLists.txt b/searchlib/src/tests/common/resultset/CMakeLists.txt index 9dd3feff349..699f43113dc 100644 --- a/searchlib/src/tests/common/resultset/CMakeLists.txt +++ b/searchlib/src/tests/common/resultset/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_resultset_test_app TEST SOURCES resultset_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_resultset_test_app COMMAND searchlib_resultset_test_app) diff --git a/searchlib/src/tests/common/resultset/resultset_test.cpp b/searchlib/src/tests/common/resultset/resultset_test.cpp index d8cba35a96c..d757785206a 100644 --- a/searchlib/src/tests/common/resultset/resultset_test.cpp +++ b/searchlib/src/tests/common/resultset/resultset_test.cpp @@ -6,7 +6,7 @@ LOG_SETUP("resultset_test"); #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/resultset.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/arraysize.h> using namespace search; diff --git a/searchlib/src/tests/common/summaryfeatures/CMakeLists.txt b/searchlib/src/tests/common/summaryfeatures/CMakeLists.txt index 0149d440fc6..655c2186bae 100644 --- a/searchlib/src/tests/common/summaryfeatures/CMakeLists.txt +++ b/searchlib/src/tests/common/summaryfeatures/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_summaryfeatures_test_app TEST SOURCES summaryfeatures_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_summaryfeatures_test_app COMMAND searchlib_summaryfeatures_test_app) diff --git a/searchlib/src/tests/common/summaryfeatures/summaryfeatures_test.cpp b/searchlib/src/tests/common/summaryfeatures/summaryfeatures_test.cpp index 415c1846bdc..6c6f4db18df 100644 --- a/searchlib/src/tests/common/summaryfeatures/summaryfeatures_test.cpp +++ b/searchlib/src/tests/common/summaryfeatures/summaryfeatures_test.cpp @@ -1,18 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("summaryfeatures_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/featureset.h> using vespalib::FeatureSet; using vespalib::Memory; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("summaryfeatures_test"); +TEST("summaryfeatures_test") { { FeatureSet sf; EXPECT_EQUAL(sf.getNames().size(), 0u); @@ -152,5 +147,6 @@ Test::Main() EXPECT_TRUE(sf.getFeaturesByDocId(45) == nullptr); EXPECT_TRUE(sf.getFeaturesByDocId(55) == nullptr); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/diskindex/bitvector/CMakeLists.txt b/searchlib/src/tests/diskindex/bitvector/CMakeLists.txt index f70a1efd23c..0f2606387cc 100644 --- a/searchlib/src/tests/diskindex/bitvector/CMakeLists.txt +++ b/searchlib/src/tests/diskindex/bitvector/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_bitvector_test-diskindex_app TEST SOURCES bitvector_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_bitvector_test-diskindex_app COMMAND searchlib_bitvector_test-diskindex_app) diff --git a/searchlib/src/tests/diskindex/field_length_scanner/CMakeLists.txt b/searchlib/src/tests/diskindex/field_length_scanner/CMakeLists.txt index cdcabce20cb..a6753d331b3 100644 --- a/searchlib/src/tests/diskindex/field_length_scanner/CMakeLists.txt +++ b/searchlib/src/tests/diskindex/field_length_scanner/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_field_length_scanner_test_app TEST SOURCES field_length_scanner_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/diskindex/fieldwriter/CMakeLists.txt b/searchlib/src/tests/diskindex/fieldwriter/CMakeLists.txt index 75ab7423d2f..76a250587a5 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/CMakeLists.txt +++ b/searchlib/src/tests/diskindex/fieldwriter/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(searchlib_fieldwriter_test_app TEST fieldwriter_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_fieldwriter_test_app COMMAND searchlib_fieldwriter_test_app COST 200) diff --git a/searchlib/src/tests/diskindex/pagedict4/CMakeLists.txt b/searchlib/src/tests/diskindex/pagedict4/CMakeLists.txt index 34114e195bc..c3c777cc463 100644 --- a/searchlib/src/tests/diskindex/pagedict4/CMakeLists.txt +++ b/searchlib/src/tests/diskindex/pagedict4/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_pagedict4_test_app TEST pagedict4_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_pagedict4_test_app COMMAND searchlib_pagedict4_test_app) @@ -13,7 +13,7 @@ vespa_add_executable(searchlib_pagedict4_hugeword_cornercase_test_app TEST pagedict4_hugeword_cornercase_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_pagedict4_hugeword_cornercase_test_app COMMAND searchlib_pagedict4_hugeword_cornercase_test_app) @@ -22,5 +22,5 @@ vespa_add_executable(searchlib_pagedict4_long_words_test_app TEST pagedict4_long_words_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib ) diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4_hugeword_cornercase_test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4_hugeword_cornercase_test.cpp index 76efb2ec788..c17d78b25f3 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4_hugeword_cornercase_test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4_hugeword_cornercase_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/countcompression.h> diff --git a/searchlib/src/tests/docstore/chunk/CMakeLists.txt b/searchlib/src/tests/docstore/chunk/CMakeLists.txt index 1068863e846..329221d80e1 100644 --- a/searchlib/src/tests/docstore/chunk/CMakeLists.txt +++ b/searchlib/src/tests/docstore/chunk/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_chunk_test_app TEST SOURCES chunk_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_chunk_test_app COMMAND searchlib_chunk_test_app) diff --git a/searchlib/src/tests/docstore/document_store/CMakeLists.txt b/searchlib/src/tests/docstore/document_store/CMakeLists.txt index 9386ab8fa33..fe14320597f 100644 --- a/searchlib/src/tests/docstore/document_store/CMakeLists.txt +++ b/searchlib/src/tests/docstore/document_store/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(searchlib_document_store_test_app TEST SOURCES document_store_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_document_store_test_app COMMAND searchlib_document_store_test_app) vespa_add_executable(searchlib_visitcache_test_app TEST SOURCES visitcache_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_visitcache_test_app COMMAND searchlib_visitcache_test_app) diff --git a/searchlib/src/tests/docstore/document_store_visitor/CMakeLists.txt b/searchlib/src/tests/docstore/document_store_visitor/CMakeLists.txt index 3f51a6b1fd0..afca02e342b 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/CMakeLists.txt +++ b/searchlib/src/tests/docstore/document_store_visitor/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_document_store_visitor_test_app TEST SOURCES document_store_visitor_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_document_store_visitor_test_app COMMAND searchlib_document_store_visitor_test_app) diff --git a/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt b/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt index 4c3e5f2358e..cae93c15060 100644 --- a/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt +++ b/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_file_chunk_test_app TEST SOURCES file_chunk_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_file_chunk_test_app COMMAND searchlib_file_chunk_test_app) diff --git a/searchlib/src/tests/docstore/lid_info/CMakeLists.txt b/searchlib/src/tests/docstore/lid_info/CMakeLists.txt index 419a79b4714..167ed1ace80 100644 --- a/searchlib/src/tests/docstore/lid_info/CMakeLists.txt +++ b/searchlib/src/tests/docstore/lid_info/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_lid_info_test_app TEST SOURCES lid_info_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_lid_info_test_app COMMAND searchlib_lid_info_test_app) diff --git a/searchlib/src/tests/docstore/logdatastore/CMakeLists.txt b/searchlib/src/tests/docstore/logdatastore/CMakeLists.txt index 479d34bad9d..e5c66c2b91b 100644 --- a/searchlib/src/tests/docstore/logdatastore/CMakeLists.txt +++ b/searchlib/src/tests/docstore/logdatastore/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_logdatastore_test_app TEST SOURCES logdatastore_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_logdatastore_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/logdatastore_test.sh DEPENDS searchlib_logdatastore_test_app) diff --git a/searchlib/src/tests/docstore/store_by_bucket/CMakeLists.txt b/searchlib/src/tests/docstore/store_by_bucket/CMakeLists.txt index 71abe3c6564..cb30a6dd0a8 100644 --- a/searchlib/src/tests/docstore/store_by_bucket/CMakeLists.txt +++ b/searchlib/src/tests/docstore/store_by_bucket/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_store_by_bucket_test_app TEST SOURCES store_by_bucket_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_store_by_bucket_test_app COMMAND searchlib_store_by_bucket_test_app) diff --git a/searchlib/src/tests/engine/proto_converter/CMakeLists.txt b/searchlib/src/tests/engine/proto_converter/CMakeLists.txt index ee5f2c2a64c..97b987ec0b8 100644 --- a/searchlib/src/tests/engine/proto_converter/CMakeLists.txt +++ b/searchlib/src/tests/engine/proto_converter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_engine_proto_converter_test_app TEST SOURCES proto_converter_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_engine_proto_converter_test_app COMMAND searchlib_engine_proto_converter_test_app) diff --git a/searchlib/src/tests/engine/proto_rpc_adapter/CMakeLists.txt b/searchlib/src/tests/engine/proto_rpc_adapter/CMakeLists.txt index 52757f870bc..a30bb91ac64 100644 --- a/searchlib/src/tests/engine/proto_rpc_adapter/CMakeLists.txt +++ b/searchlib/src/tests/engine/proto_rpc_adapter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_engine_proto_rpc_adapter_test_app TEST SOURCES proto_rpc_adapter_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_engine_proto_rpc_adapter_test_app COMMAND searchlib_engine_proto_rpc_adapter_test_app) diff --git a/searchlib/src/tests/expression/attributenode/CMakeLists.txt b/searchlib/src/tests/expression/attributenode/CMakeLists.txt index 841deb0cb45..dd388572cfa 100644 --- a/searchlib/src/tests/expression/attributenode/CMakeLists.txt +++ b/searchlib/src/tests/expression/attributenode/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attribute_node_test_app TEST SOURCES attribute_node_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_attribute_node_test_app COMMAND searchlib_attribute_node_test_app) diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index 62fee24f972..d2b07c6c552 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -14,7 +14,7 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/test/make_attribute_map_lookup_node.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("attribute_node_test"); diff --git a/searchlib/src/tests/expression/current_index_setup/CMakeLists.txt b/searchlib/src/tests/expression/current_index_setup/CMakeLists.txt index 80d59cab617..125e9aa74f1 100644 --- a/searchlib/src/tests/expression/current_index_setup/CMakeLists.txt +++ b/searchlib/src/tests/expression/current_index_setup/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_current_index_setup_test_app TEST SOURCES current_index_setup_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_current_index_setup_test_app COMMAND searchlib_current_index_setup_test_app) diff --git a/searchlib/src/tests/features/CMakeLists.txt b/searchlib/src/tests/features/CMakeLists.txt index ea2410734b5..3a2b596a529 100644 --- a/searchlib/src/tests/features/CMakeLists.txt +++ b/searchlib/src/tests/features/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(searchlib_prod_features_test_app TEST prod_features_fieldmatch.cpp prod_features_fieldtermmatch.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_prod_features_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/prod_features_test.sh @@ -16,7 +16,7 @@ vespa_add_executable(searchlib_featurebenchmark_app SOURCES featurebenchmark.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_featurebenchmark_app COMMAND searchlib_featurebenchmark_app BENCHMARK) diff --git a/searchlib/src/tests/features/beta/CMakeLists.txt b/searchlib/src/tests/features/beta/CMakeLists.txt index db45f02d898..9f2aa16f086 100644 --- a/searchlib/src/tests/features/beta/CMakeLists.txt +++ b/searchlib/src/tests/features/beta/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_beta_features_test_app TEST SOURCES beta_features_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test( diff --git a/searchlib/src/tests/features/bm25/CMakeLists.txt b/searchlib/src/tests/features/bm25/CMakeLists.txt index 5d5c7f63578..7d735c13a06 100644 --- a/searchlib/src/tests/features/bm25/CMakeLists.txt +++ b/searchlib/src/tests/features/bm25/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_features_bm25_test_app TEST SOURCES bm25_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/features/closest/CMakeLists.txt b/searchlib/src/tests/features/closest/CMakeLists.txt index b124066afee..7777d526b40 100644 --- a/searchlib/src/tests/features/closest/CMakeLists.txt +++ b/searchlib/src/tests/features/closest/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_closest_test_app TEST SOURCES closest_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_closest_test_app COMMAND searchlib_closest_test_app) diff --git a/searchlib/src/tests/features/constant/CMakeLists.txt b/searchlib/src/tests/features/constant/CMakeLists.txt index bd76bf654bf..c873e647b7d 100644 --- a/searchlib/src/tests/features/constant/CMakeLists.txt +++ b/searchlib/src/tests/features/constant/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_constant_test_app TEST SOURCES constant_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_constant_test_app COMMAND searchlib_constant_test_app) diff --git a/searchlib/src/tests/features/element_completeness/CMakeLists.txt b/searchlib/src/tests/features/element_completeness/CMakeLists.txt index 046b061b884..8486ce0efcf 100644 --- a/searchlib/src/tests/features/element_completeness/CMakeLists.txt +++ b/searchlib/src/tests/features/element_completeness/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_element_completeness_test_app TEST SOURCES element_completeness_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_element_completeness_test_app COMMAND searchlib_element_completeness_test_app) diff --git a/searchlib/src/tests/features/element_similarity_feature/CMakeLists.txt b/searchlib/src/tests/features/element_similarity_feature/CMakeLists.txt index 748556b0fcd..62bf8d7162f 100644 --- a/searchlib/src/tests/features/element_similarity_feature/CMakeLists.txt +++ b/searchlib/src/tests/features/element_similarity_feature/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_element_similarity_feature_test_app TEST SOURCES element_similarity_feature_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) -vespa_add_test(NAME searchlib_element_similarity_feature_test_app COMMAND searchlib_element_similarity_feature_test_app) +vespa_add_test(NAME searchlib_element_similarity_feature_test_app COMMAND searchlib_element_similarity_feature_test_app COST 50) diff --git a/searchlib/src/tests/features/euclidean_distance/CMakeLists.txt b/searchlib/src/tests/features/euclidean_distance/CMakeLists.txt index df55b8f834c..03e9456e4a9 100644 --- a/searchlib/src/tests/features/euclidean_distance/CMakeLists.txt +++ b/searchlib/src/tests/features/euclidean_distance/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_euclidean_distance_test_app TEST SOURCES euclidean_distance_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_euclidean_distance_test_app COMMAND searchlib_euclidean_distance_test_app) diff --git a/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp b/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp index e7523288e8f..82e227e0d35 100644 --- a/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp +++ b/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp @@ -1,5 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/attributefactory.h> diff --git a/searchlib/src/tests/features/first_phase_rank/CMakeLists.txt b/searchlib/src/tests/features/first_phase_rank/CMakeLists.txt new file mode 100644 index 00000000000..dfb69077b64 --- /dev/null +++ b/searchlib/src/tests/features/first_phase_rank/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(searchlib_features_first_phase_rank_test_app TEST + SOURCES + first_phase_rank_test.cpp + DEPENDS + vespa_searchlib + searchlib_test + GTest::GTest +) +vespa_add_test(NAME searchlib_features_first_phase_rank_test_app COMMAND searchlib_features_first_phase_rank_test_app) diff --git a/searchlib/src/tests/features/first_phase_rank/first_phase_rank_test.cpp b/searchlib/src/tests/features/first_phase_rank/first_phase_rank_test.cpp new file mode 100644 index 00000000000..01ba6c36124 --- /dev/null +++ b/searchlib/src/tests/features/first_phase_rank/first_phase_rank_test.cpp @@ -0,0 +1,143 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/features/first_phase_rank_feature.h> +#include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/fef/blueprintfactory.h> +#include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> +#define ENABLE_GTEST_MIGRATION +#include <vespa/searchlib/test/ft_test_app_base.h> +#include <vespa/vespalib/gtest/gtest.h> + +using search::features::FirstPhaseRankBlueprint; +using search::features::FirstPhaseRankLookup; +using search::features::setup_search_features; +using search::fef::Blueprint; +using search::fef::BlueprintFactory; +using search::fef::ObjectStore; +using search::fef::test::IndexEnvironment; +using search::fef::test::DummyDependencyHandler; +using StringVector = std::vector<vespalib::string>; + +constexpr feature_t unranked = std::numeric_limits<feature_t>::max(); + +struct FirstPhaseRankBlueprintTest : public ::testing::Test { + BlueprintFactory factory; + IndexEnvironment index_env; + + FirstPhaseRankBlueprintTest() + : ::testing::Test(), + factory(), + index_env() + { + setup_search_features(factory); + } + + ~FirstPhaseRankBlueprintTest() override; + + std::shared_ptr<Blueprint> make_blueprint() const { + return factory.createBlueprint("firstPhaseRank"); + } + + void expect_setup_fail(const StringVector& params, const vespalib::string& exp_fail_msg) { + auto blueprint = make_blueprint(); + DummyDependencyHandler deps(*blueprint); + EXPECT_FALSE(blueprint->setup(index_env, params)); + EXPECT_EQ(exp_fail_msg, deps.fail_msg); + } + + std::shared_ptr<Blueprint> expect_setup_succeed(const StringVector& params) { + auto blueprint = make_blueprint(); + DummyDependencyHandler deps(*blueprint); + EXPECT_TRUE(blueprint->setup(index_env, params)); + EXPECT_EQ(0, deps.input.size()); + EXPECT_EQ(StringVector({"score"}), deps.output); + return blueprint; + } +}; + +FirstPhaseRankBlueprintTest::~FirstPhaseRankBlueprintTest() = default; + +TEST_F(FirstPhaseRankBlueprintTest, blueprint_can_be_created_from_factory) +{ + auto bp = make_blueprint(); + EXPECT_TRUE(bp); + EXPECT_TRUE(dynamic_pointer_cast<FirstPhaseRankBlueprint>(bp)); +} + +TEST_F(FirstPhaseRankBlueprintTest, blueprint_setup_fails_when_parameter_list_is_not_empty) +{ + expect_setup_fail({"is"}, + "The parameter list used for setting up rank feature firstPhaseRank is not valid: " + "Expected 0 parameter(s), but got 1"); +} + +TEST_F(FirstPhaseRankBlueprintTest, blueprint_setup_succeeds) +{ + expect_setup_succeed({}); +} + +TEST_F(FirstPhaseRankBlueprintTest, blueprint_can_prepare_shared_state) +{ + auto blueprint = expect_setup_succeed({}); + search::fef::test::QueryEnvironment query_env; + ObjectStore store; + EXPECT_EQ(nullptr, FirstPhaseRankLookup::get_mutable_shared_state(store)); + EXPECT_EQ(nullptr, FirstPhaseRankLookup::get_shared_state(store)); + blueprint->prepareSharedState(query_env, store); + EXPECT_NE(nullptr, FirstPhaseRankLookup::get_mutable_shared_state(store)); + EXPECT_NE(nullptr, FirstPhaseRankLookup::get_shared_state(store)); +} + +TEST_F(FirstPhaseRankBlueprintTest, dump_features) +{ + FtTestAppBase::FT_DUMP_EMPTY(factory, "firstPhaseRank", index_env); +} + +struct FirstPhaseRankExecutorTest : public ::testing::Test { + BlueprintFactory factory; + FtFeatureTest test; + + FirstPhaseRankExecutorTest() + : ::testing::Test(), + factory(), + test(factory, "firstPhaseRank") + { + setup_search_features(factory); + } + ~FirstPhaseRankExecutorTest() override; + void setup(std::vector<std::pair<uint32_t,uint32_t>> ranks) { + EXPECT_TRUE(test.setup()); + auto* lookup = FirstPhaseRankLookup::get_mutable_shared_state(test.getQueryEnv().getObjectStore()); + ASSERT_NE(nullptr, lookup); + for (auto& entry : ranks) { + lookup->add(entry.first, entry.second); + } + } + bool execute(feature_t exp_score, uint32_t docid) { + return test.execute(exp_score, 0.000001, docid); + } +}; + +FirstPhaseRankExecutorTest::~FirstPhaseRankExecutorTest() = default; + +TEST_F(FirstPhaseRankExecutorTest, unranked_docid_gives_huge_output) +{ + setup({}); + EXPECT_TRUE(execute(unranked, 1)); +} + +TEST_F(FirstPhaseRankExecutorTest, ranked_docid_gives_expected_output) +{ + setup({{3, 5}, {7, 4}}); + EXPECT_TRUE(execute(unranked, 2)); + EXPECT_TRUE(execute(5, 3)); + EXPECT_TRUE(execute(unranked, 4)); + EXPECT_TRUE(execute(unranked, 5)); + EXPECT_TRUE(execute(unranked, 6)); + EXPECT_TRUE(execute(4, 7)); + EXPECT_TRUE(execute(unranked, 8)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/features/imported_dot_product/CMakeLists.txt b/searchlib/src/tests/features/imported_dot_product/CMakeLists.txt index 25828f14bf1..8fcc3faa094 100644 --- a/searchlib/src/tests/features/imported_dot_product/CMakeLists.txt +++ b/searchlib/src/tests/features/imported_dot_product/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_imported_dot_product_test_app TEST SOURCES imported_dot_product_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_imported_dot_product_test_app COMMAND searchlib_imported_dot_product_test_app) diff --git a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt index e7fc3126e2f..cfdd52bc393 100644 --- a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt +++ b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_internal_max_reduce_prod_join_feature_test_app TE SOURCES internal_max_reduce_prod_join_feature_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_internal_max_reduce_prod_join_feature_test_app COMMAND searchlib_internal_max_reduce_prod_join_feature_test_app) diff --git a/searchlib/src/tests/features/item_raw_score/CMakeLists.txt b/searchlib/src/tests/features/item_raw_score/CMakeLists.txt index 27329de4c67..47e4aa4a3dd 100644 --- a/searchlib/src/tests/features/item_raw_score/CMakeLists.txt +++ b/searchlib/src/tests/features/item_raw_score/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_item_raw_score_test_app TEST SOURCES item_raw_score_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_item_raw_score_test_app COMMAND searchlib_item_raw_score_test_app) diff --git a/searchlib/src/tests/features/max_reduce_prod_join_replacer/CMakeLists.txt b/searchlib/src/tests/features/max_reduce_prod_join_replacer/CMakeLists.txt index d24b6ae65d0..41f6828d19d 100644 --- a/searchlib/src/tests/features/max_reduce_prod_join_replacer/CMakeLists.txt +++ b/searchlib/src/tests/features/max_reduce_prod_join_replacer/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_max_reduce_prod_join_replacer_test_app TEST SOURCES max_reduce_prod_join_replacer_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_max_reduce_prod_join_replacer_test_app COMMAND searchlib_max_reduce_prod_join_replacer_test_app) diff --git a/searchlib/src/tests/features/native_dot_product/CMakeLists.txt b/searchlib/src/tests/features/native_dot_product/CMakeLists.txt index dbdd5803c2b..0f301d81619 100644 --- a/searchlib/src/tests/features/native_dot_product/CMakeLists.txt +++ b/searchlib/src/tests/features/native_dot_product/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_native_dot_product_test_app TEST SOURCES native_dot_product_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_native_dot_product_test_app COMMAND searchlib_native_dot_product_test_app) diff --git a/searchlib/src/tests/features/nns_closeness/CMakeLists.txt b/searchlib/src/tests/features/nns_closeness/CMakeLists.txt index 1b31d773199..373274d25f0 100644 --- a/searchlib/src/tests/features/nns_closeness/CMakeLists.txt +++ b/searchlib/src/tests/features/nns_closeness/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_nns_closeness_test_app TEST SOURCES nns_closeness_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_nns_closeness_test_app COMMAND searchlib_nns_closeness_test_app) diff --git a/searchlib/src/tests/features/nns_distance/CMakeLists.txt b/searchlib/src/tests/features/nns_distance/CMakeLists.txt index 4b85bdbd682..753be89a849 100644 --- a/searchlib/src/tests/features/nns_distance/CMakeLists.txt +++ b/searchlib/src/tests/features/nns_distance/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_nns_distance_test_app TEST SOURCES nns_distance_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_nns_distance_test_app COMMAND searchlib_nns_distance_test_app) diff --git a/searchlib/src/tests/features/onnx_feature/CMakeLists.txt b/searchlib/src/tests/features/onnx_feature/CMakeLists.txt index 9291e833035..630990d40ab 100644 --- a/searchlib/src/tests/features/onnx_feature/CMakeLists.txt +++ b/searchlib/src/tests/features/onnx_feature/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_onnx_feature_test_app TEST SOURCES onnx_feature_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_onnx_feature_test_app COMMAND searchlib_onnx_feature_test_app) diff --git a/searchlib/src/tests/features/ranking_expression/CMakeLists.txt b/searchlib/src/tests/features/ranking_expression/CMakeLists.txt index 849811ac802..526860320e7 100644 --- a/searchlib/src/tests/features/ranking_expression/CMakeLists.txt +++ b/searchlib/src/tests/features/ranking_expression/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_ranking_expression_test_app TEST SOURCES ranking_expression_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_ranking_expression_test_app COMMAND searchlib_ranking_expression_test_app) diff --git a/searchlib/src/tests/features/raw_score/CMakeLists.txt b/searchlib/src/tests/features/raw_score/CMakeLists.txt index 10f07b20e23..824690b8d12 100644 --- a/searchlib/src/tests/features/raw_score/CMakeLists.txt +++ b/searchlib/src/tests/features/raw_score/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_raw_score_test_app TEST SOURCES raw_score_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_raw_score_test_app COMMAND searchlib_raw_score_test_app) diff --git a/searchlib/src/tests/features/subqueries/CMakeLists.txt b/searchlib/src/tests/features/subqueries/CMakeLists.txt index cbf0ce2a467..e7c1b396d2f 100644 --- a/searchlib/src/tests/features/subqueries/CMakeLists.txt +++ b/searchlib/src/tests/features/subqueries/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_subqueries_test_app TEST SOURCES subqueries_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_subqueries_test_app COMMAND searchlib_subqueries_test_app) diff --git a/searchlib/src/tests/features/tensor/CMakeLists.txt b/searchlib/src/tests/features/tensor/CMakeLists.txt index 7666c613bb3..537080aa168 100644 --- a/searchlib/src/tests/features/tensor/CMakeLists.txt +++ b/searchlib/src/tests/features/tensor/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_tensor_test_app TEST SOURCES tensor_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_tensor_test_app COMMAND searchlib_tensor_test_app) diff --git a/searchlib/src/tests/features/tensor_from_labels/CMakeLists.txt b/searchlib/src/tests/features/tensor_from_labels/CMakeLists.txt index 3ecceffd422..a02c96aa1b3 100644 --- a/searchlib/src/tests/features/tensor_from_labels/CMakeLists.txt +++ b/searchlib/src/tests/features/tensor_from_labels/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_tensor_from_labels_test_app TEST SOURCES tensor_from_labels_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_tensor_from_labels_test_app COMMAND searchlib_tensor_from_labels_test_app) diff --git a/searchlib/src/tests/features/tensor_from_weighted_set/CMakeLists.txt b/searchlib/src/tests/features/tensor_from_weighted_set/CMakeLists.txt index b5322c1a64c..130ac165e8a 100644 --- a/searchlib/src/tests/features/tensor_from_weighted_set/CMakeLists.txt +++ b/searchlib/src/tests/features/tensor_from_weighted_set/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_tensor_from_weighted_set_test_app TEST SOURCES tensor_from_weighted_set_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_tensor_from_weighted_set_test_app COMMAND searchlib_tensor_from_weighted_set_test_app) diff --git a/searchlib/src/tests/features/text_similarity_feature/CMakeLists.txt b/searchlib/src/tests/features/text_similarity_feature/CMakeLists.txt index cfa715af516..29fa11ccb61 100644 --- a/searchlib/src/tests/features/text_similarity_feature/CMakeLists.txt +++ b/searchlib/src/tests/features/text_similarity_feature/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_text_similarity_feature_test_app TEST SOURCES text_similarity_feature_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_text_similarity_feature_test_app COMMAND searchlib_text_similarity_feature_test_app) diff --git a/searchlib/src/tests/features/util/CMakeLists.txt b/searchlib/src/tests/features/util/CMakeLists.txt index 33054cddf36..0eee4d3b7ac 100644 --- a/searchlib/src/tests/features/util/CMakeLists.txt +++ b/searchlib/src/tests/features/util/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_util_test_app TEST SOURCES util_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_util_test_app COMMAND searchlib_util_test_app) diff --git a/searchlib/src/tests/fef/CMakeLists.txt b/searchlib/src/tests/fef/CMakeLists.txt index a01cb15a492..de32f1718da 100644 --- a/searchlib/src/tests/fef/CMakeLists.txt +++ b/searchlib/src/tests/fef/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_fef_test_app TEST SOURCES fef_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_fef_test_app COMMAND searchlib_fef_test_app) diff --git a/searchlib/src/tests/fef/attributecontent/CMakeLists.txt b/searchlib/src/tests/fef/attributecontent/CMakeLists.txt index d9c88fb9eaf..966f84700a5 100644 --- a/searchlib/src/tests/fef/attributecontent/CMakeLists.txt +++ b/searchlib/src/tests/fef/attributecontent/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_attributecontent_test_app TEST SOURCES attributecontent_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_attributecontent_test_app COMMAND searchlib_attributecontent_test_app) diff --git a/searchlib/src/tests/fef/featurenamebuilder/CMakeLists.txt b/searchlib/src/tests/fef/featurenamebuilder/CMakeLists.txt index 5f8dc64c564..8675e36e7b9 100644 --- a/searchlib/src/tests/fef/featurenamebuilder/CMakeLists.txt +++ b/searchlib/src/tests/fef/featurenamebuilder/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_featurenamebuilder_test_app TEST SOURCES featurenamebuilder_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_featurenamebuilder_test_app COMMAND searchlib_featurenamebuilder_test_app) diff --git a/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp b/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp index a6910ad5df8..d4e5be770d4 100644 --- a/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp +++ b/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp @@ -1,19 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("featurenamebuilder_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/featurenamebuilder.h> using namespace search::fef; using B = FeatureNameBuilder; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("featurenamebuilder_test"); +TEST("featurenamebuilder_test") { // normal cases EXPECT_EQUAL(B().baseName("foo").buildName(), "foo"); @@ -72,6 +67,6 @@ Test::Main() EXPECT_EQUAL(B().baseName("foo").parameter(" bar ( a , b ) . out ", false).buildName(), "foo(bar(a,b).out)"); EXPECT_EQUAL(B().baseName("foo").parameter(" bar ( a , b ) . out.2 ", false).buildName(), "foo(bar(a,b).out.2)"); EXPECT_EQUAL(B().baseName("foo").parameter(" bar ( a , b ) . out . 2 ", false).buildName(), "foo(\" bar ( a , b ) . out . 2 \")"); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/fef/featurenameparser/CMakeLists.txt b/searchlib/src/tests/fef/featurenameparser/CMakeLists.txt index cb040472a6f..ff224d9ab5f 100644 --- a/searchlib/src/tests/fef/featurenameparser/CMakeLists.txt +++ b/searchlib/src/tests/fef/featurenameparser/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_featurenameparser_test_app TEST SOURCES featurenameparser_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_featurenameparser_test_app COMMAND searchlib_featurenameparser_test_app) diff --git a/searchlib/src/tests/fef/featureoverride/CMakeLists.txt b/searchlib/src/tests/fef/featureoverride/CMakeLists.txt index 789903633a3..7f3c5bf3e32 100644 --- a/searchlib/src/tests/fef/featureoverride/CMakeLists.txt +++ b/searchlib/src/tests/fef/featureoverride/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_featureoverride_test_app TEST SOURCES featureoverride_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_featureoverride_test_app COMMAND searchlib_featureoverride_test_app) diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp index 5b3e3c356ac..f7c6b9fd7ed 100644 --- a/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp +++ b/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/indexenvironment.h> diff --git a/searchlib/src/tests/fef/fef_test.cpp b/searchlib/src/tests/fef/fef_test.cpp index 0f2de1665e8..37089cfa5ea 100644 --- a/searchlib/src/tests/fef/fef_test.cpp +++ b/searchlib/src/tests/fef/fef_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/objectstore.h> diff --git a/searchlib/src/tests/fef/object_passing/CMakeLists.txt b/searchlib/src/tests/fef/object_passing/CMakeLists.txt index 2c8e0400b92..db9b95b22a6 100644 --- a/searchlib/src/tests/fef/object_passing/CMakeLists.txt +++ b/searchlib/src/tests/fef/object_passing/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_object_passing_test_app TEST SOURCES object_passing_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_object_passing_test_app COMMAND searchlib_object_passing_test_app) diff --git a/searchlib/src/tests/fef/parameter/CMakeLists.txt b/searchlib/src/tests/fef/parameter/CMakeLists.txt index b238e09c98b..58ae4c412a4 100644 --- a/searchlib/src/tests/fef/parameter/CMakeLists.txt +++ b/searchlib/src/tests/fef/parameter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_parameter_test_app TEST SOURCES parameter_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_parameter_test_app NO_VALGRIND COMMAND searchlib_parameter_test_app) diff --git a/searchlib/src/tests/fef/phrasesplitter/CMakeLists.txt b/searchlib/src/tests/fef/phrasesplitter/CMakeLists.txt index 8f3946d110c..6c4c7ae124a 100644 --- a/searchlib/src/tests/fef/phrasesplitter/CMakeLists.txt +++ b/searchlib/src/tests/fef/phrasesplitter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_phrasesplitter_test_app TEST SOURCES phrasesplitter_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_phrasesplitter_test_app COMMAND searchlib_phrasesplitter_test_app) @@ -11,6 +11,6 @@ vespa_add_executable(searchlib_benchmark_app SOURCES benchmark.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_benchmark_app COMMAND searchlib_benchmark_app BENCHMARK) diff --git a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp index 93a5a01262d..6be252380da 100644 --- a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp +++ b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp @@ -1,18 +1,16 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> + #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/phrasesplitter.h> #include <vespa/searchlib/fef/phrase_splitter_query_env.h> #include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/vespalib/util/time.h> #include <iomanip> #include <iostream> -#include <vespa/log/log.h> -LOG_SETUP("phrasesplitter_test"); - namespace search::fef { -class Benchmark : public vespalib::TestApp +class Benchmark { private: vespalib::Timer _timer; @@ -20,12 +18,16 @@ private: void start() { _timer = vespalib::Timer(); } void sample() { _sample = _timer.elapsed(); } - void run(size_t numRuns, size_t numPositions); public: - Benchmark() : _timer(), _sample(0) {} - ~Benchmark() override; - int Main() override; + Benchmark() + : _timer(), + _sample(0) + { + } + ~Benchmark(); + void run(size_t numRuns, size_t numPositions); + vespalib::duration get_sample() const noexcept { return _sample; } }; Benchmark::~Benchmark() = default; @@ -61,28 +63,24 @@ Benchmark::run(size_t numRuns, size_t numPositions) sample(); } +} + int -Benchmark::Main() +main(int argc, char* argv[]) { - - TEST_INIT("benchmark"); - - if (_argc != 3) { + if (argc != 3) { std::cout << "Must specify <numRuns> and <numPositions>" << std::endl; return 0; } - size_t numRuns = strtoull(_argv[1], nullptr, 10); - size_t numPositions = strtoull(_argv[2], nullptr, 10); + size_t numRuns = strtoull(argv[1], nullptr, 10); + size_t numPositions = strtoull(argv[2], nullptr, 10); - run(numRuns, numPositions); + auto app = std::make_unique<search::fef::Benchmark>(); + app->run(numRuns, numPositions); + auto sample = app->get_sample(); - std::cout << "TET: " << vespalib::count_ms(_sample) << " (ms)" << std::endl; - std::cout << "ETPD: " << std::fixed << std::setprecision(10) << (vespalib::count_ns(_sample) / (numRuns * 1000000.0)) << " (ms)" << std::endl; + std::cout << "TET: " << vespalib::count_ms(sample) << " (ms)" << std::endl; + std::cout << "ETPD: " << std::fixed << std::setprecision(10) << (vespalib::count_ns(sample) / (numRuns * 1000000.0)) << " (ms)" << std::endl; - TEST_DONE(); } - -} - -TEST_APPHOOK(search::fef::Benchmark); diff --git a/searchlib/src/tests/fef/properties/CMakeLists.txt b/searchlib/src/tests/fef/properties/CMakeLists.txt index dd1eb83b0c2..4eae94344c4 100644 --- a/searchlib/src/tests/fef/properties/CMakeLists.txt +++ b/searchlib/src/tests/fef/properties/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(searchlib_properties_test_app TEST SOURCES properties_test.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_properties_test_app COMMAND searchlib_properties_test_app) diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp index c8073739b3e..80a7b64a2e0 100644 --- a/searchlib/src/tests/fef/properties/properties_test.cpp +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/gtest/gtest.h> #include <limits> using namespace search::fef; @@ -30,7 +30,8 @@ Properties make_props(std::initializer_list<std::pair<const char *, std::initial return props; } -TEST("require that namespace visitation works") { +TEST(PropertiesTest, require_that_namespace_visitation_works) +{ Properties props = make_props({ {"foo", {"outside"}}, {"foo.a", {"a_value"}}, {"foo.b", {"b_value"}}, @@ -39,20 +40,21 @@ TEST("require that namespace visitation works") { Properties result; CopyVisitor copy_visitor(result); props.visitNamespace("foo", copy_visitor); - EXPECT_EQUAL(2u, result.numKeys()); - EXPECT_EQUAL(result.lookup("a").get(), Property::Value("a_value")); - EXPECT_EQUAL(result.lookup("b").get(), Property::Value("b_value")); + EXPECT_EQ(2u, result.numKeys()); + EXPECT_EQ(result.lookup("a").get(), Property::Value("a_value")); + EXPECT_EQ(result.lookup("b").get(), Property::Value("b_value")); } -TEST("test stuff") { +TEST(PropertiesTest, test_stuff) +{ { // empty lookup result Property p; - EXPECT_EQUAL(p.found(), false); - EXPECT_EQUAL(p.get(), Property::Value("")); - EXPECT_EQUAL(p.get("fb"), Property::Value("fb")); - EXPECT_EQUAL(p.size(), 0u); - EXPECT_EQUAL(p.getAt(0), Property::Value("")); + EXPECT_EQ(p.found(), false); + EXPECT_EQ(p.get(), Property::Value("")); + EXPECT_EQ(p.get("fb"), Property::Value("fb")); + EXPECT_EQ(p.size(), 0u); + EXPECT_EQ(p.getAt(0), Property::Value("")); } { // add / count / remove Properties p = make_props({ {"a", {"a1", "a2", "a3"}}, @@ -61,48 +63,48 @@ TEST("test stuff") { }); const Properties &pc = p; - EXPECT_EQUAL(pc.numKeys(), 3u); - EXPECT_EQUAL(pc.numValues(), 6u); - EXPECT_EQUAL(pc.count("a"), 3u); - EXPECT_EQUAL(pc.count("b"), 2u); - EXPECT_EQUAL(pc.count("c"), 1u); - EXPECT_EQUAL(pc.count("d"), 0u); + EXPECT_EQ(pc.numKeys(), 3u); + EXPECT_EQ(pc.numValues(), 6u); + EXPECT_EQ(pc.count("a"), 3u); + EXPECT_EQ(pc.count("b"), 2u); + EXPECT_EQ(pc.count("c"), 1u); + EXPECT_EQ(pc.count("d"), 0u); p.remove("d"); - EXPECT_EQUAL(pc.numKeys(), 3u); - EXPECT_EQUAL(pc.numValues(), 6u); - EXPECT_EQUAL(pc.count("a"), 3u); - EXPECT_EQUAL(pc.count("b"), 2u); - EXPECT_EQUAL(pc.count("c"), 1u); - EXPECT_EQUAL(pc.count("d"), 0u); + EXPECT_EQ(pc.numKeys(), 3u); + EXPECT_EQ(pc.numValues(), 6u); + EXPECT_EQ(pc.count("a"), 3u); + EXPECT_EQ(pc.count("b"), 2u); + EXPECT_EQ(pc.count("c"), 1u); + EXPECT_EQ(pc.count("d"), 0u); p.remove("c"); - EXPECT_EQUAL(pc.numKeys(), 2u); - EXPECT_EQUAL(pc.numValues(), 5u); - EXPECT_EQUAL(pc.count("a"), 3u); - EXPECT_EQUAL(pc.count("b"), 2u); - EXPECT_EQUAL(pc.count("c"), 0u); - EXPECT_EQUAL(pc.count("d"), 0u); + EXPECT_EQ(pc.numKeys(), 2u); + EXPECT_EQ(pc.numValues(), 5u); + EXPECT_EQ(pc.count("a"), 3u); + EXPECT_EQ(pc.count("b"), 2u); + EXPECT_EQ(pc.count("c"), 0u); + EXPECT_EQ(pc.count("d"), 0u); p.remove("b"); - EXPECT_EQUAL(pc.numKeys(), 1u); - EXPECT_EQUAL(pc.numValues(), 3u); - EXPECT_EQUAL(pc.count("a"), 3u); - EXPECT_EQUAL(pc.count("b"), 0u); - EXPECT_EQUAL(pc.count("c"), 0u); - EXPECT_EQUAL(pc.count("d"), 0u); + EXPECT_EQ(pc.numKeys(), 1u); + EXPECT_EQ(pc.numValues(), 3u); + EXPECT_EQ(pc.count("a"), 3u); + EXPECT_EQ(pc.count("b"), 0u); + EXPECT_EQ(pc.count("c"), 0u); + EXPECT_EQ(pc.count("d"), 0u); p.remove("a"); - EXPECT_EQUAL(pc.numKeys(), 0u); - EXPECT_EQUAL(pc.numValues(), 0u); - EXPECT_EQUAL(pc.count("a"), 0u); - EXPECT_EQUAL(pc.count("b"), 0u); - EXPECT_EQUAL(pc.count("c"), 0u); - EXPECT_EQUAL(pc.count("d"), 0u); + EXPECT_EQ(pc.numKeys(), 0u); + EXPECT_EQ(pc.numValues(), 0u); + EXPECT_EQ(pc.count("a"), 0u); + EXPECT_EQ(pc.count("b"), 0u); + EXPECT_EQ(pc.count("c"), 0u); + EXPECT_EQ(pc.count("d"), 0u); } { // lookup / import / visit / compare / hash Properties p; @@ -114,38 +116,38 @@ TEST("test stuff") { p.add("list", "e1").add("list", "e2").add("list", "e3"); - EXPECT_EQUAL(p.numKeys(), 5u); - EXPECT_EQUAL(p.numValues(), 7u); - - EXPECT_EQUAL(p.lookup("x").found(), true); - EXPECT_EQUAL(p.lookup("a.x").found(), true); - EXPECT_EQUAL(p.lookup("a.b.x").found(), true); - EXPECT_EQUAL(p.lookup("a.b.c.x").found(), true); - EXPECT_EQUAL(p.lookup("list").found(), true); - EXPECT_EQUAL(p.lookup("y").found(), false); - - EXPECT_EQUAL(p.lookup("x").get(), Property::Value("x1")); - EXPECT_EQUAL(p.lookup("a.x").get(), Property::Value("x2")); - EXPECT_EQUAL(p.lookup("a.b.x").get(), Property::Value("x3")); - EXPECT_EQUAL(p.lookup("a.b.c.x").get(), Property::Value("x4")); - EXPECT_EQUAL(p.lookup("list").get(), Property::Value("e1")); - EXPECT_EQUAL(p.lookup("y").get(), Property::Value("")); - - EXPECT_EQUAL(p.lookup("x").get(), Property::Value("x1")); - EXPECT_EQUAL(p.lookup("a", "x").get(), Property::Value("x2")); - EXPECT_EQUAL(p.lookup("a", "b", "x").get(), Property::Value("x3")); - EXPECT_EQUAL(p.lookup("a", "b", "c", "x").get(), Property::Value("x4")); - - EXPECT_EQUAL(p.lookup("x").get("fallback"), Property::Value("x1")); - EXPECT_EQUAL(p.lookup("y").get("fallback"), Property::Value("fallback")); - - EXPECT_EQUAL(p.lookup("y").size(), 0u); - EXPECT_EQUAL(p.lookup("x").size(), 1u); - EXPECT_EQUAL(p.lookup("list").size(), 3u); - EXPECT_EQUAL(p.lookup("list").getAt(0), Property::Value("e1")); - EXPECT_EQUAL(p.lookup("list").getAt(1), Property::Value("e2")); - EXPECT_EQUAL(p.lookup("list").getAt(2), Property::Value("e3")); - EXPECT_EQUAL(p.lookup("list").getAt(3), Property::Value("")); + EXPECT_EQ(p.numKeys(), 5u); + EXPECT_EQ(p.numValues(), 7u); + + EXPECT_EQ(p.lookup("x").found(), true); + EXPECT_EQ(p.lookup("a.x").found(), true); + EXPECT_EQ(p.lookup("a.b.x").found(), true); + EXPECT_EQ(p.lookup("a.b.c.x").found(), true); + EXPECT_EQ(p.lookup("list").found(), true); + EXPECT_EQ(p.lookup("y").found(), false); + + EXPECT_EQ(p.lookup("x").get(), Property::Value("x1")); + EXPECT_EQ(p.lookup("a.x").get(), Property::Value("x2")); + EXPECT_EQ(p.lookup("a.b.x").get(), Property::Value("x3")); + EXPECT_EQ(p.lookup("a.b.c.x").get(), Property::Value("x4")); + EXPECT_EQ(p.lookup("list").get(), Property::Value("e1")); + EXPECT_EQ(p.lookup("y").get(), Property::Value("")); + + EXPECT_EQ(p.lookup("x").get(), Property::Value("x1")); + EXPECT_EQ(p.lookup("a", "x").get(), Property::Value("x2")); + EXPECT_EQ(p.lookup("a", "b", "x").get(), Property::Value("x3")); + EXPECT_EQ(p.lookup("a", "b", "c", "x").get(), Property::Value("x4")); + + EXPECT_EQ(p.lookup("x").get("fallback"), Property::Value("x1")); + EXPECT_EQ(p.lookup("y").get("fallback"), Property::Value("fallback")); + + EXPECT_EQ(p.lookup("y").size(), 0u); + EXPECT_EQ(p.lookup("x").size(), 1u); + EXPECT_EQ(p.lookup("list").size(), 3u); + EXPECT_EQ(p.lookup("list").getAt(0), Property::Value("e1")); + EXPECT_EQ(p.lookup("list").getAt(1), Property::Value("e2")); + EXPECT_EQ(p.lookup("list").getAt(2), Property::Value("e3")); + EXPECT_EQ(p.lookup("list").getAt(3), Property::Value("")); Properties p2; @@ -153,29 +155,29 @@ TEST("test stuff") { p2.add("y", "y1"); p2.add("list", "foo").add("list", "bar"); - EXPECT_EQUAL(p2.numKeys(), 3u); - EXPECT_EQUAL(p2.numValues(), 4u); + EXPECT_EQ(p2.numKeys(), 3u); + EXPECT_EQ(p2.numValues(), 4u); p.import(p2); - EXPECT_EQUAL(p.numKeys(), 6u); - EXPECT_EQUAL(p.numValues(), 7u); + EXPECT_EQ(p.numKeys(), 6u); + EXPECT_EQ(p.numValues(), 7u); - EXPECT_EQUAL(p.lookup("y").size(), 1u); - EXPECT_EQUAL(p.lookup("y").get(), Property::Value("y1")); + EXPECT_EQ(p.lookup("y").size(), 1u); + EXPECT_EQ(p.lookup("y").get(), Property::Value("y1")); - EXPECT_EQUAL(p.lookup("x").size(), 1u); - EXPECT_EQUAL(p.lookup("x").get(), Property::Value("new_x")); + EXPECT_EQ(p.lookup("x").size(), 1u); + EXPECT_EQ(p.lookup("x").get(), Property::Value("new_x")); - EXPECT_EQUAL(p.lookup("z").size(), 0u); + EXPECT_EQ(p.lookup("z").size(), 0u); - EXPECT_EQUAL(p.lookup("a", "x").size(), 1u); - EXPECT_EQUAL(p.lookup("a", "x").get(), Property::Value("x2")); + EXPECT_EQ(p.lookup("a", "x").size(), 1u); + EXPECT_EQ(p.lookup("a", "x").get(), Property::Value("x2")); - EXPECT_EQUAL(p.lookup("list").size(), 2u); - EXPECT_EQUAL(p.lookup("list").getAt(0), Property::Value("foo")); - EXPECT_EQUAL(p.lookup("list").getAt(1), Property::Value("bar")); - EXPECT_EQUAL(p.lookup("list").getAt(2), Property::Value("")); + EXPECT_EQ(p.lookup("list").size(), 2u); + EXPECT_EQ(p.lookup("list").getAt(0), Property::Value("foo")); + EXPECT_EQ(p.lookup("list").getAt(1), Property::Value("bar")); + EXPECT_EQ(p.lookup("list").getAt(2), Property::Value("")); Properties p3; @@ -189,32 +191,32 @@ TEST("test stuff") { CopyVisitor cv(p3); p.visitProperties(cv); - EXPECT_EQUAL(p3.numKeys(), 6u); - EXPECT_EQUAL(p3.numValues(), 7u); + EXPECT_EQ(p3.numKeys(), 6u); + EXPECT_EQ(p3.numValues(), 7u); EXPECT_TRUE(p == p3); EXPECT_TRUE(p3 == p); - EXPECT_EQUAL(p.hashCode(), p3.hashCode()); + EXPECT_EQ(p.hashCode(), p3.hashCode()); p.clear(); - EXPECT_EQUAL(p.numKeys(), 0u); - EXPECT_EQUAL(p.numValues(), 0u); + EXPECT_EQ(p.numKeys(), 0u); + EXPECT_EQ(p.numValues(), 0u); EXPECT_TRUE(!(p == p3)); EXPECT_TRUE(!(p3 == p)); Properties p4; CopyVisitor cv2(p4); p.visitProperties(cv); - EXPECT_EQUAL(p4.numKeys(), 0u); - EXPECT_EQUAL(p4.numValues(), 0u); + EXPECT_EQ(p4.numKeys(), 0u); + EXPECT_EQ(p4.numValues(), 0u); EXPECT_TRUE(p == p4); EXPECT_TRUE(p4 == p); - EXPECT_EQUAL(p.hashCode(), p4.hashCode()); + EXPECT_EQ(p.hashCode(), p4.hashCode()); } { // test index properties known by the framework { // vespa.eval.lazy_expressions - EXPECT_EQUAL(eval::LazyExpressions::NAME, vespalib::string("vespa.eval.lazy_expressions")); + EXPECT_EQ(eval::LazyExpressions::NAME, vespalib::string("vespa.eval.lazy_expressions")); { Properties p; EXPECT_TRUE(eval::LazyExpressions::check(p, true)); @@ -234,201 +236,203 @@ TEST("test stuff") { } } { // vespa.eval.use_fast_forest - EXPECT_EQUAL(eval::UseFastForest::NAME, vespalib::string("vespa.eval.use_fast_forest")); - EXPECT_EQUAL(eval::UseFastForest::DEFAULT_VALUE, false); + EXPECT_EQ(eval::UseFastForest::NAME, vespalib::string("vespa.eval.use_fast_forest")); + EXPECT_EQ(eval::UseFastForest::DEFAULT_VALUE, false); Properties p; - EXPECT_EQUAL(eval::UseFastForest::check(p), false); + EXPECT_EQ(eval::UseFastForest::check(p), false); p.add("vespa.eval.use_fast_forest", "true"); - EXPECT_EQUAL(eval::UseFastForest::check(p), true); + EXPECT_EQ(eval::UseFastForest::check(p), true); } { // vespa.rank.firstphase - EXPECT_EQUAL(rank::FirstPhase::NAME, vespalib::string("vespa.rank.firstphase")); - EXPECT_EQUAL(rank::FirstPhase::DEFAULT_VALUE, vespalib::string("nativeRank")); + EXPECT_EQ(rank::FirstPhase::NAME, vespalib::string("vespa.rank.firstphase")); + EXPECT_EQ(rank::FirstPhase::DEFAULT_VALUE, vespalib::string("nativeRank")); Properties p; - EXPECT_EQUAL(rank::FirstPhase::lookup(p), vespalib::string("nativeRank")); + EXPECT_EQ(rank::FirstPhase::lookup(p), vespalib::string("nativeRank")); p.add("vespa.rank.firstphase", "specialrank"); - EXPECT_EQUAL(rank::FirstPhase::lookup(p), vespalib::string("specialrank")); + EXPECT_EQ(rank::FirstPhase::lookup(p), vespalib::string("specialrank")); } { // vespa.rank.secondphase - EXPECT_EQUAL(rank::SecondPhase::NAME, vespalib::string("vespa.rank.secondphase")); - EXPECT_EQUAL(rank::SecondPhase::DEFAULT_VALUE, vespalib::string("")); + EXPECT_EQ(rank::SecondPhase::NAME, vespalib::string("vespa.rank.secondphase")); + EXPECT_EQ(rank::SecondPhase::DEFAULT_VALUE, vespalib::string("")); Properties p; - EXPECT_EQUAL(rank::SecondPhase::lookup(p), vespalib::string("")); + EXPECT_EQ(rank::SecondPhase::lookup(p), vespalib::string("")); p.add("vespa.rank.secondphase", "specialrank"); - EXPECT_EQUAL(rank::SecondPhase::lookup(p), vespalib::string("specialrank")); + EXPECT_EQ(rank::SecondPhase::lookup(p), vespalib::string("specialrank")); } { // vespa.dump.feature - EXPECT_EQUAL(dump::Feature::NAME, vespalib::string("vespa.dump.feature")); - EXPECT_EQUAL(dump::Feature::DEFAULT_VALUE.size(), 0u); + EXPECT_EQ(dump::Feature::NAME, vespalib::string("vespa.dump.feature")); + EXPECT_EQ(dump::Feature::DEFAULT_VALUE.size(), 0u); Properties p; - EXPECT_EQUAL(dump::Feature::lookup(p).size(), 0u); + EXPECT_EQ(dump::Feature::lookup(p).size(), 0u); p.add("vespa.dump.feature", "foo"); p.add("vespa.dump.feature", "bar"); std::vector<vespalib::string> a = dump::Feature::lookup(p); ASSERT_TRUE(a.size() == 2); - EXPECT_EQUAL(a[0], vespalib::string("foo")); - EXPECT_EQUAL(a[1], vespalib::string("bar")); + EXPECT_EQ(a[0], vespalib::string("foo")); + EXPECT_EQ(a[1], vespalib::string("bar")); } { // vespa.dump.ignoredefaultfeatures - EXPECT_EQUAL(dump::IgnoreDefaultFeatures::NAME, vespalib::string("vespa.dump.ignoredefaultfeatures")); - EXPECT_EQUAL(dump::IgnoreDefaultFeatures::DEFAULT_VALUE, "false"); + EXPECT_EQ(dump::IgnoreDefaultFeatures::NAME, vespalib::string("vespa.dump.ignoredefaultfeatures")); + EXPECT_EQ(dump::IgnoreDefaultFeatures::DEFAULT_VALUE, "false"); Properties p; EXPECT_TRUE(!dump::IgnoreDefaultFeatures::check(p)); p.add("vespa.dump.ignoredefaultfeatures", "true"); EXPECT_TRUE(dump::IgnoreDefaultFeatures::check(p)); } { // vespa.matching.termwise_limit - EXPECT_EQUAL(matching::TermwiseLimit::NAME, vespalib::string("vespa.matching.termwise_limit")); - EXPECT_EQUAL(matching::TermwiseLimit::DEFAULT_VALUE, 1.0); + EXPECT_EQ(matching::TermwiseLimit::NAME, vespalib::string("vespa.matching.termwise_limit")); + EXPECT_EQ(matching::TermwiseLimit::DEFAULT_VALUE, 1.0); Properties p; - EXPECT_EQUAL(matching::TermwiseLimit::lookup(p), 1.0); + EXPECT_EQ(matching::TermwiseLimit::lookup(p), 1.0); p.add("vespa.matching.termwise_limit", "0.05"); - EXPECT_EQUAL(matching::TermwiseLimit::lookup(p), 0.05); + EXPECT_EQ(matching::TermwiseLimit::lookup(p), 0.05); } { // vespa.matching.numthreads - EXPECT_EQUAL(matching::NumThreadsPerSearch::NAME, vespalib::string("vespa.matching.numthreadspersearch")); - EXPECT_EQUAL(matching::NumThreadsPerSearch::DEFAULT_VALUE, std::numeric_limits<uint32_t>::max()); + EXPECT_EQ(matching::NumThreadsPerSearch::NAME, vespalib::string("vespa.matching.numthreadspersearch")); + EXPECT_EQ(matching::NumThreadsPerSearch::DEFAULT_VALUE, std::numeric_limits<uint32_t>::max()); Properties p; - EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), std::numeric_limits<uint32_t>::max()); + EXPECT_EQ(matching::NumThreadsPerSearch::lookup(p), std::numeric_limits<uint32_t>::max()); p.add("vespa.matching.numthreadspersearch", "50"); - EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + EXPECT_EQ(matching::NumThreadsPerSearch::lookup(p), 50u); } { // vespa.matching.minhitsperthread - EXPECT_EQUAL(matching::MinHitsPerThread::NAME, vespalib::string("vespa.matching.minhitsperthread")); - EXPECT_EQUAL(matching::MinHitsPerThread::DEFAULT_VALUE, 0u); + EXPECT_EQ(matching::MinHitsPerThread::NAME, vespalib::string("vespa.matching.minhitsperthread")); + EXPECT_EQ(matching::MinHitsPerThread::DEFAULT_VALUE, 0u); Properties p; - EXPECT_EQUAL(matching::MinHitsPerThread::lookup(p), 0u); + EXPECT_EQ(matching::MinHitsPerThread::lookup(p), 0u); p.add("vespa.matching.minhitsperthread", "50"); - EXPECT_EQUAL(matching::MinHitsPerThread::lookup(p), 50u); + EXPECT_EQ(matching::MinHitsPerThread::lookup(p), 50u); } { - EXPECT_EQUAL(matching::NumSearchPartitions::NAME, vespalib::string("vespa.matching.numsearchpartitions")); - EXPECT_EQUAL(matching::NumSearchPartitions::DEFAULT_VALUE, 1u); + EXPECT_EQ(matching::NumSearchPartitions::NAME, vespalib::string("vespa.matching.numsearchpartitions")); + EXPECT_EQ(matching::NumSearchPartitions::DEFAULT_VALUE, 1u); Properties p; - EXPECT_EQUAL(matching::NumSearchPartitions::lookup(p), 1u); + EXPECT_EQ(matching::NumSearchPartitions::lookup(p), 1u); p.add("vespa.matching.numsearchpartitions", "50"); - EXPECT_EQUAL(matching::NumSearchPartitions::lookup(p), 50u); + EXPECT_EQ(matching::NumSearchPartitions::lookup(p), 50u); } { // vespa.matchphase.degradation.attribute - EXPECT_EQUAL(matchphase::DegradationAttribute::NAME, vespalib::string("vespa.matchphase.degradation.attribute")); - EXPECT_EQUAL(matchphase::DegradationAttribute::DEFAULT_VALUE, ""); + EXPECT_EQ(matchphase::DegradationAttribute::NAME, vespalib::string("vespa.matchphase.degradation.attribute")); + EXPECT_EQ(matchphase::DegradationAttribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(matchphase::DegradationAttribute::lookup(p), ""); + EXPECT_EQ(matchphase::DegradationAttribute::lookup(p), ""); p.add("vespa.matchphase.degradation.attribute", "foobar"); - EXPECT_EQUAL(matchphase::DegradationAttribute::lookup(p), "foobar"); + EXPECT_EQ(matchphase::DegradationAttribute::lookup(p), "foobar"); } { // vespa.matchphase.degradation.ascending - EXPECT_EQUAL(matchphase::DegradationAscendingOrder::NAME, vespalib::string("vespa.matchphase.degradation.ascendingorder")); - EXPECT_EQUAL(matchphase::DegradationAscendingOrder::DEFAULT_VALUE, false); + EXPECT_EQ(matchphase::DegradationAscendingOrder::NAME, vespalib::string("vespa.matchphase.degradation.ascendingorder")); + EXPECT_EQ(matchphase::DegradationAscendingOrder::DEFAULT_VALUE, false); Properties p; - EXPECT_EQUAL(matchphase::DegradationAscendingOrder::lookup(p), false); + EXPECT_EQ(matchphase::DegradationAscendingOrder::lookup(p), false); p.add("vespa.matchphase.degradation.ascendingorder", "true"); - EXPECT_EQUAL(matchphase::DegradationAscendingOrder::lookup(p), true); + EXPECT_EQ(matchphase::DegradationAscendingOrder::lookup(p), true); } { // vespa.matchphase.degradation.maxhits - EXPECT_EQUAL(matchphase::DegradationMaxHits::NAME, vespalib::string("vespa.matchphase.degradation.maxhits")); - EXPECT_EQUAL(matchphase::DegradationMaxHits::DEFAULT_VALUE, 0u); + EXPECT_EQ(matchphase::DegradationMaxHits::NAME, vespalib::string("vespa.matchphase.degradation.maxhits")); + EXPECT_EQ(matchphase::DegradationMaxHits::DEFAULT_VALUE, 0u); Properties p; - EXPECT_EQUAL(matchphase::DegradationMaxHits::lookup(p), 0u); + EXPECT_EQ(matchphase::DegradationMaxHits::lookup(p), 0u); p.add("vespa.matchphase.degradation.maxhits", "123789"); - EXPECT_EQUAL(matchphase::DegradationMaxHits::lookup(p), 123789u); + EXPECT_EQ(matchphase::DegradationMaxHits::lookup(p), 123789u); } { // vespa.matchphase.degradation.samplepercentage - EXPECT_EQUAL(matchphase::DegradationSamplePercentage::NAME, vespalib::string("vespa.matchphase.degradation.samplepercentage")); - EXPECT_EQUAL(matchphase::DegradationSamplePercentage::DEFAULT_VALUE, 0.2); + EXPECT_EQ(matchphase::DegradationSamplePercentage::NAME, vespalib::string("vespa.matchphase.degradation.samplepercentage")); + EXPECT_EQ(matchphase::DegradationSamplePercentage::DEFAULT_VALUE, 0.2); Properties p; - EXPECT_EQUAL(matchphase::DegradationSamplePercentage::lookup(p), 0.2); + EXPECT_EQ(matchphase::DegradationSamplePercentage::lookup(p), 0.2); p.add("vespa.matchphase.degradation.samplepercentage", "0.9"); - EXPECT_EQUAL(matchphase::DegradationSamplePercentage::lookup(p), 0.9); + EXPECT_EQ(matchphase::DegradationSamplePercentage::lookup(p), 0.9); } { // vespa.matchphase.degradation.maxfiltercoverage - EXPECT_EQUAL(matchphase::DegradationMaxFilterCoverage::NAME, vespalib::string("vespa.matchphase.degradation.maxfiltercoverage")); - EXPECT_EQUAL(matchphase::DegradationMaxFilterCoverage::DEFAULT_VALUE, 0.2); + EXPECT_EQ(matchphase::DegradationMaxFilterCoverage::NAME, vespalib::string("vespa.matchphase.degradation.maxfiltercoverage")); + EXPECT_EQ(matchphase::DegradationMaxFilterCoverage::DEFAULT_VALUE, 0.2); Properties p; - EXPECT_EQUAL(matchphase::DegradationMaxFilterCoverage::lookup(p), 0.2); + EXPECT_EQ(matchphase::DegradationMaxFilterCoverage::lookup(p), 0.2); p.add("vespa.matchphase.degradation.maxfiltercoverage", "0.076"); - EXPECT_EQUAL(matchphase::DegradationMaxFilterCoverage::lookup(p), 0.076); + EXPECT_EQ(matchphase::DegradationMaxFilterCoverage::lookup(p), 0.076); } { // vespa.matchphase.degradation.postfiltermultiplier - EXPECT_EQUAL(matchphase::DegradationPostFilterMultiplier::NAME, vespalib::string("vespa.matchphase.degradation.postfiltermultiplier")); - EXPECT_EQUAL(matchphase::DegradationPostFilterMultiplier::DEFAULT_VALUE, 1.0); + EXPECT_EQ(matchphase::DegradationPostFilterMultiplier::NAME, vespalib::string("vespa.matchphase.degradation.postfiltermultiplier")); + EXPECT_EQ(matchphase::DegradationPostFilterMultiplier::DEFAULT_VALUE, 1.0); Properties p; - EXPECT_EQUAL(matchphase::DegradationPostFilterMultiplier::lookup(p), 1.0); + EXPECT_EQ(matchphase::DegradationPostFilterMultiplier::lookup(p), 1.0); p.add("vespa.matchphase.degradation.postfiltermultiplier", "0.9"); - EXPECT_EQUAL(matchphase::DegradationPostFilterMultiplier::lookup(p), 0.9); + EXPECT_EQ(matchphase::DegradationPostFilterMultiplier::lookup(p), 0.9); } { // vespa.matchphase.diversity.attribute - EXPECT_EQUAL(matchphase::DiversityAttribute::NAME, vespalib::string("vespa.matchphase.diversity.attribute")); - EXPECT_EQUAL(matchphase::DiversityAttribute::DEFAULT_VALUE, ""); + EXPECT_EQ(matchphase::DiversityAttribute::NAME, vespalib::string("vespa.matchphase.diversity.attribute")); + EXPECT_EQ(matchphase::DiversityAttribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(matchphase::DiversityAttribute::lookup(p), ""); + EXPECT_EQ(matchphase::DiversityAttribute::lookup(p), ""); p.add("vespa.matchphase.diversity.attribute", "foobar"); - EXPECT_EQUAL(matchphase::DiversityAttribute::lookup(p), "foobar"); + EXPECT_EQ(matchphase::DiversityAttribute::lookup(p), "foobar"); } { // vespa.matchphase.diversity.mingroups - EXPECT_EQUAL(matchphase::DiversityMinGroups::NAME, vespalib::string("vespa.matchphase.diversity.mingroups")); - EXPECT_EQUAL(matchphase::DiversityMinGroups::DEFAULT_VALUE, 1u); + EXPECT_EQ(matchphase::DiversityMinGroups::NAME, vespalib::string("vespa.matchphase.diversity.mingroups")); + EXPECT_EQ(matchphase::DiversityMinGroups::DEFAULT_VALUE, 1u); Properties p; - EXPECT_EQUAL(matchphase::DiversityMinGroups::lookup(p), 1u); + EXPECT_EQ(matchphase::DiversityMinGroups::lookup(p), 1u); p.add("vespa.matchphase.diversity.mingroups", "5"); - EXPECT_EQUAL(matchphase::DiversityMinGroups::lookup(p), 5u); + EXPECT_EQ(matchphase::DiversityMinGroups::lookup(p), 5u); } { // vespa.hitcollector.heapsize - EXPECT_EQUAL(hitcollector::HeapSize::NAME, vespalib::string("vespa.hitcollector.heapsize")); - EXPECT_EQUAL(hitcollector::HeapSize::DEFAULT_VALUE, 100u); + EXPECT_EQ(hitcollector::HeapSize::NAME, vespalib::string("vespa.hitcollector.heapsize")); + EXPECT_EQ(hitcollector::HeapSize::DEFAULT_VALUE, 100u); Properties p; - EXPECT_EQUAL(hitcollector::HeapSize::lookup(p), 100u); + EXPECT_EQ(hitcollector::HeapSize::lookup(p), 100u); p.add("vespa.hitcollector.heapsize", "50"); - EXPECT_EQUAL(hitcollector::HeapSize::lookup(p), 50u); + EXPECT_EQ(hitcollector::HeapSize::lookup(p), 50u); } { // vespa.hitcollector.arraysize - EXPECT_EQUAL(hitcollector::ArraySize::NAME, vespalib::string("vespa.hitcollector.arraysize")); - EXPECT_EQUAL(hitcollector::ArraySize::DEFAULT_VALUE, 10000u); + EXPECT_EQ(hitcollector::ArraySize::NAME, vespalib::string("vespa.hitcollector.arraysize")); + EXPECT_EQ(hitcollector::ArraySize::DEFAULT_VALUE, 10000u); Properties p; - EXPECT_EQUAL(hitcollector::ArraySize::lookup(p), 10000u); + EXPECT_EQ(hitcollector::ArraySize::lookup(p), 10000u); p.add("vespa.hitcollector.arraysize", "50"); - EXPECT_EQUAL(hitcollector::ArraySize::lookup(p), 50u); + EXPECT_EQ(hitcollector::ArraySize::lookup(p), 50u); } { // vespa.hitcollector.estimatepoint - EXPECT_EQUAL(hitcollector::EstimatePoint::NAME, vespalib::string("vespa.hitcollector.estimatepoint")); - EXPECT_EQUAL(hitcollector::EstimatePoint::DEFAULT_VALUE, 0xffffffffu); + EXPECT_EQ(hitcollector::EstimatePoint::NAME, vespalib::string("vespa.hitcollector.estimatepoint")); + EXPECT_EQ(hitcollector::EstimatePoint::DEFAULT_VALUE, 0xffffffffu); Properties p; - EXPECT_EQUAL(hitcollector::EstimatePoint::lookup(p), 0xffffffffu); + EXPECT_EQ(hitcollector::EstimatePoint::lookup(p), 0xffffffffu); p.add("vespa.hitcollector.estimatepoint", "50"); - EXPECT_EQUAL(hitcollector::EstimatePoint::lookup(p), 50u); + EXPECT_EQ(hitcollector::EstimatePoint::lookup(p), 50u); } { // vespa.hitcollector.estimatelimit - EXPECT_EQUAL(hitcollector::EstimateLimit::NAME, vespalib::string("vespa.hitcollector.estimatelimit")); - EXPECT_EQUAL(hitcollector::EstimateLimit::DEFAULT_VALUE, 0xffffffffu); + EXPECT_EQ(hitcollector::EstimateLimit::NAME, vespalib::string("vespa.hitcollector.estimatelimit")); + EXPECT_EQ(hitcollector::EstimateLimit::DEFAULT_VALUE, 0xffffffffu); Properties p; - EXPECT_EQUAL(hitcollector::EstimateLimit::lookup(p), 0xffffffffu); + EXPECT_EQ(hitcollector::EstimateLimit::lookup(p), 0xffffffffu); p.add("vespa.hitcollector.estimatelimit", "50"); - EXPECT_EQUAL(hitcollector::EstimateLimit::lookup(p), 50u); + EXPECT_EQ(hitcollector::EstimateLimit::lookup(p), 50u); } { // vespa.hitcollector.rankscoredroplimit - EXPECT_EQUAL(hitcollector::RankScoreDropLimit::NAME, vespalib::string("vespa.hitcollector.rankscoredroplimit")); - search::feature_t got1 = hitcollector::RankScoreDropLimit::DEFAULT_VALUE; - EXPECT_TRUE(got1 != got1); - Properties p; - search::feature_t got2= hitcollector::RankScoreDropLimit::lookup(p); - EXPECT_TRUE(got2 != got2); + EXPECT_EQ(vespalib::string("vespa.hitcollector.rankscoredroplimit"), hitcollector::FirstPhaseRankScoreDropLimit::NAME); + Properties p; + auto got2 = hitcollector::FirstPhaseRankScoreDropLimit::lookup(p); + EXPECT_EQ(std::optional<search::feature_t>(), got2); + got2 = hitcollector::FirstPhaseRankScoreDropLimit::lookup(p, std::nullopt); + EXPECT_EQ(std::optional<search::feature_t>(), got2); + got2 = hitcollector::FirstPhaseRankScoreDropLimit::lookup(p, 4.5); + EXPECT_EQ(std::optional<search::feature_t>(4.5), got2); p.add("vespa.hitcollector.rankscoredroplimit", "-123456789.12345"); - EXPECT_EQUAL(hitcollector::RankScoreDropLimit::lookup(p), -123456789.12345); + EXPECT_EQ(std::optional<search::feature_t>(-123456789.12345), hitcollector::FirstPhaseRankScoreDropLimit::lookup(p)); p.clear().add("vespa.hitcollector.rankscoredroplimit", "123456789.12345"); - EXPECT_EQUAL(hitcollector::RankScoreDropLimit::lookup(p), 123456789.12345); + EXPECT_EQ(std::optional<search::feature_t>(123456789.12345), hitcollector::FirstPhaseRankScoreDropLimit::lookup(p)); } { // vespa.fieldweight. - EXPECT_EQUAL(FieldWeight::BASE_NAME, vespalib::string("vespa.fieldweight.")); - EXPECT_EQUAL(FieldWeight::DEFAULT_VALUE, 100u); + EXPECT_EQ(FieldWeight::BASE_NAME, vespalib::string("vespa.fieldweight.")); + EXPECT_EQ(FieldWeight::DEFAULT_VALUE, 100u); Properties p; - EXPECT_EQUAL(FieldWeight::lookup(p, "foo"), 100u); + EXPECT_EQ(FieldWeight::lookup(p, "foo"), 100u); p.add("vespa.fieldweight.foo", "200"); - EXPECT_EQUAL(FieldWeight::lookup(p, "foo"), 200u); + EXPECT_EQ(FieldWeight::lookup(p, "foo"), 200u); } { // vespa.isfilterfield. - EXPECT_EQUAL(IsFilterField::BASE_NAME, "vespa.isfilterfield."); - EXPECT_EQUAL(IsFilterField::DEFAULT_VALUE, "false"); + EXPECT_EQ(IsFilterField::BASE_NAME, "vespa.isfilterfield."); + EXPECT_EQ(IsFilterField::DEFAULT_VALUE, "false"); Properties p; EXPECT_TRUE(!IsFilterField::check(p, "foo")); p.add("vespa.isfilterfield.foo", "true"); @@ -438,192 +442,206 @@ TEST("test stuff") { EXPECT_TRUE(IsFilterField::check(p, "bar")); } { - EXPECT_EQUAL(mutate::on_match::Attribute::NAME, vespalib::string("vespa.mutate.on_match.attribute")); - EXPECT_EQUAL(mutate::on_match::Attribute::DEFAULT_VALUE, ""); + EXPECT_EQ(mutate::on_match::Attribute::NAME, vespalib::string("vespa.mutate.on_match.attribute")); + EXPECT_EQ(mutate::on_match::Attribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(mutate::on_match::Attribute::lookup(p), ""); + EXPECT_EQ(mutate::on_match::Attribute::lookup(p), ""); p.add("vespa.mutate.on_match.attribute", "foobar"); - EXPECT_EQUAL(mutate::on_match::Attribute::lookup(p), "foobar"); + EXPECT_EQ(mutate::on_match::Attribute::lookup(p), "foobar"); } { - EXPECT_EQUAL(mutate::on_match::Operation::NAME, vespalib::string("vespa.mutate.on_match.operation")); - EXPECT_EQUAL(mutate::on_match::Operation::DEFAULT_VALUE, ""); + EXPECT_EQ(mutate::on_match::Operation::NAME, vespalib::string("vespa.mutate.on_match.operation")); + EXPECT_EQ(mutate::on_match::Operation::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(mutate::on_match::Operation::lookup(p), ""); + EXPECT_EQ(mutate::on_match::Operation::lookup(p), ""); p.add("vespa.mutate.on_match.operation", "+=1"); - EXPECT_EQUAL(mutate::on_match::Operation::lookup(p), "+=1"); + EXPECT_EQ(mutate::on_match::Operation::lookup(p), "+=1"); } { - EXPECT_EQUAL(mutate::on_first_phase::Attribute::NAME, vespalib::string("vespa.mutate.on_first_phase.attribute")); - EXPECT_EQUAL(mutate::on_first_phase::Attribute::DEFAULT_VALUE, ""); + EXPECT_EQ(mutate::on_first_phase::Attribute::NAME, vespalib::string("vespa.mutate.on_first_phase.attribute")); + EXPECT_EQ(mutate::on_first_phase::Attribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(mutate::on_first_phase::Attribute::lookup(p), ""); + EXPECT_EQ(mutate::on_first_phase::Attribute::lookup(p), ""); p.add("vespa.mutate.on_first_phase.attribute", "foobar"); - EXPECT_EQUAL(mutate::on_first_phase::Attribute::lookup(p), "foobar"); + EXPECT_EQ(mutate::on_first_phase::Attribute::lookup(p), "foobar"); } { - EXPECT_EQUAL(mutate::on_first_phase::Operation::NAME, vespalib::string("vespa.mutate.on_first_phase.operation")); - EXPECT_EQUAL(mutate::on_first_phase::Operation::DEFAULT_VALUE, ""); + EXPECT_EQ(mutate::on_first_phase::Operation::NAME, vespalib::string("vespa.mutate.on_first_phase.operation")); + EXPECT_EQ(mutate::on_first_phase::Operation::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(mutate::on_first_phase::Operation::lookup(p), ""); + EXPECT_EQ(mutate::on_first_phase::Operation::lookup(p), ""); p.add("vespa.mutate.on_first_phase.operation", "+=1"); - EXPECT_EQUAL(mutate::on_first_phase::Operation::lookup(p), "+=1"); + EXPECT_EQ(mutate::on_first_phase::Operation::lookup(p), "+=1"); } { - EXPECT_EQUAL(mutate::on_second_phase::Attribute::NAME, vespalib::string("vespa.mutate.on_second_phase.attribute")); - EXPECT_EQUAL(mutate::on_second_phase::Attribute::DEFAULT_VALUE, ""); + EXPECT_EQ(mutate::on_second_phase::Attribute::NAME, vespalib::string("vespa.mutate.on_second_phase.attribute")); + EXPECT_EQ(mutate::on_second_phase::Attribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(mutate::on_second_phase::Attribute::lookup(p), ""); + EXPECT_EQ(mutate::on_second_phase::Attribute::lookup(p), ""); p.add("vespa.mutate.on_second_phase.attribute", "foobar"); - EXPECT_EQUAL(mutate::on_second_phase::Attribute::lookup(p), "foobar"); + EXPECT_EQ(mutate::on_second_phase::Attribute::lookup(p), "foobar"); } { - EXPECT_EQUAL(mutate::on_second_phase::Operation::NAME, vespalib::string("vespa.mutate.on_second_phase.operation")); - EXPECT_EQUAL(mutate::on_second_phase::Operation::DEFAULT_VALUE, ""); + EXPECT_EQ(mutate::on_second_phase::Operation::NAME, vespalib::string("vespa.mutate.on_second_phase.operation")); + EXPECT_EQ(mutate::on_second_phase::Operation::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(mutate::on_second_phase::Operation::lookup(p), ""); + EXPECT_EQ(mutate::on_second_phase::Operation::lookup(p), ""); p.add("vespa.mutate.on_second_phase.operation", "+=1"); - EXPECT_EQUAL(mutate::on_second_phase::Operation::lookup(p), "+=1"); + EXPECT_EQ(mutate::on_second_phase::Operation::lookup(p), "+=1"); } { - EXPECT_EQUAL(mutate::on_summary::Attribute::NAME, vespalib::string("vespa.mutate.on_summary.attribute")); - EXPECT_EQUAL(mutate::on_summary::Attribute::DEFAULT_VALUE, ""); + EXPECT_EQ(mutate::on_summary::Attribute::NAME, vespalib::string("vespa.mutate.on_summary.attribute")); + EXPECT_EQ(mutate::on_summary::Attribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(mutate::on_summary::Attribute::lookup(p), ""); + EXPECT_EQ(mutate::on_summary::Attribute::lookup(p), ""); p.add("vespa.mutate.on_summary.attribute", "foobar"); - EXPECT_EQUAL(mutate::on_summary::Attribute::lookup(p), "foobar"); + EXPECT_EQ(mutate::on_summary::Attribute::lookup(p), "foobar"); } { - EXPECT_EQUAL(mutate::on_summary::Operation::NAME, vespalib::string("vespa.mutate.on_summary.operation")); - EXPECT_EQUAL(mutate::on_summary::Operation::DEFAULT_VALUE, ""); + EXPECT_EQ(mutate::on_summary::Operation::NAME, vespalib::string("vespa.mutate.on_summary.operation")); + EXPECT_EQ(mutate::on_summary::Operation::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(mutate::on_summary::Operation::lookup(p), ""); + EXPECT_EQ(mutate::on_summary::Operation::lookup(p), ""); p.add("vespa.mutate.on_summary.operation", "+=1"); - EXPECT_EQUAL(mutate::on_summary::Operation::lookup(p), "+=1"); + EXPECT_EQ(mutate::on_summary::Operation::lookup(p), "+=1"); } { - EXPECT_EQUAL(execute::onmatch::Attribute::NAME, vespalib::string("vespa.execute.onmatch.attribute")); - EXPECT_EQUAL(execute::onmatch::Attribute::DEFAULT_VALUE, ""); + EXPECT_EQ(execute::onmatch::Attribute::NAME, vespalib::string("vespa.execute.onmatch.attribute")); + EXPECT_EQ(execute::onmatch::Attribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(execute::onmatch::Attribute::lookup(p), ""); + EXPECT_EQ(execute::onmatch::Attribute::lookup(p), ""); p.add("vespa.execute.onmatch.attribute", "foobar"); - EXPECT_EQUAL(execute::onmatch::Attribute::lookup(p), "foobar"); + EXPECT_EQ(execute::onmatch::Attribute::lookup(p), "foobar"); } { - EXPECT_EQUAL(execute::onmatch::Operation::NAME, vespalib::string("vespa.execute.onmatch.operation")); - EXPECT_EQUAL(execute::onmatch::Operation::DEFAULT_VALUE, ""); + EXPECT_EQ(execute::onmatch::Operation::NAME, vespalib::string("vespa.execute.onmatch.operation")); + EXPECT_EQ(execute::onmatch::Operation::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(execute::onmatch::Operation::lookup(p), ""); + EXPECT_EQ(execute::onmatch::Operation::lookup(p), ""); p.add("vespa.execute.onmatch.operation", "++"); - EXPECT_EQUAL(execute::onmatch::Operation::lookup(p), "++"); + EXPECT_EQ(execute::onmatch::Operation::lookup(p), "++"); } { - EXPECT_EQUAL(execute::onrerank::Attribute::NAME, vespalib::string("vespa.execute.onrerank.attribute")); - EXPECT_EQUAL(execute::onrerank::Attribute::DEFAULT_VALUE, ""); + EXPECT_EQ(execute::onrerank::Attribute::NAME, vespalib::string("vespa.execute.onrerank.attribute")); + EXPECT_EQ(execute::onrerank::Attribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(execute::onrerank::Attribute::lookup(p), ""); + EXPECT_EQ(execute::onrerank::Attribute::lookup(p), ""); p.add("vespa.execute.onrerank.attribute", "foobar"); - EXPECT_EQUAL(execute::onrerank::Attribute::lookup(p), "foobar"); + EXPECT_EQ(execute::onrerank::Attribute::lookup(p), "foobar"); } { - EXPECT_EQUAL(execute::onrerank::Operation::NAME, vespalib::string("vespa.execute.onrerank.operation")); - EXPECT_EQUAL(execute::onrerank::Operation::DEFAULT_VALUE, ""); + EXPECT_EQ(execute::onrerank::Operation::NAME, vespalib::string("vespa.execute.onrerank.operation")); + EXPECT_EQ(execute::onrerank::Operation::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(execute::onrerank::Operation::lookup(p), ""); + EXPECT_EQ(execute::onrerank::Operation::lookup(p), ""); p.add("vespa.execute.onrerank.operation", "++"); - EXPECT_EQUAL(execute::onrerank::Operation::lookup(p), "++"); + EXPECT_EQ(execute::onrerank::Operation::lookup(p), "++"); } { - EXPECT_EQUAL(execute::onsummary::Attribute::NAME, vespalib::string("vespa.execute.onsummary.attribute")); - EXPECT_EQUAL(execute::onsummary::Attribute::DEFAULT_VALUE, ""); + EXPECT_EQ(execute::onsummary::Attribute::NAME, vespalib::string("vespa.execute.onsummary.attribute")); + EXPECT_EQ(execute::onsummary::Attribute::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(execute::onsummary::Attribute::lookup(p), ""); + EXPECT_EQ(execute::onsummary::Attribute::lookup(p), ""); p.add("vespa.execute.onsummary.attribute", "foobar"); - EXPECT_EQUAL(execute::onsummary::Attribute::lookup(p), "foobar"); + EXPECT_EQ(execute::onsummary::Attribute::lookup(p), "foobar"); } { - EXPECT_EQUAL(execute::onsummary::Operation::NAME, vespalib::string("vespa.execute.onsummary.operation")); - EXPECT_EQUAL(execute::onsummary::Operation::DEFAULT_VALUE, ""); + EXPECT_EQ(execute::onsummary::Operation::NAME, vespalib::string("vespa.execute.onsummary.operation")); + EXPECT_EQ(execute::onsummary::Operation::DEFAULT_VALUE, ""); Properties p; - EXPECT_EQUAL(execute::onsummary::Operation::lookup(p), ""); + EXPECT_EQ(execute::onsummary::Operation::lookup(p), ""); p.add("vespa.execute.onsummary.operation", "++"); - EXPECT_EQUAL(execute::onsummary::Operation::lookup(p), "++"); + EXPECT_EQ(execute::onsummary::Operation::lookup(p), "++"); } { - EXPECT_EQUAL(softtimeout::Enabled::NAME, vespalib::string("vespa.softtimeout.enable")); + EXPECT_EQ(softtimeout::Enabled::NAME, vespalib::string("vespa.softtimeout.enable")); EXPECT_TRUE(softtimeout::Enabled::DEFAULT_VALUE); Properties p; p.add(softtimeout::Enabled::NAME, "false"); EXPECT_FALSE(softtimeout::Enabled::lookup(p)); } { - EXPECT_EQUAL(softtimeout::Factor::NAME, vespalib::string("vespa.softtimeout.factor")); - EXPECT_EQUAL(0.5, softtimeout::Factor::DEFAULT_VALUE); + EXPECT_EQ(softtimeout::Factor::NAME, vespalib::string("vespa.softtimeout.factor")); + EXPECT_EQ(0.5, softtimeout::Factor::DEFAULT_VALUE); Properties p; p.add(softtimeout::Factor::NAME, "0.33"); - EXPECT_EQUAL(0.33, softtimeout::Factor::lookup(p)); + EXPECT_EQ(0.33, softtimeout::Factor::lookup(p)); } { - EXPECT_EQUAL(softtimeout::TailCost::NAME, vespalib::string("vespa.softtimeout.tailcost")); - EXPECT_EQUAL(0.1, softtimeout::TailCost::DEFAULT_VALUE); + EXPECT_EQ(softtimeout::TailCost::NAME, vespalib::string("vespa.softtimeout.tailcost")); + EXPECT_EQ(0.1, softtimeout::TailCost::DEFAULT_VALUE); Properties p; p.add(softtimeout::TailCost::NAME, "0.17"); - EXPECT_EQUAL(0.17, softtimeout::TailCost::lookup(p)); + EXPECT_EQ(0.17, softtimeout::TailCost::lookup(p)); } } } -TEST("test attribute type properties") +TEST(PropertiesTest, test_attribute_type_properties) { Properties p; p.add("vespa.type.attribute.foo", "tensor(x[10])"); - EXPECT_EQUAL("tensor(x[10])", type::Attribute::lookup(p, "foo")); - EXPECT_EQUAL("", type::Attribute::lookup(p, "bar")); + EXPECT_EQ("tensor(x[10])", type::Attribute::lookup(p, "foo")); + EXPECT_EQ("", type::Attribute::lookup(p, "bar")); } -TEST("test query feature type properties") +TEST(PropertiesTest, test_query_feature_type_properties) { Properties p; p.add("vespa.type.query.foo", "tensor(x[10])"); - EXPECT_EQUAL("tensor(x[10])", type::QueryFeature::lookup(p, "foo")); - EXPECT_EQUAL("", type::QueryFeature::lookup(p, "bar")); + EXPECT_EQ("tensor(x[10])", type::QueryFeature::lookup(p, "foo")); + EXPECT_EQ("", type::QueryFeature::lookup(p, "bar")); } -TEST("test integer lookup") +TEST(PropertiesTest, test_integer_lookup) { - EXPECT_EQUAL(matching::NumThreadsPerSearch::NAME, vespalib::string("vespa.matching.numthreadspersearch")); - EXPECT_EQUAL(matching::NumThreadsPerSearch::DEFAULT_VALUE, std::numeric_limits<uint32_t>::max()); + EXPECT_EQ(matching::NumThreadsPerSearch::NAME, vespalib::string("vespa.matching.numthreadspersearch")); + EXPECT_EQ(matching::NumThreadsPerSearch::DEFAULT_VALUE, std::numeric_limits<uint32_t>::max()); { Properties p; p.add("vespa.matching.numthreadspersearch", "50"); - EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + EXPECT_EQ(matching::NumThreadsPerSearch::lookup(p), 50u); } { Properties p; p.add("vespa.matching.numthreadspersearch", "50 "); - EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + EXPECT_EQ(matching::NumThreadsPerSearch::lookup(p), 50u); } { Properties p; p.add("vespa.matching.numthreadspersearch", " 50"); - EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + EXPECT_EQ(matching::NumThreadsPerSearch::lookup(p), 50u); } { Properties p; p.add("vespa.matching.numthreadspersearch", " "); - EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), matching::NumThreadsPerSearch::DEFAULT_VALUE); + EXPECT_EQ(matching::NumThreadsPerSearch::lookup(p), matching::NumThreadsPerSearch::DEFAULT_VALUE); } { Properties p; p.add("vespa.matching.numthreadspersearch", "50x"); - EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + EXPECT_EQ(matching::NumThreadsPerSearch::lookup(p), 50u); } { Properties p; p.add("vespa.matching.numthreadspersearch", "x"); - EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), matching::NumThreadsPerSearch::DEFAULT_VALUE); + EXPECT_EQ(matching::NumThreadsPerSearch::lookup(p), matching::NumThreadsPerSearch::DEFAULT_VALUE); } } +TEST(PropertiesTest, second_phase_rank_score_drop_limit) +{ + vespalib::stringref name = hitcollector::SecondPhaseRankScoreDropLimit::NAME; + EXPECT_EQ(vespalib::string("vespa.hitcollector.secondphase.rankscoredroplimit"), name); + Properties p; + EXPECT_EQ(std::optional<search::feature_t>(), hitcollector::SecondPhaseRankScoreDropLimit::lookup(p)); + EXPECT_EQ(std::optional<search::feature_t>(4.0), hitcollector::SecondPhaseRankScoreDropLimit::lookup(p, 4.0)); + p.add(name, "-123456789.12345"); + EXPECT_EQ(std::optional<search::feature_t>(-123456789.12345), hitcollector::SecondPhaseRankScoreDropLimit::lookup(p)); + EXPECT_EQ(std::optional<search::feature_t>(-123456789.12345), hitcollector::SecondPhaseRankScoreDropLimit::lookup(p, 4.0)); + p.clear().add(name, "123456789.12345"); + EXPECT_EQ(std::optional<search::feature_t>(123456789.12345), hitcollector::SecondPhaseRankScoreDropLimit::lookup(p)); + EXPECT_EQ(std::optional<search::feature_t>(123456789.12345), hitcollector::SecondPhaseRankScoreDropLimit::lookup(p, 4.0)); +} -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/fef/rank_program/CMakeLists.txt b/searchlib/src/tests/fef/rank_program/CMakeLists.txt index 2956e9ccfd8..23e5ff007fc 100644 --- a/searchlib/src/tests/fef/rank_program/CMakeLists.txt +++ b/searchlib/src/tests/fef/rank_program/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_rank_program_test_app TEST SOURCES rank_program_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_rank_program_test_app COMMAND searchlib_rank_program_test_app) diff --git a/searchlib/src/tests/fef/resolver/CMakeLists.txt b/searchlib/src/tests/fef/resolver/CMakeLists.txt index 107b2daf46a..8c7f806a1a6 100644 --- a/searchlib/src/tests/fef/resolver/CMakeLists.txt +++ b/searchlib/src/tests/fef/resolver/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_resolver_test_app TEST SOURCES resolver_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_resolver_test_app COMMAND searchlib_resolver_test_app) diff --git a/searchlib/src/tests/fef/table/CMakeLists.txt b/searchlib/src/tests/fef/table/CMakeLists.txt index 6cc6856e0ce..c7408da325d 100644 --- a/searchlib/src/tests/fef/table/CMakeLists.txt +++ b/searchlib/src/tests/fef/table/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(searchlib_table_test_app TEST SOURCES table_test.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_table_test_app COMMAND searchlib_table_test_app) diff --git a/searchlib/src/tests/fef/table/table_test.cpp b/searchlib/src/tests/fef/table/table_test.cpp index b0a47a8bdbc..9c15274c093 100644 --- a/searchlib/src/tests/fef/table/table_test.cpp +++ b/searchlib/src/tests/fef/table/table_test.cpp @@ -1,18 +1,21 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/fef/filetablefactory.h> #include <vespa/searchlib/fef/functiontablefactory.h> #include <vespa/searchlib/fef/tablemanager.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> #include <fstream> #include <iostream> -namespace search { -namespace fef { +namespace search::fef { -class TableTest : public vespalib::TestApp +class TableTest : public ::testing::Test { -private: +protected: + const std::string _tables1Dir; + const std::string _tables2Dir; + bool assertTable(const Table & act, const Table & exp); bool assertCreateTable(const ITableFactory & tf, const vespalib::string & name, const Table & exp); void testTable(); @@ -20,29 +23,32 @@ private: void testFunctionTableFactory(); void testTableManager(); - const std::string _tables1Dir; - const std::string _tables2Dir; -public: TableTest(); - ~TableTest(); - int Main() override; + ~TableTest() override; }; -TableTest::TableTest() : - vespalib::TestApp(), - _tables1Dir(TEST_PATH("tables1")), - _tables2Dir(TEST_PATH("tables2")) +TableTest::TableTest() + : ::testing::Test(), + _tables1Dir(TEST_PATH("tables1")), + _tables2Dir(TEST_PATH("tables2")) { } -TableTest::~TableTest() {} +TableTest::~TableTest() = default; bool TableTest::assertTable(const Table & act, const Table & exp) { - if (!EXPECT_EQUAL(act.size(), exp.size())) return false; + bool failed = false; + EXPECT_EQ(act.size(), exp.size()) << (failed = true, ""); + if (failed) { + return false; + } for (size_t i = 0; i < act.size(); ++i) { - if (!EXPECT_APPROX(act[i], exp[i], 0.01)) return false; + EXPECT_NEAR(act[i], exp[i], 0.01) << (failed = true, ""); + if (failed) { + return false; + } } return true; } @@ -51,33 +57,35 @@ bool TableTest::assertCreateTable(const ITableFactory & tf, const vespalib::string & name, const Table & exp) { Table::SP t = tf.createTable(name); - if (!EXPECT_TRUE(t.get() != NULL)) return false; + bool failed = false; + EXPECT_TRUE(t.get() != nullptr) << (failed = true, ""); + if (failed) { + return false; + } return assertTable(*t, exp); } -void -TableTest::testTable() +TEST_F(TableTest, table) { Table t; - EXPECT_EQUAL(t.size(), 0u); - EXPECT_EQUAL(t.max(), -std::numeric_limits<double>::max()); + EXPECT_EQ(t.size(), 0u); + EXPECT_EQ(t.max(), -std::numeric_limits<double>::max()); t.add(1).add(2); - EXPECT_EQUAL(t.size(), 2u); - EXPECT_EQUAL(t.max(), 2); - EXPECT_EQUAL(t[0], 1); - EXPECT_EQUAL(t[1], 2); + EXPECT_EQ(t.size(), 2u); + EXPECT_EQ(t.max(), 2); + EXPECT_EQ(t[0], 1); + EXPECT_EQ(t[1], 2); t.add(10); - EXPECT_EQUAL(t.size(), 3u); - EXPECT_EQUAL(t.max(), 10); - EXPECT_EQUAL(t[2], 10); + EXPECT_EQ(t.size(), 3u); + EXPECT_EQ(t.max(), 10); + EXPECT_EQ(t[2], 10); t.add(5); - EXPECT_EQUAL(t.size(), 4u); - EXPECT_EQUAL(t.max(), 10); - EXPECT_EQUAL(t[3], 5); + EXPECT_EQ(t.size(), 4u); + EXPECT_EQ(t.max(), 10); + EXPECT_EQ(t[3], 5); } -void -TableTest::testFileTableFactory() +TEST_F(TableTest, file_table_factory) { { FileTableFactory ftf(_tables1Dir); @@ -90,8 +98,7 @@ TableTest::testFileTableFactory() } } -void -TableTest::testFunctionTableFactory() +TEST_F(TableTest, function_table_factory) { FunctionTableFactory ftf(2); EXPECT_TRUE(assertCreateTable(ftf, "expdecay(400,12)", @@ -117,8 +124,7 @@ TableTest::testFunctionTableFactory() EXPECT_TRUE(ftf.createTable("none)(").get() == NULL); } -void -TableTest::testTableManager() +TEST_F(TableTest, table_manager) { { TableManager tm; @@ -148,20 +154,6 @@ TableTest::testTableManager() } } -int -TableTest::Main() -{ - TEST_INIT("table_test"); - - testTable(); - testFileTableFactory(); - testFunctionTableFactory(); - testTableManager(); - - TEST_DONE(); -} - -} } -TEST_APPHOOK(search::fef::TableTest); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/fef/termfieldmodel/CMakeLists.txt b/searchlib/src/tests/fef/termfieldmodel/CMakeLists.txt index 70c3b952e49..3dc46b06b19 100644 --- a/searchlib/src/tests/fef/termfieldmodel/CMakeLists.txt +++ b/searchlib/src/tests/fef/termfieldmodel/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_termfieldmodel_test_app TEST SOURCES termfieldmodel_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_termfieldmodel_test_app COMMAND searchlib_termfieldmodel_test_app) diff --git a/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt b/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt index fc4de86cc24..d757999def0 100644 --- a/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt +++ b/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_termmatchdatamerger_test_app TEST SOURCES termmatchdatamerger_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_termmatchdatamerger_test_app COMMAND searchlib_termmatchdatamerger_test_app) diff --git a/searchlib/src/tests/fileheadertk/CMakeLists.txt b/searchlib/src/tests/fileheadertk/CMakeLists.txt index 78e642271f1..3dc90ca6250 100644 --- a/searchlib/src/tests/fileheadertk/CMakeLists.txt +++ b/searchlib/src/tests/fileheadertk/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_fileheadertk_test_app TEST SOURCES fileheadertk_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_fileheadertk_test_app COMMAND searchlib_fileheadertk_test_app) diff --git a/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp b/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp index 7200566d735..cb1dcc6fb8b 100644 --- a/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp +++ b/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/util/fileheadertk.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/forcelink/CMakeLists.txt b/searchlib/src/tests/forcelink/CMakeLists.txt index 9f98737c158..e5c14129a98 100644 --- a/searchlib/src/tests/forcelink/CMakeLists.txt +++ b/searchlib/src/tests/forcelink/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_forcelink_test_app TEST SOURCES forcelink_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_forcelink_test_app COMMAND searchlib_forcelink_test_app) diff --git a/searchlib/src/tests/forcelink/forcelink_test.cpp b/searchlib/src/tests/forcelink/forcelink_test.cpp index 38f02df1782..bf25194acf6 100644 --- a/searchlib/src/tests/forcelink/forcelink_test.cpp +++ b/searchlib/src/tests/forcelink/forcelink_test.cpp @@ -1,17 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("forcelink_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/expression/forcelink.hpp> #include <vespa/searchlib/aggregation/forcelink.hpp> -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("forcelink_test"); +TEST("forcelink_test") { forcelink_searchlib_expression(); forcelink_searchlib_aggregation(); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/grouping/CMakeLists.txt b/searchlib/src/tests/grouping/CMakeLists.txt index 3bfedc17c36..b9273cefaa8 100644 --- a/searchlib/src/tests/grouping/CMakeLists.txt +++ b/searchlib/src/tests/grouping/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_grouping_test_app TEST SOURCES grouping_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_grouping_test_app COMMAND searchlib_grouping_test_app) @@ -11,20 +11,20 @@ vespa_add_executable(searchlib_hyperloglog_test_app TEST SOURCES hyperloglog_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_hyperloglog_test_app COMMAND searchlib_hyperloglog_test_app) vespa_add_executable(searchlib_sketch_test_app TEST SOURCES sketch_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_sketch_test_app COMMAND searchlib_sketch_test_app) vespa_add_executable(searchlib_grouping_serialization_test_app TEST SOURCES grouping_serialization_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_grouping_serialization_test_app COMMAND searchlib_grouping_serialization_test_app) diff --git a/searchlib/src/tests/grouping/grouping_test.cpp b/searchlib/src/tests/grouping/grouping_test.cpp index d28ab1d3d66..952f5d2a5db 100644 --- a/searchlib/src/tests/grouping/grouping_test.cpp +++ b/searchlib/src/tests/grouping/grouping_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/extendableattributes.h> @@ -8,6 +8,8 @@ #include <vespa/searchlib/aggregation/hitsaggregationresult.h> #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/aggregation/predicates.h> +#include <vespa/searchlib/aggregation/modifiers.h> +#include <vespa/searchlib/expression/documentfieldnode.h> #include <vespa/searchlib/expression/fixedwidthbucketfunctionnode.h> #include <vespa/searchlib/test/make_attribute_map_lookup_node.h> #include <vespa/searchcommon/common/undefinedvalues.h> @@ -51,7 +53,7 @@ public: add(val); } } - AttrBuilder(const std::string &name) + explicit AttrBuilder(const std::string &name) : _attr(new A(name)), _attrSP(_attr) { @@ -127,19 +129,19 @@ private: ResultBuilder _result; IAttributeContext::UP _attrCtx; - AggregationContext(const AggregationContext &); - AggregationContext &operator=(const AggregationContext &); - public: AggregationContext(); + AggregationContext(const AggregationContext &) = delete; + AggregationContext &operator=(const AggregationContext &) = delete; ~AggregationContext(); ResultBuilder &result() { return _result; } - void add(AttributeVector::SP attr) { + void add(const AttributeVector::SP & attr) { _attrMan.add(attr); } void setup(Grouping &g) { - g.configureStaticStuff(ConfigureStaticParams(_attrCtx.get(), 0)); + g.configureStaticStuff(ConfigureStaticParams(_attrCtx.get(), nullptr)); } + const IAttributeContext & attrCtx() const { return *_attrCtx; } }; AggregationContext::AggregationContext() : _attrMan(), _result(), _attrCtx(_attrMan.createContext()) {} @@ -688,7 +690,7 @@ TEST("testAggregationGroupCapping") EXPECT_TRUE(testAggregation(ctx, request, expect)); } { - AddFunctionNode *add = new AddFunctionNode(); + auto add = std::make_unique<AddFunctionNode>(); add->addArg(MU<AggregationRefNode>(0)); add->appendArg(MU<ConstantNode>(MU<Int64ResultNode>(3))); @@ -697,7 +699,7 @@ TEST("testAggregationGroupCapping") .setLastLevel(1) .addLevel(std::move(GroupingLevel().setMaxGroups(3).setExpression(MU<AttributeNode>("attr")) .addAggregationResult(createAggr<SumAggregationResult>(MU<AttributeNode>("attr"))) - .addOrderBy(ExpressionNode::UP(add), false))); + .addOrderBy(std::move(add), false))); Group expect; expect.addChild(Group().setId(Int64ResultNode(7)).setRank(RawRank(7)) @@ -1826,4 +1828,31 @@ TEST("testAttributeMapLookup") testAggregationSimple(ctx, MaxAggregationResult(), Int64ResultNode(100), "smap{attribute(key2)}.weight"); } +TEST("test that non-attributes are converted to document field nodes") { + AggregationContext ctx; + ctx.add(IntAttrBuilder("attr").sp()); + + Grouping attrRequest; + attrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("attr")))); + aggregation::NonAttribute2DocumentAccessor optional2DocumentAccessor(ctx.attrCtx()); + attrRequest.select(optional2DocumentAccessor, optional2DocumentAccessor); + EXPECT_TRUE(attrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(AttributeNode::classId)); + + Grouping nonAttrRequest; + nonAttrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("non-attr")))); + nonAttrRequest.select(optional2DocumentAccessor, optional2DocumentAccessor); + EXPECT_TRUE(nonAttrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(DocumentFieldNode::classId)); +} + +TEST("test that attributes can be unconditionally converted to document field nodes") { + AggregationContext ctx; + ctx.add(IntAttrBuilder("attr").sp()); + + Grouping attrRequest; + attrRequest.setRoot(Group().addResult(SumAggregationResult().setExpression(MU<AttributeNode>("attr")))); + aggregation::Attribute2DocumentAccessor attr2DocumentAccessor; + attrRequest.select(attr2DocumentAccessor, attr2DocumentAccessor); + EXPECT_TRUE(attrRequest.getRoot().getAggregationResult(0).getExpression()->inherits(DocumentFieldNode::classId)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/grouping/hyperloglog_test.cpp b/searchlib/src/tests/grouping/hyperloglog_test.cpp index 61d81b76b08..281bad55735 100644 --- a/searchlib/src/tests/grouping/hyperloglog_test.cpp +++ b/searchlib/src/tests/grouping/hyperloglog_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("hyperloglog_test"); #include <vespa/searchlib/grouping/hyperloglog.h> #include <vespa/vespalib/objects/nboserializer.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using vespalib::NBOSerializer; using vespalib::nbostream; diff --git a/searchlib/src/tests/grouping/sketch_test.cpp b/searchlib/src/tests/grouping/sketch_test.cpp index e75e47266da..ae1052e9f59 100644 --- a/searchlib/src/tests/grouping/sketch_test.cpp +++ b/searchlib/src/tests/grouping/sketch_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchlib/grouping/sketch.h> #include <vespa/vespalib/objects/nboserializer.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/groupingengine/CMakeLists.txt b/searchlib/src/tests/groupingengine/CMakeLists.txt index bfbcaa1535b..1ee6231a82b 100644 --- a/searchlib/src/tests/groupingengine/CMakeLists.txt +++ b/searchlib/src/tests/groupingengine/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(searchlib_groupingengine_test_app SOURCES groupingengine_test.cpp DEPENDS - searchlib + vespa_searchlib ) #vespa_add_test(NAME searchlib_groupingengine_test_app COMMAND searchlib_groupingengine_test_app) vespa_add_executable(searchlib_groupingengine_benchmark_app SOURCES groupingengine_benchmark.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_groupingengine_benchmark_app COMMAND searchlib_groupingengine_benchmark_app BENCHMARK) diff --git a/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp b/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp index 0c201da48d9..e65c5d812a4 100644 --- a/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp +++ b/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/extendableattributes.h> @@ -133,27 +133,18 @@ AggregationContext::AggregationContext() AggregationContext::~AggregationContext() = default; //----------------------------------------------------------------------------- -class Test : public TestApp +class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { public: + CheckAttributeReferences() : _numrefs(0) { } + int _numrefs; private: - bool testAggregation(AggregationContext &ctx, const Grouping &request, bool useEngine); - void benchmarkIntegerSum(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups); - void benchmarkIntegerCount(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups); - class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate - { - public: - CheckAttributeReferences() : _numrefs(0) { } - int _numrefs; - private: - void execute(vespalib::Identifiable &obj) override { - if (static_cast<AttributeNode &>(obj).getAttribute() != nullptr) { - _numrefs++; - } + void execute(vespalib::Identifiable &obj) override { + if (static_cast<AttributeNode &>(obj).getAttribute() != nullptr) { + _numrefs++; } - virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } - }; - int Main() override; + } + virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } }; //----------------------------------------------------------------------------- @@ -162,9 +153,7 @@ private: * Run the given grouping request and verify that the resulting group * tree matches the expected value. **/ -bool -Test::testAggregation(AggregationContext &ctx, const Grouping &request, bool useEngine) -{ +bool testAggregation(AggregationContext &ctx, const Grouping &request, bool useEngine) { Grouping tmp = request; // create local copy ctx.setup(tmp); if (useEngine) { @@ -183,9 +172,7 @@ Test::testAggregation(AggregationContext &ctx, const Grouping &request, bool use #define MU std::make_unique -void -Test::benchmarkIntegerSum(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups) -{ +void benchmarkIntegerSum(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups) { IntAttrBuilder attrB("attr0"); for (size_t i=0; i < numDocs; i++) { attrB.add(i); @@ -212,9 +199,7 @@ Test::benchmarkIntegerSum(bool useEngine, size_t numDocs, size_t numQueries, int } } -void -Test::benchmarkIntegerCount(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups) -{ +void benchmarkIntegerCount(bool useEngine, size_t numDocs, size_t numQueries, int64_t maxGroups) { IntAttrBuilder attrB("attr0"); for (size_t i=0; i < numDocs; i++) { attrB.add(i); @@ -241,34 +226,31 @@ Test::benchmarkIntegerCount(bool useEngine, size_t numDocs, size_t numQueries, i } } -int -Test::Main() -{ +TEST_MAIN() { size_t numDocs = 1000000; size_t numQueries = 1000; int64_t maxGroups = -1; bool useEngine = true; vespalib::string idType = "int"; vespalib::string aggrType = "sum"; - if (_argc > 1) { - useEngine = (strcmp(_argv[1], "tree") != 0); + if (argc > 1) { + useEngine = (strcmp(argv[1], "tree") != 0); } - if (_argc > 2) { - idType = _argv[2]; + if (argc > 2) { + idType = argv[2]; } - if (_argc > 3) { - aggrType = _argv[3]; + if (argc > 3) { + aggrType = argv[3]; } - if (_argc > 4) { - numDocs = strtol(_argv[4], nullptr, 0); + if (argc > 4) { + numDocs = strtol(argv[4], nullptr, 0); } - if (_argc > 5) { - numQueries = strtol(_argv[5], nullptr, 0); + if (argc > 5) { + numQueries = strtol(argv[5], nullptr, 0); } - if (_argc > 6) { - maxGroups = strtol(_argv[6], nullptr, 0); + if (argc > 6) { + maxGroups = strtol(argv[6], nullptr, 0); } - TEST_INIT("grouping_benchmark"); LOG(info, "sizeof(Group) = %ld", sizeof(Group)); LOG(info, "sizeof(ResultNode::CP) = %ld", sizeof(ResultNode::CP)); LOG(info, "sizeof(RawRank) = %ld", sizeof(RawRank)); @@ -290,7 +272,4 @@ Test::Main() } LOG(info, "rusage = {\n%s\n}", vespalib::RUsage::createSelf(start).toString().c_str()); ASSERT_EQUAL(0, kill(0, SIGPROF)); - TEST_DONE(); } - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/groupingengine/groupingengine_test.cpp b/searchlib/src/tests/groupingengine/groupingengine_test.cpp index e00d2dcdd46..140304e0da9 100644 --- a/searchlib/src/tests/groupingengine/groupingengine_test.cpp +++ b/searchlib/src/tests/groupingengine/groupingengine_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/extendableattributes.h> @@ -131,69 +131,37 @@ AggregationContext::~AggregationContext() {} //----------------------------------------------------------------------------- -class Test : public TestApp +class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { public: - bool testAggregation(AggregationContext &ctx, - const Grouping &request, - const Group &expect); - bool testMerge(const Grouping &a, const Grouping &b, - const Group &expect); - bool testMerge(const Grouping &a, const Grouping &b, const Grouping &c, - const Group &expect); - bool testPrune(const Grouping &a, const Grouping &b, - const Group &expect); - bool testPartialMerge(const Grouping &a, const Grouping &b, - const Group &expect); - void testAggregationSimple(); - void testAggregationLevels(); - void testAggregationMaxGroups(); - void testAggregationGroupOrder(); - void testAggregationGroupRank(); - void testAggregationGroupCapping(); - void testMergeSimpleSum(); - void testMergeLevels(); - void testMergeGroups(); - void testMergeTrees(); - void testPruneSimple(); - void testPruneComplex(); - void testPartialMerging(); - void testCount(); - void testTopN(); - void testFS4HitCollection(); - bool checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket); - bool checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt); - void testFixedWidthBuckets(); - void testThatNanIsConverted(); - void testNanSorting(); - void testGroupingEngineFromRequest(); - int Main() override; + CheckAttributeReferences() : _numrefs(0) { } + int _numrefs; private: - bool verifyEqual(const Group & a, const Group & b); - void testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr); - class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate - { - public: - CheckAttributeReferences() : _numrefs(0) { } - int _numrefs; - private: - virtual void execute(vespalib::Identifiable &obj) override { - if (static_cast<AttributeNode &>(obj).getAttribute() != NULL) { - _numrefs++; - } + virtual void execute(vespalib::Identifiable &obj) override { + if (static_cast<AttributeNode &>(obj).getAttribute() != NULL) { + _numrefs++; } - virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } - }; + } + virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } }; //----------------------------------------------------------------------------- +bool verifyEqual(const Group & a, const Group & b) +{ + bool ok = EXPECT_EQUAL(a.asString(), b.asString()); + if (!ok) { + std::cerr << a.asString() << std::endl << b.asString() << std::endl; + } + return ok; +} + /** * Run the given grouping request and verify that the resulting group * tree matches the expected value. **/ bool -Test::testAggregation(AggregationContext &ctx, +testAggregation(AggregationContext &ctx, const Grouping &request, const Group &expect) { @@ -210,21 +178,12 @@ Test::testAggregation(AggregationContext &ctx, return verifyEqual(*result, expect); } -bool Test::verifyEqual(const Group & a, const Group & b) -{ - bool ok = EXPECT_EQUAL(a.asString(), b.asString()); - if (!ok) { - std::cerr << a.asString() << std::endl << b.asString() << std::endl; - } - return ok; -} - /** * Merge the given grouping requests and verify that the resulting * group tree matches the expected value. **/ bool -Test::testMerge(const Grouping &a, const Grouping &b, +testMerge(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy @@ -249,7 +208,7 @@ Test::testMerge(const Grouping &a, const Grouping &b, * group tree matches the expected value. **/ bool -Test::testPrune(const Grouping &a, const Grouping &b, +testPrune(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy @@ -266,7 +225,7 @@ Test::testPrune(const Grouping &a, const Grouping &b, * partial request is correct. **/ bool -Test::testPartialMerge(const Grouping &a, const Grouping &b, +testPartialMerge(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy @@ -283,7 +242,7 @@ Test::testPartialMerge(const Grouping &a, const Grouping &b, * group tree matches the expected value. **/ bool -Test::testMerge(const Grouping &a, const Grouping &b, const Grouping &c, +testMerge(const Grouping &a, const Grouping &b, const Grouping &c, const Group &expect) { Grouping tmp = a; // create local copy @@ -298,28 +257,9 @@ Test::testMerge(const Grouping &a, const Grouping &b, const Grouping &c, //----------------------------------------------------------------------------- -/** - * Test collecting the sum of the values from a single attribute - * vector directly into the root node. Consider this a smoke test. - **/ -void -Test::testAggregationSimple() -{ - AggregationContext ctx; - ctx.result().add(0).add(1).add(2); - ctx.add(IntAttrBuilder("int").add(3).add(7).add(15).sp()); - ctx.add(FloatAttrBuilder("float").add(3).add(7).add(15).sp()); - ctx.add(StringAttrBuilder("string").add("3").add("7").add("15").sp()); - - char strsum[3] = {(char)-101, '5', 0}; - testAggregationSimpleSum(ctx, SumAggregationResult(), Int64ResultNode(25), FloatResultNode(25), StringResultNode(strsum)); - testAggregationSimpleSum(ctx, MinAggregationResult(), Int64ResultNode(3), FloatResultNode(3), StringResultNode("15")); - testAggregationSimpleSum(ctx, MaxAggregationResult(), Int64ResultNode(15), FloatResultNode(15), StringResultNode("7")); -} - #define MU std::make_unique -void Test::testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr) +void testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr) { ExpressionNode::CP clone(aggr); Grouping request; @@ -337,6 +277,25 @@ void Test::testAggregationSimpleSum(AggregationContext & ctx, const AggregationR EXPECT_TRUE(testAggregation(ctx, request, expect)); } +/** + * Test collecting the sum of the values from a single attribute + * vector directly into the root node. Consider this a smoke test. + **/ +void +testAggregationSimple() +{ + AggregationContext ctx; + ctx.result().add(0).add(1).add(2); + ctx.add(IntAttrBuilder("int").add(3).add(7).add(15).sp()); + ctx.add(FloatAttrBuilder("float").add(3).add(7).add(15).sp()); + ctx.add(StringAttrBuilder("string").add("3").add("7").add("15").sp()); + + char strsum[3] = {(char)-101, '5', 0}; + testAggregationSimpleSum(ctx, SumAggregationResult(), Int64ResultNode(25), FloatResultNode(25), StringResultNode(strsum)); + testAggregationSimpleSum(ctx, MinAggregationResult(), Int64ResultNode(3), FloatResultNode(3), StringResultNode("15")); + testAggregationSimpleSum(ctx, MaxAggregationResult(), Int64ResultNode(15), FloatResultNode(15), StringResultNode("7")); +} + GroupingLevel createGL(ExpressionNode::UP expr, ExpressionNode::UP result) { GroupingLevel l; @@ -374,7 +333,7 @@ createGL(size_t maxGroups, ExpressionNode::UP expr, ExpressionNode::UP result) { * lastLevel parameters. **/ void -Test::testAggregationLevels() +testAggregationLevels() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr0").add(10).add(10).sp()); @@ -496,7 +455,7 @@ Test::testAggregationLevels() * indicated by the maxgroups parameter. **/ void -Test::testAggregationMaxGroups() +testAggregationMaxGroups() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr").add(5).add(10).add(15).sp()); @@ -547,7 +506,7 @@ Test::testAggregationMaxGroups() * Verify that groups are sorted by group id **/ void -Test::testAggregationGroupOrder() +testAggregationGroupOrder() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr").add(10).add(25).add(35).add(5).add(20).add(15).add(30).sp()); @@ -574,7 +533,7 @@ Test::testAggregationGroupOrder() * Verify that groups are tagged with the appropriate rank value. **/ void -Test::testAggregationGroupRank() +testAggregationGroupRank() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr") @@ -624,7 +583,7 @@ createNumAggr(NumericResultNode::UP r, ExpressionNode::UP e) { } void -Test::testAggregationGroupCapping() +testAggregationGroupCapping() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr") @@ -748,7 +707,7 @@ Test::testAggregationGroupCapping() * smoke test. **/ void -Test::testMergeSimpleSum() +testMergeSimpleSum() { Grouping a = Grouping() .setRoot(Group() @@ -777,7 +736,7 @@ Test::testMergeSimpleSum() * Verify that frozen levels are not touched during merge. **/ void -Test::testMergeLevels() +testMergeLevels() { Grouping request = Grouping() .addLevel(createGL(MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -957,7 +916,7 @@ Test::testMergeLevels() * and that they are sorted by group id. **/ void -Test::testMergeGroups() +testMergeGroups() { Grouping request = Grouping().addLevel(createGL(MU<AttributeNode>("attr"))); @@ -1018,7 +977,7 @@ Test::testMergeGroups() * end result is as expected. **/ void -Test::testMergeTrees() +testMergeTrees() { Grouping request; request.addLevel(createGL(3, MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -1283,7 +1242,7 @@ Test::testMergeTrees() } void -Test::testPruneComplex() +testPruneComplex() { { // First level Group baseTree = Group() @@ -1423,7 +1382,7 @@ Test::testPruneComplex() * results. **/ void -Test::testPartialMerging() +testPartialMerging() { Grouping baseRequest; baseRequest.addLevel(createGL(MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -1588,7 +1547,7 @@ Test::testPartialMerging() * Test that pruning a simple grouping tree works. **/ void -Test::testPruneSimple() +testPruneSimple() { { Grouping request; @@ -1614,7 +1573,7 @@ Test::testPruneSimple() * that we init, calculate and ignore. **/ void -Test::testTopN() +testTopN() { AggregationContext ctx; ctx.result().add(0).add(1).add(2); @@ -1655,7 +1614,7 @@ Test::testTopN() * that we init, calculate and ignore. **/ void -Test::testCount() +testCount() { AggregationContext ctx; ctx.result().add(0).add(1).add(2); @@ -1676,7 +1635,7 @@ Test::testCount() //----------------------------------------------------------------------------- bool -Test::checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt) +checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt) { CountFS4Hits pop; Grouping tmp = g; @@ -1685,7 +1644,7 @@ Test::checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt) } void -Test::testFS4HitCollection() +testFS4HitCollection() { { // aggregation AggregationContext ctx; @@ -1801,7 +1760,7 @@ Test::testFS4HitCollection() } bool -Test::checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket) +checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket) { AggregationContext ctx; ctx.result().add(0); @@ -1821,7 +1780,7 @@ Test::checkBucket(const NumericResultNode &width, const NumericResultNode &value } void -Test::testFixedWidthBuckets() +testFixedWidthBuckets() { using Int = Int64ResultNode; using Float = FloatResultNode; @@ -1879,7 +1838,7 @@ Test::testFixedWidthBuckets() void -Test::testNanSorting() +testNanSorting() { // Attempt at reproducing issue with segfault when setting NaN value. Not // successful yet, so no point in running test. @@ -1913,7 +1872,7 @@ Test::testNanSorting() } void -Test::testThatNanIsConverted() +testThatNanIsConverted() { Group g; double myNan = std::sqrt(-1); @@ -1923,7 +1882,7 @@ Test::testThatNanIsConverted() } void -Test::testGroupingEngineFromRequest() +testGroupingEngineFromRequest() { AggregationContext ctx; ctx.add(IntAttrBuilder("attr0").add(10).add(10).sp()); @@ -1943,19 +1902,8 @@ Test::testGroupingEngineFromRequest() //----------------------------------------------------------------------------- -struct RunDiff { ~RunDiff() { - [[maybe_unused]] int system_result = system("diff -u lhs.out rhs.out > diff.txt"); -}}; - -//----------------------------------------------------------------------------- - -int -Test::Main() -{ - RunDiff runDiff; - (void) runDiff; +TEST_MAIN() { TEST_DEBUG("lhs.out", "rhs.out"); - TEST_INIT("groupingengine_test"); testGroupingEngineFromRequest(); testAggregationSimple(); testAggregationLevels(); @@ -1978,7 +1926,6 @@ Test::Main() testTopN(); testThatNanIsConverted(); testNanSorting(); - TEST_DONE(); + TEST_DEBUG_STOP(); + [[maybe_unused]] int system_result = system("diff -u lhs.out rhs.out > diff.txt"); } - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/hitcollector/CMakeLists.txt b/searchlib/src/tests/hitcollector/CMakeLists.txt index 5cedbcbd7e6..ec5ae78a658 100644 --- a/searchlib/src/tests/hitcollector/CMakeLists.txt +++ b/searchlib/src/tests/hitcollector/CMakeLists.txt @@ -3,13 +3,15 @@ vespa_add_executable(searchlib_hitcollector_test_app TEST SOURCES hitcollector_test.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_hitcollector_test_app COMMAND searchlib_hitcollector_test_app) vespa_add_executable(searchlib_sorted_hit_sequence_test_app TEST SOURCES sorted_hit_sequence_test.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_sorted_hit_sequence_test_app COMMAND searchlib_sorted_hit_sequence_test_app) diff --git a/searchlib/src/tests/hitcollector/hitcollector_test.cpp b/searchlib/src/tests/hitcollector/hitcollector_test.cpp index e6e38181412..784afea7801 100644 --- a/searchlib/src/tests/hitcollector/hitcollector_test.cpp +++ b/searchlib/src/tests/hitcollector/hitcollector_test.cpp @@ -1,9 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/queryeval/hitcollector.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("hitcollector_test"); @@ -13,6 +13,8 @@ using namespace search::fef; using namespace search::queryeval; using ScoreMap = std::map<uint32_t, feature_t>; +using DocidVector = std::vector<uint32_t>; +using RankedHitVector = std::vector<RankedHit>; using Ranges = std::pair<Scores, Scores>; @@ -67,11 +69,11 @@ void checkResult(const ResultSet & rs, const std::vector<RankedHit> & exp) if ( ! exp.empty()) { const RankedHit * rh = rs.getArray(); ASSERT_TRUE(rh != nullptr); - ASSERT_EQUAL(rs.getArrayUsed(), exp.size()); + ASSERT_EQ(rs.getArrayUsed(), exp.size()); for (uint32_t i = 0; i < exp.size(); ++i) { - EXPECT_EQUAL(rh[i].getDocId(), exp[i].getDocId()); - EXPECT_EQUAL(rh[i].getRank() + 1.0, exp[i].getRank() + 1.0); + EXPECT_EQ(rh[i].getDocId(), exp[i].getDocId()); + EXPECT_DOUBLE_EQ(rh[i].getRank() + 64.0, exp[i].getRank() + 64.0); } } else { ASSERT_TRUE(rs.getArray() == nullptr); @@ -93,21 +95,24 @@ void checkResult(ResultSet & rs, BitVector * exp) } } -void testAddHit(uint32_t numDocs, uint32_t maxHitsSize) +void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, const vespalib::string& label) { + SCOPED_TRACE(label); LOG(info, "testAddHit: no hits"); - { // no hits + { + SCOPED_TRACE("no hits"); HitCollector hc(numDocs, maxHitsSize); std::vector<RankedHit> expRh; std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, nullptr)); + checkResult(*rs, expRh); + checkResult(*rs, nullptr); } LOG(info, "testAddHit: only ranked hits"); - { // only ranked hits + { + SCOPED_TRACE("only ranked hits"); HitCollector hc(numDocs, maxHitsSize); std::vector<RankedHit> expRh; @@ -121,12 +126,13 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize) } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, nullptr)); + checkResult(*rs, expRh); + checkResult(*rs, nullptr); } LOG(info, "testAddHit: both ranked hits and bit vector hits"); - { // both ranked hits and bit vector hits + { + SCOPED_TRACE("both ranked hits and bitvector hits"); HitCollector hc(numDocs, maxHitsSize); std::vector<RankedHit> expRh; BitVector::UP expBv(BitVector::create(numDocs)); @@ -144,14 +150,15 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize) } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, expBv.get())); + checkResult(*rs, expRh); + checkResult(*rs, expBv.get()); } } -TEST("testAddHit") { - TEST_DO(testAddHit(30, 10)); - TEST_DO(testAddHit(400, 10)); // 400/32 = 12 which is bigger than 10. +TEST(HitCollectorTest, testAddHit) +{ + testAddHit(30, 10, "numDocs==30"); + testAddHit(400, 10, "numDocs==400"); // 400/32 = 12 which is bigger than 10. } struct Fixture { @@ -197,14 +204,17 @@ struct DescendingScoreFixture : Fixture { DescendingScoreFixture::~DescendingScoreFixture() = default; -TEST_F("testReRank - empty", Fixture) { - EXPECT_EQUAL(0u, f.reRank()); +TEST(HitCollectorTest, rerank_empty) +{ + Fixture f; + EXPECT_EQ(0u, f.reRank()); } -TEST_F("testReRank - ascending", AscendingScoreFixture) +TEST(HitCollectorTest, rerank_ascending) { + AscendingScoreFixture f; f.addHits(); - EXPECT_EQUAL(5u, f.reRank()); + EXPECT_EQ(5u, f.reRank()); std::vector<RankedHit> expRh; for (uint32_t i = 10; i < 20; ++i) { // 10 last are the best @@ -213,17 +223,18 @@ TEST_F("testReRank - ascending", AscendingScoreFixture) expRh.back()._rankValue = i + 200; // after reranking } } - EXPECT_EQUAL(expRh.size(), 10u); + EXPECT_EQ(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, f.expBv.get())); + checkResult(*rs, expRh); + checkResult(*rs, f.expBv.get()); } -TEST_F("testReRank - descending", DescendingScoreFixture) +TEST(HitCollectorTest, rerank_descending) { + DescendingScoreFixture f; f.addHits(); - EXPECT_EQUAL(5u, f.reRank()); + EXPECT_EQ(5u, f.reRank()); std::vector<RankedHit> expRh; for (uint32_t i = 0; i < 10; ++i) { // 10 first are the best @@ -232,17 +243,18 @@ TEST_F("testReRank - descending", DescendingScoreFixture) expRh.back()._rankValue = i + 200; // after reranking } } - EXPECT_EQUAL(expRh.size(), 10u); + EXPECT_EQ(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, f.expBv.get())); + checkResult(*rs, expRh); + checkResult(*rs, f.expBv.get()); } -TEST_F("testReRank - partial", AscendingScoreFixture) +TEST(HitCollectorTest, rerank_partial) { + AscendingScoreFixture f; f.addHits(); - EXPECT_EQUAL(3u, f.reRank(3)); + EXPECT_EQ(3u, f.reRank(3)); std::vector<RankedHit> expRh; for (uint32_t i = 10; i < 20; ++i) { // 10 last are the best @@ -251,36 +263,39 @@ TEST_F("testReRank - partial", AscendingScoreFixture) expRh.back()._rankValue = i + 200; // after reranking } } - EXPECT_EQUAL(expRh.size(), 10u); + EXPECT_EQ(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, f.expBv.get())); + checkResult(*rs, expRh); + checkResult(*rs, f.expBv.get()); } -TEST_F("require that hits for 2nd phase candidates can be retrieved", DescendingScoreFixture) +TEST(HitCollectorTest, require_that_hits_for_2nd_phase_candidates_can_be_retrieved) { + DescendingScoreFixture f; f.addHits(); std::vector<HitCollector::Hit> scores = extract(f.hc.getSortedHitSequence(5)); - ASSERT_EQUAL(5u, scores.size()); - EXPECT_EQUAL(100, scores[0].second); - EXPECT_EQUAL(99, scores[1].second); - EXPECT_EQUAL(98, scores[2].second); - EXPECT_EQUAL(97, scores[3].second); - EXPECT_EQUAL(96, scores[4].second); + ASSERT_EQ(5u, scores.size()); + EXPECT_EQ(100, scores[0].second); + EXPECT_EQ(99, scores[1].second); + EXPECT_EQ(98, scores[2].second); + EXPECT_EQ(97, scores[3].second); + EXPECT_EQ(96, scores[4].second); } -TEST("require that score ranges can be read and set.") { +TEST(HitCollectorTest, require_that_score_ranges_can_be_read_and_set) +{ std::pair<Scores, Scores> ranges = std::make_pair(Scores(1.0, 2.0), Scores(3.0, 4.0)); HitCollector hc(20, 10); hc.setRanges(ranges); - EXPECT_EQUAL(ranges.first.low, hc.getRanges().first.low); - EXPECT_EQUAL(ranges.first.high, hc.getRanges().first.high); - EXPECT_EQUAL(ranges.second.low, hc.getRanges().second.low); - EXPECT_EQUAL(ranges.second.high, hc.getRanges().second.high); + EXPECT_EQ(ranges.first.low, hc.getRanges().first.low); + EXPECT_EQ(ranges.first.high, hc.getRanges().first.high); + EXPECT_EQ(ranges.second.low, hc.getRanges().second.low); + EXPECT_EQ(ranges.second.high, hc.getRanges().second.high); } -TEST("testNoHitsToReRank") { +TEST(HitCollectorTest, no_hits_to_rerank) +{ uint32_t numDocs = 20; uint32_t maxHitsSize = 10; @@ -299,8 +314,8 @@ TEST("testNoHitsToReRank") { } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, nullptr)); + checkResult(*rs, expRh); + checkResult(*rs, nullptr); } } @@ -317,14 +332,15 @@ void testScaling(const std::vector<feature_t> &initScores, PredefinedScorer scorer(std::move(finalScores)); // perform second phase ranking - EXPECT_EQUAL(2u, do_reRank(scorer, hc, 2)); + EXPECT_EQ(2u, do_reRank(scorer, hc, 2)); // check results std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs, expected)); + checkResult(*rs, expected); } -TEST("testScaling") { +TEST(HitCollectorTest, scaling) +{ std::vector<feature_t> initScores(5); initScores[0] = 1000; initScores[1] = 2000; @@ -338,7 +354,8 @@ TEST("testScaling") { exp[i]._docId = i; } - { // scale down and adjust down + { + SCOPED_TRACE("scale down and adjust down"); exp[0]._rankValue = 0; // scaled exp[1]._rankValue = 100; // scaled exp[2]._rankValue = 200; // scaled @@ -350,9 +367,10 @@ TEST("testScaling") { finalScores[3] = 300; finalScores[4] = 400; - TEST_DO(testScaling(initScores, std::move(finalScores), exp)); + testScaling(initScores, std::move(finalScores), exp); } - { // scale down and adjust up + { + SCOPED_TRACE("scale down and adjust up"); exp[0]._rankValue = 200; // scaled exp[1]._rankValue = 300; // scaled exp[2]._rankValue = 400; // scaled @@ -364,10 +382,10 @@ TEST("testScaling") { finalScores[3] = 500; finalScores[4] = 600; - TEST_DO(testScaling(initScores, std::move(finalScores), exp)); + testScaling(initScores, std::move(finalScores), exp); } - { // scale up and adjust down - + { + SCOPED_TRACE("scale up and adjust down"); exp[0]._rankValue = -500; // scaled (-500) exp[1]._rankValue = 750; // scaled exp[2]._rankValue = 2000; // scaled @@ -379,9 +397,10 @@ TEST("testScaling") { finalScores[3] = 3250; finalScores[4] = 4500; - TEST_DO(testScaling(initScores, std::move(finalScores), exp)); + testScaling(initScores, std::move(finalScores), exp); } - { // minimal scale (second phase range = 0 (4 - 4) -> 1) + { + SCOPED_TRACE("minimal scale (second phase range = 0 (4 - 4) -> 1)"); exp[0]._rankValue = 1; // scaled exp[1]._rankValue = 2; // scaled exp[2]._rankValue = 3; // scaled @@ -393,9 +412,10 @@ TEST("testScaling") { finalScores[3] = 4; finalScores[4] = 4; - TEST_DO(testScaling(initScores, std::move(finalScores), exp)); + testScaling(initScores, std::move(finalScores), exp); } - { // minimal scale (first phase range = 0 (4000 - 4000) -> 1) + { + SCOPED_TRACE("minimal scale (first phase range = 0 (4000 - 4000) -> 1)"); std::vector<feature_t> is(initScores); is[4] = 4000; exp[0]._rankValue = -299600; // scaled @@ -409,11 +429,12 @@ TEST("testScaling") { finalScores[3] = 400; finalScores[4] = 500; - TEST_DO(testScaling(is, std::move(finalScores), exp)); + testScaling(is, std::move(finalScores), exp); } } -TEST("testOnlyBitVector") { +TEST(HitCollectorTest, only_bitvector) +{ uint32_t numDocs = 20; LOG(info, "testOnlyBitVector: test it"); { @@ -428,8 +449,8 @@ TEST("testOnlyBitVector") { std::unique_ptr<ResultSet> rs = hc.getResultSet(); std::vector<RankedHit> expRh; - TEST_DO(checkResult(*rs, expRh)); // no ranked hits - TEST_DO(checkResult(*rs, expBv.get())); // only bit vector + checkResult(*rs, expRh); // no ranked hits + checkResult(*rs, expBv.get()); // only bit vector } } @@ -443,9 +464,9 @@ struct MergeResultSetFixture { {} }; -TEST_F("require that result set is merged correctly with first phase ranking", - MergeResultSetFixture) +TEST(HitCollectorTest, require_that_result_set_is_merged_correctly_with_first_phase_ranking) { + MergeResultSetFixture f; std::vector<RankedHit> expRh; for (uint32_t i = 0; i < f.numDocs; ++i) { f.hc.addHit(i, i + 1000); @@ -457,7 +478,7 @@ TEST_F("require that result set is merged correctly with first phase ranking", expRh.back()._rankValue = (i < f.numDocs - f.maxHitsSize) ? default_rank_value : i + 1000; } std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); + checkResult(*rs, expRh); } void @@ -474,9 +495,9 @@ addExpectedHitForMergeTest(const MergeResultSetFixture &f, std::vector<RankedHit } } -TEST_F("require that result set is merged correctly with second phase ranking (document scorer)", - MergeResultSetFixture) +TEST(HitCollectorTest, require_that_result_set_is_merged_correctly_with_second_phase_ranking_using_document_scorer) { + MergeResultSetFixture f; // with second phase ranking that triggers rescoring / scaling BasicScorer scorer(500); // second phase ranking setting score to docId + 500 std::vector<RankedHit> expRh; @@ -484,12 +505,13 @@ TEST_F("require that result set is merged correctly with second phase ranking (d f.hc.addHit(i, i + 1000); addExpectedHitForMergeTest(f, expRh, i); } - EXPECT_EQUAL(f.maxHeapSize, do_reRank(scorer, f.hc, f.maxHeapSize)); + EXPECT_EQ(f.maxHeapSize, do_reRank(scorer, f.hc, f.maxHeapSize)); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); + checkResult(*rs, expRh); } -TEST("require that hits can be added out of order") { +TEST(HitCollectorTest, require_that_hits_can_be_added_out_of_order) +{ HitCollector hc(1000, 100); std::vector<RankedHit> expRh; // produce expected result in normal order @@ -503,11 +525,12 @@ TEST("require that hits can be added out of order") { hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, nullptr)); + checkResult(*rs, expRh); + checkResult(*rs, nullptr); } -TEST("require that hits can be added out of order when passing array limit") { +TEST(HitCollectorTest, require_that_hits_can_be_added_out_of_order_when_passing_array_limit) +{ HitCollector hc(10000, 100); std::vector<RankedHit> expRh; // produce expected result in normal order @@ -525,11 +548,12 @@ TEST("require that hits can be added out of order when passing array limit") { hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, nullptr)); + checkResult(*rs, expRh); + checkResult(*rs, nullptr); } -TEST("require that hits can be added out of order only after passing array limit") { +TEST(HitCollectorTest, require_that_hits_can_be_added_out_of_order_only_after_passing_array_limit) +{ HitCollector hc(10000, 100); std::vector<RankedHit> expRh; // produce expected result in normal order @@ -548,8 +572,90 @@ TEST("require that hits can be added out of order only after passing array limit hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs, expRh)); - TEST_DO(checkResult(*rs, nullptr)); + checkResult(*rs, expRh); + checkResult(*rs, nullptr); +} + +struct RankDropFixture { + uint32_t _docid_limit; + HitCollector _hc; + std::vector<uint32_t> _dropped; + RankDropFixture(uint32_t docid_limit, uint32_t max_hits_size) + : _docid_limit(docid_limit), + _hc(docid_limit, max_hits_size) + { + } + ~RankDropFixture(); + void add(std::vector<RankedHit> hits) { + for (const auto& hit : hits) { + _hc.addHit(hit.getDocId(), hit.getRank()); + } + } + void rerank(ScoreMap score_map, size_t count) { + PredefinedScorer scorer(score_map); + EXPECT_EQ(count, do_reRank(scorer, _hc, count)); + } + std::unique_ptr<BitVector> make_bv(DocidVector docids) { + auto bv = BitVector::create(_docid_limit); + for (auto& docid : docids) { + bv->setBit(docid); + } + return bv; + } + + void setup() { + // Initial 7 hits from first phase + add({{5, 1100},{10, 1200},{11, 1300},{12, 1400},{14, 500},{15, 900},{16,1000}}); + // Rerank two best hits, calculate old and new ranges for reranked + // hits that will cause hits not reranked to later be rescored by + // dividing by 100. + rerank({{11,14},{12,13}}, 2); + } + void check_result(std::optional<double> rank_drop_limit, RankedHitVector exp_array, + std::unique_ptr<BitVector> exp_bv, DocidVector exp_dropped) { + auto rs = _hc.get_result_set(rank_drop_limit, &_dropped); + checkResult(*rs, exp_array); + checkResult(*rs, exp_bv.get()); + EXPECT_EQ(exp_dropped, _dropped); + } +}; + +RankDropFixture::~RankDropFixture() = default; + +TEST(HitCollectorTest, require_that_second_phase_rank_drop_limit_is_enforced) +{ + // Track rank score for all 7 hits from first phase + RankDropFixture f(10000, 10); + f.setup(); + f.check_result(9.0, {{5,11},{10,12},{11,14},{12,13},{16,10}}, + {}, {14, 15}); +} + +TEST(HitCollectorTest, require_that_second_phase_rank_drop_limit_is_enforced_when_docid_vector_is_used) +{ + // Track rank score for 4 best hits from first phase, overflow to docid vector + RankDropFixture f(10000, 4); + f.setup(); + f.check_result(13.0, {{11,14}}, + {}, {5,10,12,14,15,16}); +} + +TEST(HitCollectorTest, require_that_bitvector_is_not_dropped_without_second_phase_rank_drop_limit) +{ + // Track rank score for 4 best hits from first phase, overflow to bitvector + RankDropFixture f(20, 4); + f.setup(); + f.check_result(std::nullopt, {{5,11},{10,12},{11,14},{12,13}}, + f.make_bv({5,10,11,12,14,15,16}), {}); +} + +TEST(HitCollectorTest, require_that_bitvector_is_dropped_with_second_phase_rank_drop_limit) +{ + // Track rank for 4 best hits from first phase, overflow to bitvector + RankDropFixture f(20, 4); + f.setup(); + f.check_result(9.0, {{5,11},{10,12},{11,14},{12,13}}, + {}, {14,15,16}); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/hitcollector/sorted_hit_sequence_test.cpp b/searchlib/src/tests/hitcollector/sorted_hit_sequence_test.cpp index c1c3a550d9b..4eefa5b5dfa 100644 --- a/searchlib/src/tests/hitcollector/sorted_hit_sequence_test.cpp +++ b/searchlib/src/tests/hitcollector/sorted_hit_sequence_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/sorted_hit_sequence.h> +#include <vespa/vespalib/gtest/gtest.h> using search::queryeval::SortedHitSequence; using Hits = std::vector<SortedHitSequence::Hit>; @@ -10,20 +10,22 @@ using Refs = std::vector<SortedHitSequence::Ref>; Hits hits({{1,10.0},{2,30.0},{3,20.0}}); Refs refs({1,2,0}); -TEST("require that empty hit sequence is empty") { +TEST(SortedHitsSEquenceTest, require_that_empty_hit_sequence_is_empty) +{ EXPECT_TRUE(!SortedHitSequence(nullptr, nullptr, 0).valid()); EXPECT_TRUE(!SortedHitSequence(&hits[0], &refs[0], 0).valid()); } -TEST("require that sorted hit sequence can be iterated") { +TEST(SortedHitsSEquenceTest, require_that_sorted_hit_sequence_can_be_iterated) +{ SortedHitSequence seq(&hits[0], &refs[0], refs.size()); for (const auto &expect: Hits({{2,30.0},{3,20.0},{1,10.0}})) { ASSERT_TRUE(seq.valid()); - EXPECT_EQUAL(expect.first, seq.get().first); - EXPECT_EQUAL(expect.second, seq.get().second); + EXPECT_EQ(expect.first, seq.get().first); + EXPECT_EQ(expect.second, seq.get().second); seq.next(); } EXPECT_TRUE(!seq.valid()); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/index/field_length_calculator/CMakeLists.txt b/searchlib/src/tests/index/field_length_calculator/CMakeLists.txt index 4df31e04d28..a321a192e1c 100644 --- a/searchlib/src/tests/index/field_length_calculator/CMakeLists.txt +++ b/searchlib/src/tests/index/field_length_calculator/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_field_length_calculator_test_app TEST SOURCES field_length_calculator_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_field_length_calculator_test_app COMMAND searchlib_field_length_calculator_test_app) diff --git a/searchlib/src/tests/indexmetainfo/CMakeLists.txt b/searchlib/src/tests/indexmetainfo/CMakeLists.txt index 46d50106cfc..980f9dc8ae9 100644 --- a/searchlib/src/tests/indexmetainfo/CMakeLists.txt +++ b/searchlib/src/tests/indexmetainfo/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_indexmetainfo_test_app TEST SOURCES indexmetainfo_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_indexmetainfo_test_app COMMAND searchlib_indexmetainfo_test_app) diff --git a/searchlib/src/tests/indexmetainfo/indexmetainfo_test.cpp b/searchlib/src/tests/indexmetainfo/indexmetainfo_test.cpp index 78345b369aa..ed350216389 100644 --- a/searchlib/src/tests/indexmetainfo/indexmetainfo_test.cpp +++ b/searchlib/src/tests/indexmetainfo/indexmetainfo_test.cpp @@ -1,19 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/indexmetainfo.h> using search::IndexMetaInfo; using Snap = IndexMetaInfo::Snapshot; -TEST_SETUP(Test) - -int -Test::Main() -{ - TEST_INIT("indexmetainfo_test"); +TEST("indexmetainfo_test") { { // load pregenerated file IndexMetaInfo info(TEST_PATH("")); EXPECT_TRUE(info.load()); @@ -122,5 +116,6 @@ Test::Main() ASSERT_TRUE(b.snapshots().size() == 1); EXPECT_TRUE(b.snapshots()[0] == Snap(true, 50, "foo")); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt index e5ff2409cb4..2c5acb7dde3 100644 --- a/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_compact_words_store_test_app TEST SOURCES compact_words_store_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_compact_words_store_test_app COMMAND searchlib_compact_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt index 8e3764bbe6c..52d50278ad3 100644 --- a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_feature_store_test_app TEST SOURCES feature_store_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_feature_store_test_app COMMAND searchlib_feature_store_test_app) @@ -11,7 +11,7 @@ vespa_add_executable(searchlib_word_store_test_app TEST SOURCES word_store_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_word_store_test_app COMMAND searchlib_word_store_test_app) diff --git a/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt index 54ce308c53f..23fb77537cb 100644 --- a/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_document_inverter_test_app TEST document_inverter_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_document_inverter_test_app COMMAND searchlib_document_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/document_inverter_collection/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_inverter_collection/CMakeLists.txt index 79d62553e3f..2b3c6396a55 100644 --- a/searchlib/src/tests/memoryindex/document_inverter_collection/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/document_inverter_collection/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_document_inverter_collection_test_app TEST document_inverter_collection_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_document_inverter_collection_test_app COMMAND searchlib_document_inverter_collection_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt index 41a9a770370..bc541c0a46f 100644 --- a/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_field_index_test_app TEST SOURCES field_index_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) @@ -13,7 +13,7 @@ vespa_add_executable(searchlib_field_index_iterator_test_app TEST SOURCES field_index_iterator_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_field_index_iterator_test_app COMMAND searchlib_field_index_iterator_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt index 34f5aba4805..b7528944077 100644 --- a/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_field_index_remover_test_app TEST SOURCES field_index_remover_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_field_index_remover_test_app COMMAND searchlib_field_index_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt index d5604566025..6f659109020 100644 --- a/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_field_inverter_test_app TEST field_inverter_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_field_inverter_test_app COMMAND searchlib_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt index 9f49ae95bf2..0c60ba1b23f 100644 --- a/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_url_field_inverter_test_app TEST url_field_inverter_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_url_field_inverter_test_app COMMAND searchlib_url_field_inverter_test_app) diff --git a/searchlib/src/tests/nativerank/CMakeLists.txt b/searchlib/src/tests/nativerank/CMakeLists.txt index 2a46dd54904..6ef55f5fdc9 100644 --- a/searchlib/src/tests/nativerank/CMakeLists.txt +++ b/searchlib/src/tests/nativerank/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_nativerank_test_app TEST SOURCES nativerank_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test( diff --git a/searchlib/src/tests/nearsearch/CMakeLists.txt b/searchlib/src/tests/nearsearch/CMakeLists.txt index 4f249380063..95be9f628bf 100644 --- a/searchlib/src/tests/nearsearch/CMakeLists.txt +++ b/searchlib/src/tests/nearsearch/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(searchlib_nearsearch_test_app TEST SOURCES nearsearch_test.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_nearsearch_test_app COMMAND searchlib_nearsearch_test_app) diff --git a/searchlib/src/tests/nearsearch/nearsearch_test.cpp b/searchlib/src/tests/nearsearch/nearsearch_test.cpp index 4011366c7a1..aa578108b6b 100644 --- a/searchlib/src/tests/nearsearch/nearsearch_test.cpp +++ b/searchlib/src/tests/nearsearch/nearsearch_test.cpp @@ -11,7 +11,7 @@ LOG_SETUP("nearsearch_test"); #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/vespalib/util/stringfmt.h> #include <set> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> //////////////////////////////////////////////////////////////////////////////// // @@ -108,28 +108,20 @@ MyQuery::~MyQuery() {} // //////////////////////////////////////////////////////////////////////////////// -class Test : public vespalib::TestApp { -private: - bool testNearSearch(MyQuery &query, uint32_t matchId); +class NearSearchTest : public ::testing::Test { +protected: + void testNearSearch(MyQuery &query, uint32_t matchId, const vespalib::string& label); -public: - int Main() override; - void testBasicNear(); - void testRepeatedTerms(); + NearSearchTest(); + ~NearSearchTest() override; }; -int -Test::Main() +NearSearchTest::NearSearchTest() + : ::testing::Test() { - TEST_INIT("nearsearch_test"); - - testBasicNear(); TEST_FLUSH(); - testRepeatedTerms(); TEST_FLUSH(); - - TEST_DONE(); } -TEST_APPHOOK(Test); +NearSearchTest::~NearSearchTest() = default; //////////////////////////////////////////////////////////////////////////////// // @@ -137,84 +129,89 @@ TEST_APPHOOK(Test); // //////////////////////////////////////////////////////////////////////////////// -void -Test::testBasicNear() +TEST_F(NearSearchTest, basic_near) { MyTerm foo(UIntList().add(69), UIntList().add(6).add(11)); for (uint32_t i = 0; i <= 1; ++i) { - TEST_STATE(vespalib::make_string("i = %u", i).c_str()); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo), 69)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo), 69)); + SCOPED_TRACE(vespalib::make_string("i = %u", i)); + testNearSearch(MyQuery(false, i).addTerm(foo), 69, "near 1"); + testNearSearch(MyQuery(true, i).addTerm(foo), 69, "onear 1"); } MyTerm bar(UIntList().add(68).add(69).add(70), UIntList().add(7).add(10)); - TEST_DO(testNearSearch(MyQuery(false, 0).addTerm(foo).addTerm(bar), 0)); - TEST_DO(testNearSearch(MyQuery(true, 0).addTerm(foo).addTerm(bar), 0)); + testNearSearch(MyQuery(false, 0).addTerm(foo).addTerm(bar), 0, "near 2"); + testNearSearch(MyQuery(true, 0).addTerm(foo).addTerm(bar), 0, "onear 2"); for (uint32_t i = 1; i <= 2; ++i) { - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(bar), 69)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(bar), 69)); + SCOPED_TRACE(vespalib::make_string("i = %u", i)); + testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(bar), 69, "near 3"); + testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(bar), 69, "onear 3"); } MyTerm baz(UIntList().add(69).add(70).add(71), UIntList().add(8).add(9)); for (uint32_t i = 0; i <= 1; ++i) { - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(bar).addTerm(baz), 0)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(baz).addTerm(bar), 0)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(bar).addTerm(baz).addTerm(foo), 0)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(bar).addTerm(foo).addTerm(baz), 0)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(baz).addTerm(foo).addTerm(bar), 0)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(baz).addTerm(bar).addTerm(foo), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(bar).addTerm(baz), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(baz).addTerm(bar), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(bar).addTerm(baz).addTerm(foo), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(bar).addTerm(foo).addTerm(baz), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(baz).addTerm(foo).addTerm(bar), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(baz).addTerm(bar).addTerm(foo), 0)); + SCOPED_TRACE(vespalib::make_string("i = %u", i)); + testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(bar).addTerm(baz), 0, "near 10"); + testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(baz).addTerm(bar), 0, "near 11"); + testNearSearch(MyQuery(false, i).addTerm(bar).addTerm(baz).addTerm(foo), 0, "near 12"); + testNearSearch(MyQuery(false, i).addTerm(bar).addTerm(foo).addTerm(baz), 0, "near 13"); + testNearSearch(MyQuery(false, i).addTerm(baz).addTerm(foo).addTerm(bar), 0, "near 14"); + testNearSearch(MyQuery(false, i).addTerm(baz).addTerm(bar).addTerm(foo), 0, "near 15"); + testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(bar).addTerm(baz), 0, "onear 10"); + testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(baz).addTerm(bar), 0, "onear 11"); + testNearSearch(MyQuery(true, i).addTerm(bar).addTerm(baz).addTerm(foo), 0, "onear 12"); + testNearSearch(MyQuery(true, i).addTerm(bar).addTerm(foo).addTerm(baz), 0, "onear 13"); + testNearSearch(MyQuery(true, i).addTerm(baz).addTerm(foo).addTerm(bar), 0, "onear 14"); + testNearSearch(MyQuery(true, i).addTerm(baz).addTerm(bar).addTerm(foo), 0, "onear 15"); } for (uint32_t i = 2; i <= 3; ++i) { - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(bar).addTerm(baz), 69)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(baz).addTerm(bar), 69)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(bar).addTerm(baz).addTerm(foo), 69)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(bar).addTerm(foo).addTerm(baz), 69)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(baz).addTerm(foo).addTerm(bar), 69)); - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(baz).addTerm(bar).addTerm(foo), 69)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(bar).addTerm(baz), 69)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(baz).addTerm(bar), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(bar).addTerm(baz).addTerm(foo), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(bar).addTerm(foo).addTerm(baz), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(baz).addTerm(foo).addTerm(bar), 0)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(baz).addTerm(bar).addTerm(foo), 69)); + SCOPED_TRACE(vespalib::make_string("i = %u", i)); + testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(bar).addTerm(baz), 69, "near 20"); + testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(baz).addTerm(bar), 69, "near 21"); + testNearSearch(MyQuery(false, i).addTerm(bar).addTerm(baz).addTerm(foo), 69, "near 22"); + testNearSearch(MyQuery(false, i).addTerm(bar).addTerm(foo).addTerm(baz), 69, "near 23"); + testNearSearch(MyQuery(false, i).addTerm(baz).addTerm(foo).addTerm(bar), 69, "near 24"); + testNearSearch(MyQuery(false, i).addTerm(baz).addTerm(bar).addTerm(foo), 69, "near 25"); + testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(bar).addTerm(baz), 69, "onear 20"); + testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(baz).addTerm(bar), 0, "onear 21"); + testNearSearch(MyQuery(true, i).addTerm(bar).addTerm(baz).addTerm(foo), 0, "onear 22"); + testNearSearch(MyQuery(true, i).addTerm(bar).addTerm(foo).addTerm(baz), 0, "onear 23"); + testNearSearch(MyQuery(true, i).addTerm(baz).addTerm(foo).addTerm(bar), 0, "onear 24"); + testNearSearch(MyQuery(true, i).addTerm(baz).addTerm(bar).addTerm(foo), 69, "onear 25"); } } -void -Test::testRepeatedTerms() + +TEST_F(NearSearchTest, repeated_terms) { MyTerm foo(UIntList().add(69), UIntList().add(1).add(2).add(3)); - TEST_DO(testNearSearch(MyQuery(false, 0).addTerm(foo).addTerm(foo), 69)); - TEST_DO(testNearSearch(MyQuery(true, 0).addTerm(foo).addTerm(foo), 0)); + testNearSearch(MyQuery(false, 0).addTerm(foo).addTerm(foo), 69, "near 50"); + testNearSearch(MyQuery(true, 0).addTerm(foo).addTerm(foo), 0, "onear 50"); for (uint32_t i = 1; i <= 2; ++i) { - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(foo), 69)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(foo), 69)); + SCOPED_TRACE(vespalib::make_string("i = %u", i)); + testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(foo), 69, "near 51"); + testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(foo), 69, "onear 51"); } for (uint32_t i = 0; i <= 1; ++i) { - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(foo).addTerm(foo), 69)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(foo).addTerm(foo), 0)); + SCOPED_TRACE(vespalib::make_string("i = %u", i)); + testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(foo).addTerm(foo), 69, "near 52"); + testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(foo).addTerm(foo), 0, "onear 52"); } for (uint32_t i = 2; i <= 3; ++i) { - TEST_DO(testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(foo).addTerm(foo), 69)); - TEST_DO(testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(foo).addTerm(foo), 69)); + SCOPED_TRACE(vespalib::make_string("i = %u", i)); + testNearSearch(MyQuery(false, i).addTerm(foo).addTerm(foo).addTerm(foo), 69, "near 53"); + testNearSearch(MyQuery(true, i).addTerm(foo).addTerm(foo).addTerm(foo), 69, "onear 53"); } } -bool -Test::testNearSearch(MyQuery &query, uint32_t matchId) +void +NearSearchTest::testNearSearch(MyQuery &query, uint32_t matchId, const vespalib::string& label) { - LOG(info, "testNearSearch(%d)", matchId); + SCOPED_TRACE(vespalib::make_string("%s - %u", label.c_str(), matchId)); search::queryeval::IntermediateBlueprint *near_b = nullptr; if (query.isOrdered()) { near_b = new search::queryeval::ONearBlueprint(query.getWindow()); @@ -240,13 +237,14 @@ Test::testNearSearch(MyQuery &query, uint32_t matchId) if (docId == matchId) { foundMatch = true; } else { - LOG(info, "Document %d matched unexpectedly.", docId); - return false; + FAIL() << "Document " << docId << " matched unexpectedly."; } } if (matchId == 0) { - return EXPECT_TRUE(!foundMatch); + EXPECT_TRUE(!foundMatch); } else { - return EXPECT_TRUE(foundMatch); + EXPECT_TRUE(foundMatch); } } + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/postinglistbm/CMakeLists.txt b/searchlib/src/tests/postinglistbm/CMakeLists.txt index 27fe52386ed..92ea96b73b6 100644 --- a/searchlib/src/tests/postinglistbm/CMakeLists.txt +++ b/searchlib/src/tests/postinglistbm/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_posting_list_test_app TEST posting_list_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_posting_list_test_app NO_VALGRIND COMMAND searchlib_posting_list_test_app) @@ -15,5 +15,5 @@ vespa_add_executable(searchlib_postinglistbm_app TEST stress_runner.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib ) diff --git a/searchlib/src/tests/predicate/.gitignore b/searchlib/src/tests/predicate/.gitignore index eea4d347d05..56e1330505c 100644 --- a/searchlib/src/tests/predicate/.gitignore +++ b/searchlib/src/tests/predicate/.gitignore @@ -1,13 +1 @@ -searchlib_document_features_store_test_app -searchlib_predicate_bounds_posting_list_test_app -searchlib_predicate_index_test_app -searchlib_predicate_interval_posting_list_test_app -searchlib_predicate_interval_store_test_app -searchlib_predicate_range_term_expander_test_app -searchlib_predicate_ref_cache_test_app -searchlib_predicate_tree_analyzer_test_app -searchlib_predicate_tree_annotator_test_app -searchlib_predicate_zero_constraint_posting_list_test_app -searchlib_predicate_zstar_compressed_posting_list_test_app -searchlib_simple_index_test_app -searchlib_tree_crumbs_test_app +searchlib_predicate_vespa_test_app diff --git a/searchlib/src/tests/predicate/CMakeLists.txt b/searchlib/src/tests/predicate/CMakeLists.txt index b4d385a32f6..4ae06af1886 100644 --- a/searchlib/src/tests/predicate/CMakeLists.txt +++ b/searchlib/src/tests/predicate/CMakeLists.txt @@ -1,92 +1,21 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_predicate_index_test_app TEST +vespa_add_executable(searchlib_predicate_vespa_test_app TEST SOURCES + vespa_testrunner.cpp predicate_index_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_index_test_app COMMAND searchlib_predicate_index_test_app) -vespa_add_executable(searchlib_simple_index_test_app TEST - SOURCES simple_index_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_simple_index_test_app COMMAND searchlib_simple_index_test_app) -vespa_add_executable(searchlib_tree_crumbs_test_app TEST - SOURCES tree_crumbs_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_tree_crumbs_test_app COMMAND searchlib_tree_crumbs_test_app) -vespa_add_executable(searchlib_predicate_tree_analyzer_test_app TEST - SOURCES predicate_tree_analyzer_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_tree_analyzer_test_app COMMAND searchlib_predicate_tree_analyzer_test_app) -vespa_add_executable(searchlib_predicate_tree_annotator_test_app TEST - SOURCES predicate_tree_annotator_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_tree_annotator_test_app COMMAND searchlib_predicate_tree_annotator_test_app) -vespa_add_executable(searchlib_predicate_interval_store_test_app TEST - SOURCES predicate_interval_store_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_interval_store_test_app COMMAND searchlib_predicate_interval_store_test_app) -vespa_add_executable(searchlib_document_features_store_test_app TEST - SOURCES document_features_store_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_document_features_store_test_app COMMAND searchlib_document_features_store_test_app) -vespa_add_executable(searchlib_predicate_ref_cache_test_app TEST - SOURCES predicate_ref_cache_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_ref_cache_test_app COMMAND searchlib_predicate_ref_cache_test_app) -vespa_add_executable(searchlib_predicate_interval_posting_list_test_app TEST - SOURCES predicate_interval_posting_list_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_interval_posting_list_test_app COMMAND searchlib_predicate_interval_posting_list_test_app) -vespa_add_executable(searchlib_predicate_bounds_posting_list_test_app TEST - SOURCES predicate_bounds_posting_list_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_bounds_posting_list_test_app COMMAND searchlib_predicate_bounds_posting_list_test_app) -vespa_add_executable(searchlib_predicate_zero_constraint_posting_list_test_app TEST - SOURCES predicate_zero_constraint_posting_list_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_zero_constraint_posting_list_test_app COMMAND searchlib_predicate_zero_constraint_posting_list_test_app) -vespa_add_executable(searchlib_predicate_zstar_compressed_posting_list_test_app TEST - SOURCES predicate_zstar_compressed_posting_list_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_predicate_zstar_compressed_posting_list_test_app COMMAND searchlib_predicate_zstar_compressed_posting_list_test_app) -vespa_add_executable(searchlib_predicate_range_term_expander_test_app TEST - SOURCES predicate_range_term_expander_test.cpp DEPENDS - searchlib + vespa_searchlib ) -vespa_add_test(NAME searchlib_predicate_range_term_expander_test_app COMMAND searchlib_predicate_range_term_expander_test_app) +vespa_add_test(NAME searchlib_predicate_vespa_test_app COMMAND searchlib_predicate_vespa_test_app) diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp index 3d479a6d39a..01eaa75a71a 100644 --- a/searchlib/src/tests/predicate/document_features_store_test.cpp +++ b/searchlib/src/tests/predicate/document_features_store_test.cpp @@ -7,11 +7,7 @@ #include <vespa/searchlib/predicate/predicate_index.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/predicate/predicate_hash.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <string> - -#include <vespa/log/log.h> -LOG_SETUP("document_features_store_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -233,5 +229,3 @@ TEST("require that serialization cleans up wordstore") { } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp index 228b0eb242d..be7f4516bb2 100644 --- a/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp @@ -7,10 +7,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("predicate_bounds_posting_list_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -106,5 +103,3 @@ TEST("require that bounds posting list checks bounds.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_index_test.cpp b/searchlib/src/tests/predicate/predicate_index_test.cpp index 461fa46d4da..d0af12f93c7 100644 --- a/searchlib/src/tests/predicate/predicate_index_test.cpp +++ b/searchlib/src/tests/predicate/predicate_index_test.cpp @@ -5,16 +5,13 @@ #include <vespa/searchlib/predicate/simple_index.hpp> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/util/data_buffer_writer.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/log/log.h> -LOG_SETUP("predicate_index_test"); - using namespace search; using namespace search::predicate; using std::make_pair; @@ -454,5 +451,3 @@ TEST("require that predicate index saver protected by a generation guard observe } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp index ab49e28bb96..098c0238412 100644 --- a/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp @@ -7,10 +7,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("predicate_interval_posting_list_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -79,5 +76,3 @@ TEST("require that posting list can iterate.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp index 819563f64b8..d8c9691d421 100644 --- a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp +++ b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp @@ -1,13 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicate_interval_store. -#include <vespa/log/log.h> -LOG_SETUP("predicate_interval_store_test"); - #include <vespa/searchlib/predicate/predicate_interval_store.h> #include <vespa/searchlib/predicate/predicate_index.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vector> using namespace search; @@ -147,5 +144,3 @@ TEST("require that interval refs are reused for identical data.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp b/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp index 162829be5a3..2dce3c50b0a 100644 --- a/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp +++ b/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp @@ -1,12 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicate_range_term_expander. -#include <vespa/log/log.h> -LOG_SETUP("predicate_range_term_expander_test"); - #include <vespa/searchlib/predicate/predicate_range_term_expander.h> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using search::predicate::PredicateRangeTermExpander; using std::vector; @@ -328,5 +325,3 @@ TEST("require that search close to max uneven upper bound is sensible") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp b/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp index c8327033a8c..f62f3f807c5 100644 --- a/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp +++ b/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp @@ -1,11 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicate_ref_cache. -#include <vespa/log/log.h> -LOG_SETUP("predicate_ref_cache_test"); - #include <vespa/searchlib/predicate/predicate_ref_cache.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vector> using namespace search; @@ -101,5 +98,3 @@ TEST("require that cache handles large entries") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp b/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp index c766aa70bad..73a236aa443 100644 --- a/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp +++ b/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp @@ -1,13 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for PredicateTreeAnalyzer. -#include <vespa/log/log.h> -LOG_SETUP("PredicateTreeAnalyzer_test"); - #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_slime_builder.h> #include <vespa/searchlib/predicate/predicate_tree_analyzer.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::PredicateSlimeBuilder; using namespace search; @@ -152,5 +149,3 @@ TEST("require that multilevel AND stores sizes") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp b/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp index 82629527b4d..4d71f585910 100644 --- a/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp +++ b/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp @@ -7,12 +7,9 @@ #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/predicate/predicate_hash.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <sstream> -#include <vespa/log/log.h> -LOG_SETUP("PredicateTreeAnnotator_test"); - using document::Predicate; using std::ostringstream; using std::pair; @@ -403,5 +400,3 @@ TEST("require that open range works") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp index 9e9ac45f1ae..5907dce72ba 100644 --- a/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp @@ -4,10 +4,7 @@ #include <vespa/searchlib/predicate/predicate_zero_constraint_posting_list.h> #include <vespa/searchlib/predicate/predicate_index.h> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("predicate_zero_constraint_posting_list_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -54,5 +51,3 @@ TEST("require that posting list can iterate.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp index 0e99379568d..20cd0809473 100644 --- a/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp @@ -6,10 +6,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("predicate_zstar_compressed_posting_list_test"); +#include <vespa/vespalib/testkit/test_kit.h> using namespace search; using namespace search::predicate; @@ -93,5 +90,3 @@ TEST("require that posting list can iterate.") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/simple_index_test.cpp b/searchlib/src/tests/predicate/simple_index_test.cpp index 9b99ff8e809..964ff67bd3a 100644 --- a/searchlib/src/tests/predicate/simple_index_test.cpp +++ b/searchlib/src/tests/predicate/simple_index_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/predicate/simple_index_saver.hpp> #include <vespa/searchlib/predicate/nbo_write.h> #include <vespa/searchlib/util/data_buffer_writer.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreeroot.hpp> @@ -16,9 +16,6 @@ #include <vespa/vespalib/util/rcuvector.hpp> #include <map> -#include <vespa/log/log.h> -LOG_SETUP("simple_index_test"); - using namespace search; using namespace search::predicate; using vespalib::GenerationHolder; @@ -342,5 +339,3 @@ TEST_F("require that vector contains correct postings", Fixture) { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/tree_crumbs_test.cpp b/searchlib/src/tests/predicate/tree_crumbs_test.cpp index f5ff488fdc0..76bfd02ee50 100644 --- a/searchlib/src/tests/predicate/tree_crumbs_test.cpp +++ b/searchlib/src/tests/predicate/tree_crumbs_test.cpp @@ -1,11 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for TreeCrumbs. -#include <vespa/log/log.h> -LOG_SETUP("TreeCrumbs_test"); - #include <vespa/searchlib/predicate/tree_crumbs.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace search::predicate; @@ -60,5 +57,3 @@ TEST("require that crumbs can set custom initial char") { } } // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/predicate/vespa_testrunner.cpp b/searchlib/src/tests/predicate/vespa_testrunner.cpp new file mode 100644 index 00000000000..d812605710e --- /dev/null +++ b/searchlib/src/tests/predicate/vespa_testrunner.cpp @@ -0,0 +1,8 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Unit tests for predicate_index. +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/log/log.h> +LOG_SETUP("predicate_test"); + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/query/CMakeLists.txt b/searchlib/src/tests/query/CMakeLists.txt index 0cb6c9413b0..c1bfc5a8da2 100644 --- a/searchlib/src/tests/query/CMakeLists.txt +++ b/searchlib/src/tests/query/CMakeLists.txt @@ -3,42 +3,43 @@ vespa_add_executable(searchlib_query_visitor_test_app TEST SOURCES query_visitor_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_query_visitor_test_app COMMAND searchlib_query_visitor_test_app) vespa_add_executable(searchlib_customtypevisitor_test_app TEST SOURCES customtypevisitor_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_customtypevisitor_test_app COMMAND searchlib_customtypevisitor_test_app) vespa_add_executable(searchlib_templatetermvisitor_test_app TEST SOURCES templatetermvisitor_test.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_templatetermvisitor_test_app COMMAND searchlib_templatetermvisitor_test_app) vespa_add_executable(searchlib_querybuilder_test_app TEST SOURCES querybuilder_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_querybuilder_test_app COMMAND searchlib_querybuilder_test_app) vespa_add_executable(searchlib_stackdumpquerycreator_test_app TEST SOURCES stackdumpquerycreator_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_stackdumpquerycreator_test_app COMMAND searchlib_stackdumpquerycreator_test_app) vespa_add_executable(searchlib_streaming_query_test_app TEST SOURCES streaming_query_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_streaming_query_test_app COMMAND searchlib_streaming_query_test_app) @@ -46,6 +47,6 @@ vespa_add_executable(searchlib_streaming_query_large_test_app TEST SOURCES streaming_query_large_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_streaming_query_large_test_app COMMAND searchlib_streaming_query_large_test_app) diff --git a/searchlib/src/tests/query/customtypevisitor_test.cpp b/searchlib/src/tests/query/customtypevisitor_test.cpp index fb89f2ef061..702bed6e50e 100644 --- a/searchlib/src/tests/query/customtypevisitor_test.cpp +++ b/searchlib/src/tests/query/customtypevisitor_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/string_term_vector.h> #include <vespa/searchlib/query/tree/termnodes.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("customtypevisitor_test"); diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index bfce382b684..e1265680f7b 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/tree/string_term_vector.h> #include <vespa/searchlib/query/tree/termnodes.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("query_visitor_test"); diff --git a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp index 29ef179385d..4ae94c17804 100644 --- a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp +++ b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/util/rawbuf.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("stackdumpquerycreator_test"); diff --git a/searchlib/src/tests/query/streaming/CMakeLists.txt b/searchlib/src/tests/query/streaming/CMakeLists.txt index 5ed450ecbc8..c0c89a5cc17 100644 --- a/searchlib/src/tests/query/streaming/CMakeLists.txt +++ b/searchlib/src/tests/query/streaming/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_query_streaming_equiv_query_node_test_app TEST SOURCES equiv_query_node_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_query_streaming_equiv_query_node_test_app COMMAND searchlib_query_streaming_equiv_query_node_test_app) @@ -13,7 +13,7 @@ vespa_add_executable(searchlib_query_streaming_hit_iterator_test_app TEST SOURCES hit_iterator_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_query_streaming_hit_iterator_test_app COMMAND searchlib_query_streaming_hit_iterator_test_app) @@ -22,7 +22,7 @@ vespa_add_executable(searchlib_query_streaming_hit_iterator_pack_test_app TEST SOURCES hit_iterator_pack_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_query_streaming_hit_iterator_pack_test_app COMMAND searchlib_query_streaming_hit_iterator_pack_test_app) @@ -31,7 +31,7 @@ vespa_add_executable(searchlib_query_streaming_near_test_app TEST SOURCES near_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_query_streaming_near_test_app COMMAND searchlib_query_streaming_near_test_app) @@ -40,7 +40,7 @@ vespa_add_executable(searchlib_query_streaming_same_element_query_node_test_app SOURCES same_element_query_node_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_query_streaming_same_element_query_node_test_app COMMAND searchlib_query_streaming_same_element_query_node_test_app) @@ -49,7 +49,7 @@ vespa_add_executable(searchlib_query_streaming_phrase_query_node_test_app TEST SOURCES phrase_query_node_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::gtest ) vespa_add_test(NAME searchlib_query_streaming_phrase_query_node_test_app COMMAND searchlib_query_streaming_phrase_query_node_test_app) diff --git a/searchlib/src/tests/query/templatetermvisitor_test.cpp b/searchlib/src/tests/query/templatetermvisitor_test.cpp index b6dd6ceab8d..591aaffcbee 100644 --- a/searchlib/src/tests/query/templatetermvisitor_test.cpp +++ b/searchlib/src/tests/query/templatetermvisitor_test.cpp @@ -1,14 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for templatetermvisitor. -#include <vespa/log/log.h> -LOG_SETUP("templatetermvisitor_test"); - #include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/templatetermvisitor.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/tree/termnodes.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace search::query; @@ -16,23 +13,6 @@ namespace { class MyVisitor; -class Test : public vespalib::TestApp { - void requireThatAllTermsCanBeVisited(); - -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("templatetermvisitor_test"); - - TEST_DO(requireThatAllTermsCanBeVisited()); - - TEST_DONE(); -} - class MyVisitor : public TemplateTermVisitor<MyVisitor, SimpleQueryNodeTypes> { public: @@ -60,7 +40,8 @@ bool checkVisit() { return checkVisit(new T(typename T::Type(), "field", 0, Weight(0))); } -void Test::requireThatAllTermsCanBeVisited() { +TEST(TemplateTermVisitorTest, require_that_all_terms_can_be_visited) +{ EXPECT_TRUE(checkVisit<SimpleNumberTerm>()); EXPECT_TRUE(checkVisit<SimpleLocationTerm>()); EXPECT_TRUE(checkVisit<SimplePrefixTerm>()); @@ -83,4 +64,4 @@ void Test::requireThatAllTermsCanBeVisited() { } // namespace -TEST_APPHOOK(Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/queryeval/CMakeLists.txt b/searchlib/src/tests/queryeval/CMakeLists.txt index 55207e5705d..807c665d0e3 100644 --- a/searchlib/src/tests/queryeval/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_queryeval_test_app TEST SOURCES queryeval_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_queryeval_test_app COMMAND searchlib_queryeval_test_app) diff --git a/searchlib/src/tests/queryeval/blueprint/CMakeLists.txt b/searchlib/src/tests/queryeval/blueprint/CMakeLists.txt index e46ad1085e3..4bb0bf29754 100644 --- a/searchlib/src/tests/queryeval/blueprint/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/blueprint/CMakeLists.txt @@ -3,21 +3,22 @@ vespa_add_executable(searchlib_blueprint_test_app TEST SOURCES blueprint_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_blueprint_test_app COMMAND searchlib_blueprint_test_app || diff -u lhs.out rhs.out) vespa_add_executable(searchlib_leaf_blueprints_test_app TEST SOURCES leaf_blueprints_test.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_leaf_blueprints_test_app COMMAND searchlib_leaf_blueprints_test_app || diff -u lhs.out rhs.out) vespa_add_executable(searchlib_intermediate_blueprints_test_app TEST SOURCES intermediate_blueprints_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_intermediate_blueprints_test_app COMMAND searchlib_intermediate_blueprints_test_app || diff -u lhs.out rhs.out) diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 485410e0eba..77d0099afdb 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mysearch.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/flow.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> @@ -13,7 +13,7 @@ LOG_SETUP("blueprint_test"); using namespace search::queryeval; -using namespace search::fef; +using MatchData = search::fef::MatchData; namespace { @@ -44,9 +44,7 @@ public: } SearchIterator::UP - createIntermediateSearch(MultiSearch::Children subSearches, - MatchData &md) const override - { + createIntermediateSearch(MultiSearch::Children subSearches, MatchData &md) const override { return std::make_unique<MySearch>("or", std::move(subSearches), &md, strict()); } SearchIteratorUP createFilterSearch(FilterConstraint constraint) const override { @@ -63,9 +61,7 @@ class OtherOr : public OrBlueprint private: public: SearchIterator::UP - createIntermediateSearch(MultiSearch::Children subSearches, - MatchData &md) const override - { + createIntermediateSearch(MultiSearch::Children subSearches, MatchData &md) const override { return std::make_unique<MySearch>("or", std::move(subSearches), &md, strict()); } @@ -89,9 +85,7 @@ public: } SearchIterator::UP - createIntermediateSearch(MultiSearch::Children subSearches, - MatchData &md) const override - { + createIntermediateSearch(MultiSearch::Children subSearches, MatchData &md) const override { return std::make_unique<MySearch>("and", std::move(subSearches), &md, strict()); } @@ -106,9 +100,7 @@ class OtherAnd : public AndBlueprint private: public: SearchIterator::UP - createIntermediateSearch(MultiSearch::Children subSearches, - MatchData &md) const override - { + createIntermediateSearch(MultiSearch::Children subSearches, MatchData &md) const override { return std::make_unique<MySearch>("and", std::move(subSearches), &md, strict()); } @@ -121,9 +113,7 @@ class OtherAndNot : public AndNotBlueprint { public: SearchIterator::UP - createIntermediateSearch(MultiSearch::Children subSearches, - MatchData &md) const override - { + createIntermediateSearch(MultiSearch::Children subSearches, MatchData &md) const override { return std::make_unique<MySearch>("andnot", std::move(subSearches), &md, strict()); } @@ -658,6 +648,7 @@ getExpectedBlueprint() " strict_cost: 0\n" " sourceId: 4294967295\n" " docid_limit: 0\n" + " id: 0\n" " strict: false\n" " children: std::vector {\n" " [0]: (anonymous namespace)::MyTerm {\n" @@ -681,6 +672,7 @@ getExpectedBlueprint() " strict_cost: 0\n" " sourceId: 4294967295\n" " docid_limit: 0\n" + " id: 0\n" " strict: false\n" " }\n" " }\n" @@ -714,6 +706,7 @@ getExpectedSlimeBlueprint() { " strict_cost: 0.0," " sourceId: 4294967295," " docid_limit: 0," + " id: 0," " strict: false," " children: {" " '[type]': 'std::vector'," @@ -742,6 +735,7 @@ getExpectedSlimeBlueprint() { " strict_cost: 0.0," " sourceId: 4294967295," " docid_limit: 0," + " id: 0," " strict: false" " }" " }" @@ -852,6 +846,30 @@ TEST("self strict resolving during sort") { } } +void check_ids(Blueprint &bp, const std::vector<uint32_t> &expect) { + std::vector<uint32_t> actual; + bp.each_node_post_order([&](auto &node){ actual.push_back(node.id()); }); + ASSERT_EQUAL(actual.size(), expect.size()); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQUAL(actual[i], expect[i]); + } +} + +TEST("blueprint node enumeration") { + auto a = std::make_unique<AndBlueprint>(); + a->addChild(std::make_unique<MyLeaf>()); + a->addChild(std::make_unique<MyLeaf>()); + auto b = std::make_unique<AndBlueprint>(); + b->addChild(std::make_unique<MyLeaf>()); + b->addChild(std::make_unique<MyLeaf>()); + auto root = std::make_unique<OrBlueprint>(); + root->addChild(std::move(a)); + root->addChild(std::move(b)); + TEST_DO(check_ids(*root, {0,0,0,0,0,0,0})); + root->enumerate(1); + TEST_DO(check_ids(*root, {3,4,2,6,7,5,1})); +} + TEST_MAIN() { TEST_DEBUG("lhs.out", "rhs.out"); TEST_RUN_ALL(); diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index 490f221d1d8..a8707bb6f7e 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mysearch.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/flow.h> diff --git a/searchlib/src/tests/queryeval/blueprint/leaf_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/leaf_blueprints_test.cpp index cb5473babbd..ea7f3d8fdc9 100644 --- a/searchlib/src/tests/queryeval/blueprint/leaf_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/leaf_blueprints_test.cpp @@ -1,33 +1,20 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/fef/matchdata.h> - -#include <vespa/log/log.h> -LOG_SETUP("blueprint_test"); +#include <vespa/vespalib/gtest/gtest.h> using namespace search::queryeval; using namespace search::fef; -class Test : public vespalib::TestApp -{ -public: - void testEmptyBlueprint(); - void testSimpleBlueprint(); - void testFakeBlueprint(); - int Main() override; -}; - -void -Test::testEmptyBlueprint() +TEST(LeafBlueprintsTest, empty_blueprint) { MatchData::UP md(MatchData::makeTestInstance(100, 10)); EmptyBlueprint empty(FieldSpecBase(1, 11)); ASSERT_TRUE(empty.getState().numFields() == 1u); - EXPECT_EQUAL(1u, empty.getState().field(0).getFieldId()); - EXPECT_EQUAL(11u, empty.getState().field(0).getHandle()); + EXPECT_EQ(1u, empty.getState().field(0).getFieldId()); + EXPECT_EQ(11u, empty.getState().field(0).getHandle()); empty.basic_plan(true, 100); empty.fetchPostings(ExecuteInfo::FULL); @@ -36,18 +23,17 @@ Test::testEmptyBlueprint() SimpleResult res; res.search(*search); SimpleResult expect; // empty - EXPECT_EQUAL(res, expect); + EXPECT_EQ(res, expect); } -void -Test::testSimpleBlueprint() +TEST(LeafBlueprintsTest, simple_blueprint) { MatchData::UP md(MatchData::makeTestInstance(100, 10)); SimpleResult a; a.addHit(3).addHit(5).addHit(7); SimpleBlueprint simple(a); simple.tag("tag"); - EXPECT_EQUAL("tag", simple.tag()); + EXPECT_EQ("tag", simple.tag()); simple.basic_plan(true, 100); simple.fetchPostings(ExecuteInfo::FULL); SearchIterator::UP search = simple.createSearch(*md); @@ -56,11 +42,10 @@ Test::testSimpleBlueprint() res.search(*search); SimpleResult expect; expect.addHit(3).addHit(5).addHit(7); - EXPECT_EQUAL(res, expect); + EXPECT_EQ(res, expect); } -void -Test::testFakeBlueprint() +TEST(LeafBlueprintsTest, fake_blueprint) { MatchData::UP md(MatchData::makeTestInstance(100, 10)); FakeResult fake; @@ -76,36 +61,36 @@ Test::testFakeBlueprint() SearchIterator::UP search = orig.createSearch(*md); search->initFullRange(); EXPECT_TRUE(!search->seek(1u)); - EXPECT_EQUAL(10u, search->getDocId()); + EXPECT_EQ(10u, search->getDocId()); { search->unpack(10u); TermFieldMatchData &data = *md->resolveTermField(handle); - EXPECT_EQUAL(fieldId, data.getFieldId()); - EXPECT_EQUAL(10u, data.getDocId()); - EXPECT_EQUAL(10u, data.getDocId()); + EXPECT_EQ(fieldId, data.getFieldId()); + EXPECT_EQ(10u, data.getDocId()); + EXPECT_EQ(10u, data.getDocId()); FieldPositionsIterator itr = data.getIterator(); - EXPECT_EQUAL(50u, itr.getFieldLength()); - EXPECT_EQUAL(2u, itr.size()); + EXPECT_EQ(50u, itr.getFieldLength()); + EXPECT_EQ(2u, itr.size()); ASSERT_TRUE(itr.valid()); - EXPECT_EQUAL(2u, itr.getPosition()); + EXPECT_EQ(2u, itr.getPosition()); itr.next(); ASSERT_TRUE(itr.valid()); - EXPECT_EQUAL(3u, itr.getPosition()); + EXPECT_EQ(3u, itr.getPosition()); itr.next(); EXPECT_TRUE(!itr.valid()); } EXPECT_TRUE(search->seek(25)); - EXPECT_EQUAL(25u, search->getDocId()); + EXPECT_EQ(25u, search->getDocId()); { search->unpack(25u); TermFieldMatchData &data = *md->resolveTermField(handle); - EXPECT_EQUAL(fieldId, data.getFieldId()); - EXPECT_EQUAL(25u, data.getDocId()); + EXPECT_EQ(fieldId, data.getFieldId()); + EXPECT_EQ(25u, data.getDocId()); FieldPositionsIterator itr = data.getIterator(); - EXPECT_EQUAL(10u, itr.getFieldLength()); - EXPECT_EQUAL(1u, itr.size()); + EXPECT_EQ(10u, itr.getFieldLength()); + EXPECT_EQ(1u, itr.size()); ASSERT_TRUE(itr.valid()); - EXPECT_EQUAL(5u, itr.getPosition()); + EXPECT_EQ(5u, itr.getPosition()); itr.next(); EXPECT_TRUE(!itr.valid()); } @@ -113,14 +98,4 @@ Test::testFakeBlueprint() EXPECT_TRUE(search->isAtEnd()); } -int -Test::Main() -{ - TEST_INIT("leaf_blueprints_test"); - testEmptyBlueprint(); - testSimpleBlueprint(); - testFakeBlueprint(); - TEST_DONE(); -} - -TEST_APPHOOK(Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/queryeval/dot_product/CMakeLists.txt b/searchlib/src/tests/queryeval/dot_product/CMakeLists.txt index e8b4e6d387c..0eedae38ab9 100644 --- a/searchlib/src/tests/queryeval/dot_product/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/dot_product/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_dot_product_test_app TEST SOURCES dot_product_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_dot_product_test_app COMMAND searchlib_dot_product_test_app) diff --git a/searchlib/src/tests/queryeval/equiv/CMakeLists.txt b/searchlib/src/tests/queryeval/equiv/CMakeLists.txt index a60ff8b7549..e2e350c2d18 100644 --- a/searchlib/src/tests/queryeval/equiv/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/equiv/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_equiv_test_app TEST SOURCES equiv_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_equiv_test_app COMMAND searchlib_equiv_test_app) diff --git a/searchlib/src/tests/queryeval/exact_nearest_neighbor/CMakeLists.txt b/searchlib/src/tests/queryeval/exact_nearest_neighbor/CMakeLists.txt new file mode 100644 index 00000000000..ca47743f1f9 --- /dev/null +++ b/searchlib/src/tests/queryeval/exact_nearest_neighbor/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(searchlib_exact_nearest_neighbor_test_app TEST + SOURCES + exact_nearest_neighbor_test.cpp + DEPENDS + vespa_searchlib + GTest::GTest +) +vespa_add_test(NAME searchlib_exact_nearest_neighbor_test_app COMMAND searchlib_exact_nearest_neighbor_test_app) diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/exact_nearest_neighbor/exact_nearest_neighbor_test.cpp index e4a8be121f5..bec627bb7ba 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/exact_nearest_neighbor/exact_nearest_neighbor_test.cpp @@ -7,7 +7,8 @@ #include <vespa/searchlib/common/feature.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/queryeval/global_filter.h> -#include <vespa/searchlib/queryeval/nearest_neighbor_iterator.h> +#include <vespa/searchlib/queryeval/exact_nearest_neighbor_iterator.h> +#include <vespa/searchlib/queryeval/matching_phase.h> #include <vespa/searchlib/queryeval/nns_index_iterator.h> #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> @@ -15,18 +16,15 @@ #include <vespa/searchlib/tensor/distance_function_factory.h> #include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/log/log.h> -LOG_SETUP("nearest_neighbor_test"); - #define EPS 1.0e-6 using search::AttributeVector; using search::BitVector; using search::attribute::DistanceMetric; using search::feature_t; +using search::queryeval::MatchingPhase; using search::tensor::DenseTensorAttribute; using search::tensor::DistanceCalculator; using search::tensor::SerializedFastValueAttribute; @@ -77,13 +75,15 @@ struct Fixture { vespalib::string _typeSpec; std::shared_ptr<TensorAttribute> _attr; std::shared_ptr<GlobalFilter> _global_filter; + MatchingPhase _matching_phase; Fixture(const vespalib::string &typeSpec) : _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name("test"), _typeSpec(typeSpec), _attr(), - _global_filter(GlobalFilter::create()) + _global_filter(GlobalFilter::create()), + _matching_phase(MatchingPhase::FIRST_PHASE) { _cfg.setTensorType(ValueType::from_spec(typeSpec)); _attr = make_attr(_name, _cfg); @@ -115,6 +115,7 @@ struct Fixture { auto t = createTensor(_typeSpec, v1, v2); setTensor(docId, *t); } + void set_second_phase() { _matching_phase = MatchingPhase::SECOND_PHASE; } }; template <bool strict> @@ -129,9 +130,10 @@ SimpleResult find_matches(Fixture &env, const Value &qtv, double threshold = std NearestNeighborDistanceHeap dh(2); dh.set_distance_threshold(threshold); const GlobalFilter &filter = *env._global_filter; - auto search = NearestNeighborIterator::create(strict, tfmd, - std::make_unique<DistanceCalculator>(attr, qtv), - dh, filter); + auto search = ExactNearestNeighborIterator::create(strict, tfmd, + std::make_unique<DistanceCalculator>(attr, qtv), + dh, filter, + env._matching_phase != MatchingPhase::FIRST_PHASE); if (strict) { return SimpleResult().searchStrict(*search, attr.getNumDocs()); } else { @@ -170,11 +172,29 @@ verify_iterator_returns_expected_results(const vespalib::string& attribute_tenso result = find_matches<false>(fixture, *nullTensor, 5.0); EXPECT_EQ(result, null_thr5_exp); + SimpleResult null_thr10_exp({1,2,4,6}); + result = find_matches<true>(fixture, *nullTensor, 10.0); + EXPECT_EQ(null_thr10_exp, result); + result = find_matches<false>(fixture, *nullTensor, 10.0); + EXPECT_EQ(null_thr10_exp, result); + SimpleResult far_thr4_exp({2,5}); result = find_matches<true>(fixture, *farTensor, 4.0); EXPECT_EQ(result, far_thr4_exp); result = find_matches<false>(fixture, *farTensor, 4.0); EXPECT_EQ(result, far_thr4_exp); + + fixture.set_second_phase(); + SimpleResult all_exp({1,2,3,4,5,6}); + result = find_matches<true>(fixture, *nullTensor); + EXPECT_EQ(all_exp, result); + result = find_matches<false>(fixture, *nullTensor); + EXPECT_EQ(all_exp, result); + SimpleResult null_thr10_second_phase_exp({1,2,4,5,6}); + result = find_matches<true>(fixture, *nullTensor, 10.0); + EXPECT_EQ(null_thr10_second_phase_exp, result); + result = find_matches<false>(fixture, *nullTensor, 10.0); + EXPECT_EQ(null_thr10_second_phase_exp, result); } struct TestParam { @@ -201,17 +221,17 @@ std::ostream& operator<<(std::ostream& os, const TestParam& param) return os; } -struct NnsIndexIteratorParameterizedTest : public ::testing::TestWithParam<TestParam> {}; +struct ExactNearestNeighborIteratorParameterizedTest : public ::testing::TestWithParam<TestParam> {}; -INSTANTIATE_TEST_SUITE_P(NnsTestSuite, - NnsIndexIteratorParameterizedTest, +INSTANTIATE_TEST_SUITE_P(ExactNearestNeighborIteratorTestSuite, + ExactNearestNeighborIteratorParameterizedTest, ::testing::Values( TestParam(denseSpecDouble, denseSpecDouble), TestParam(denseSpecFloat, denseSpecFloat), TestParam(mixed_spec, denseSpecDouble) )); -TEST_P(NnsIndexIteratorParameterizedTest, require_that_iterator_returns_expected_results) { +TEST_P(ExactNearestNeighborIteratorParameterizedTest, require_that_iterator_returns_expected_results) { auto param = GetParam(); verify_iterator_returns_expected_results(param.attribute_tensor_type_spec, param.query_tensor_type_spec); } @@ -243,7 +263,7 @@ verify_iterator_returns_filtered_results(const vespalib::string& attribute_tenso EXPECT_EQ(result, farExpect); } -TEST_P(NnsIndexIteratorParameterizedTest, require_that_iterator_returns_filtered_results) { +TEST_P(ExactNearestNeighborIteratorParameterizedTest, require_that_iterator_returns_filtered_results) { auto param = GetParam(); verify_iterator_returns_filtered_results(param.attribute_tensor_type_spec, param.query_tensor_type_spec); } @@ -256,9 +276,9 @@ std::vector<feature_t> get_rawscores(Fixture &env, const Value &qtv) { auto dff = search::tensor::make_distance_function_factory(DistanceMetric::Euclidean, qtv.cells().type); NearestNeighborDistanceHeap dh(2); auto dummy_filter = GlobalFilter::create(); - auto search = NearestNeighborIterator::create(strict, tfmd, - std::make_unique<DistanceCalculator>(attr, qtv), - dh, *dummy_filter); + auto search = ExactNearestNeighborIterator::create(strict, tfmd, + std::make_unique<DistanceCalculator>(attr, qtv), + dh, *dummy_filter, false); uint32_t limit = attr.getNumDocs(); uint32_t docid = 1; search->initRange(docid, limit); @@ -299,7 +319,7 @@ verify_iterator_sets_expected_rawscore(const vespalib::string& attribute_tensor_ } } -TEST_P(NnsIndexIteratorParameterizedTest, require_that_iterator_sets_expected_rawscore) { +TEST_P(ExactNearestNeighborIteratorParameterizedTest, require_that_iterator_sets_expected_rawscore) { auto param = GetParam(); verify_iterator_sets_expected_rawscore(param.attribute_tensor_type_spec, param.query_tensor_type_spec); } diff --git a/searchlib/src/tests/queryeval/fake_searchable/CMakeLists.txt b/searchlib/src/tests/queryeval/fake_searchable/CMakeLists.txt index fa51c91262d..8e0cee04877 100644 --- a/searchlib/src/tests/queryeval/fake_searchable/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/fake_searchable/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_fake_searchable_test_app TEST SOURCES fake_searchable_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_fake_searchable_test_app COMMAND searchlib_fake_searchable_test_app) diff --git a/searchlib/src/tests/queryeval/filter_search/CMakeLists.txt b/searchlib/src/tests/queryeval/filter_search/CMakeLists.txt index 1d42766b5b4..ff0c6aa515c 100644 --- a/searchlib/src/tests/queryeval/filter_search/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/filter_search/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_filter_search_test_app TEST SOURCES filter_search_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp b/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp index 16e78f77eec..9fdf1417a92 100644 --- a/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp +++ b/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp @@ -356,7 +356,7 @@ DotProductAdapter::~DotProductAdapter() = default; struct ParallelWeakAndAdapter { FieldSpec field; ParallelWeakAndBlueprint blueprint; - ParallelWeakAndAdapter() : field("foo", 3, 7), blueprint(field, 100, 0.0, 1.0) {} + ParallelWeakAndAdapter() : field("foo", 3, 7), blueprint(field, 100, 0.0, 1.0, true) {} void addChild(std::unique_ptr<Blueprint> child) { auto child_field = blueprint.getNextChildField(field); auto term = std::make_unique<LeafProxy>(child_field, std::move(child)); diff --git a/searchlib/src/tests/queryeval/flow/CMakeLists.txt b/searchlib/src/tests/queryeval/flow/CMakeLists.txt index 70658d36f21..4739494ee59 100644 --- a/searchlib/src/tests/queryeval/flow/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/flow/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_queryeval_flow_test_app TEST SOURCES queryeval_flow_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_queryeval_flow_test_app COMMAND searchlib_queryeval_flow_test_app) diff --git a/searchlib/src/tests/queryeval/getnodeweight/CMakeLists.txt b/searchlib/src/tests/queryeval/getnodeweight/CMakeLists.txt index 7720e0637cf..d581eae858b 100644 --- a/searchlib/src/tests/queryeval/getnodeweight/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/getnodeweight/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_getnodeweight_test_app TEST SOURCES getnodeweight_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_getnodeweight_test_app COMMAND searchlib_getnodeweight_test_app) diff --git a/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp b/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp index d9b7d5b3192..fda2e84402a 100644 --- a/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp +++ b/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/get_weight_from_node.h> diff --git a/searchlib/src/tests/queryeval/global_filter/CMakeLists.txt b/searchlib/src/tests/queryeval/global_filter/CMakeLists.txt index 3763fb81afa..67725cf0513 100644 --- a/searchlib/src/tests/queryeval/global_filter/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/global_filter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_queryeval_global_filter_test_app TEST SOURCES global_filter_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_queryeval_global_filter_test_app COMMAND searchlib_queryeval_global_filter_test_app) diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/CMakeLists.txt b/searchlib/src/tests/queryeval/iterator_benchmark/CMakeLists.txt index dadd06ee7cd..b8587d93fe7 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/iterator_benchmark/CMakeLists.txt @@ -8,7 +8,7 @@ vespa_add_executable(searchlib_iterator_benchmark_test_app TEST disk_index_builder.cpp iterator_benchmark_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp index db0fe76b7af..e74fefac70e 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp @@ -13,13 +13,13 @@ #include <vector> using namespace search::attribute; -using namespace search::fef; using namespace search::queryeval::test; using namespace search::queryeval; using namespace search; using namespace vespalib; using search::index::Schema; +using search::fef::MatchData; using vespalib::make_string_short::fmt; diff --git a/searchlib/src/tests/queryeval/matching_elements_search/CMakeLists.txt b/searchlib/src/tests/queryeval/matching_elements_search/CMakeLists.txt index d4ec1b83887..26963c4443f 100644 --- a/searchlib/src/tests/queryeval/matching_elements_search/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/matching_elements_search/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_matching_elements_search_test_app TEST SOURCES matching_elements_search_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/queryeval/monitoring_search_iterator/CMakeLists.txt b/searchlib/src/tests/queryeval/monitoring_search_iterator/CMakeLists.txt index bf2af32abc6..3a68707537f 100644 --- a/searchlib/src/tests/queryeval/monitoring_search_iterator/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/monitoring_search_iterator/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_monitoring_search_iterator_test_app TEST SOURCES monitoring_search_iterator_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_monitoring_search_iterator_test_app COMMAND searchlib_monitoring_search_iterator_test_app) diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/CMakeLists.txt b/searchlib/src/tests/queryeval/multibitvectoriterator/CMakeLists.txt index dff8ea2ef0c..784277447c3 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_multibitvectoriterator_test_app TEST SOURCES multibitvectoriterator_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_multibitvectoriterator_test_app COMMAND searchlib_multibitvectoriterator_test_app) @@ -11,6 +11,6 @@ vespa_add_executable(searchlib_multibitvectoriterator_bench_app SOURCES multibitvectoriterator_bench.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_multibitvectoriterator_bench_app COMMAND searchlib_multibitvectoriterator_bench_app and no no 10 100000000 50 50 50 BENCHMARK) diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp index 95e80cd08b8..ddb9cab18ab 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/multibitvectoriterator.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/common/bitvectoriterator.h> @@ -17,15 +17,11 @@ using namespace search; //----------------------------------------------------------------------------- -class Test : public vespalib::TestApp +struct Fixture { -public: - ~Test() {} void benchmark(); - int Main() override; template <typename T> void testSearch(bool strict); -private: void searchAndCompare(SearchIterator::UP s, uint32_t docIdLimit); void setup(); std::vector< BitVector::UP > _bvs; @@ -35,9 +31,22 @@ private: bool _optimize; vespalib::string _type; std::vector<int> _fillLimits; + + Fixture(int argc, char **argv) { + _type = argv[1]; + _strict = vespalib::string(argv[2]) == vespalib::string("strict"); + _optimize = vespalib::string(argv[3]) == vespalib::string("optimize"); + _numSearch = strtoul(argv[4], NULL, 0); + _numDocs = strtoul(argv[5], NULL, 0); + for (int i(6); i < argc; i++) { + _fillLimits.push_back((RAND_MAX/100) * strtoul(argv[i], NULL, 0)); + } + } + ~Fixture(); }; +Fixture::~Fixture() = default; -void Test::setup() +void Fixture::setup() { for(size_t i(0); i < _fillLimits.size(); i++) { _bvs.push_back(BitVector::create(_numDocs)); @@ -76,7 +85,7 @@ seek(SearchIterator & s, uint32_t docIdLimit) } void -Test::benchmark() +Fixture::benchmark() { if (_type == "and") { LOG(info, "Testing 'and'"); @@ -93,7 +102,7 @@ Test::benchmark() template <typename T> void -Test::testSearch(bool strict) +Fixture::testSearch(bool strict) { TermFieldMatchData tfmd; MultiSearch::Children andd; @@ -109,29 +118,15 @@ Test::testSearch(bool strict) LOG(info, "Found %ld hits", h.size()); } -int -Test::Main() -{ - TEST_INIT("multibitvectoriterator_benchmark"); - if (_argc < 6) { - LOG(info, "%s <'and/or'> <'strict/no-strict'> <'optimize/no-optimize> <numsearch> <numdocs> <fill 1> [<fill N>]", _argv[0]); - return -1; - } - _type = _argv[1]; - _strict = vespalib::string(_argv[2]) == vespalib::string("strict"); - _optimize = vespalib::string(_argv[3]) == vespalib::string("optimize"); - _numSearch = strtoul(_argv[4], NULL, 0); - _numDocs = strtoul(_argv[5], NULL, 0); - for (int i(6); i < _argc; i++) { - _fillLimits.push_back((RAND_MAX/100) * strtoul(_argv[i], NULL, 0)); +TEST_MAIN() { + if (argc < 6) { + LOG(info, "%s <'and/or'> <'strict/no-strict'> <'optimize/no-optimize> <numsearch> <numdocs> <fill 1> [<fill N>]", argv[0]); + exit(1); } - LOG(info, "Start setup of '%s' isearch with %ld vectors with %d documents", _type.c_str(), _fillLimits.size(), _numDocs); - setup(); + Fixture fixture(argc, argv); + LOG(info, "Start setup of '%s' isearch with %ld vectors with %d documents", fixture._type.c_str(), fixture._fillLimits.size(), fixture._numDocs); + fixture.setup(); LOG(info, "Start benchmark"); - benchmark(); + fixture.benchmark(); LOG(info, "Done benchmark"); - TEST_FLUSH(); - TEST_DONE(); } - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/CMakeLists.txt b/searchlib/src/tests/queryeval/nearest_neighbor/CMakeLists.txt deleted file mode 100644 index b68f7f93c18..00000000000 --- a/searchlib/src/tests/queryeval/nearest_neighbor/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -vespa_add_executable(searchlib_nearest_neighbor_test_app TEST - SOURCES - nearest_neighbor_test.cpp - DEPENDS - searchlib - GTest::GTest -) -vespa_add_test(NAME searchlib_nearest_neighbor_test_app COMMAND searchlib_nearest_neighbor_test_app) diff --git a/searchlib/src/tests/queryeval/or_speed/CMakeLists.txt b/searchlib/src/tests/queryeval/or_speed/CMakeLists.txt index 950a3a965be..f8b784b6a91 100644 --- a/searchlib/src/tests/queryeval/or_speed/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/or_speed/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_or_speed_test_app TEST SOURCES or_speed_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_or_speed_test_app COMMAND searchlib_or_speed_test_app) diff --git a/searchlib/src/tests/queryeval/parallel_weak_and/CMakeLists.txt b/searchlib/src/tests/queryeval/parallel_weak_and/CMakeLists.txt index 533e610b32a..0f16aadd8d6 100644 --- a/searchlib/src/tests/queryeval/parallel_weak_and/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/parallel_weak_and/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_parallel_weak_and_test_app TEST SOURCES parallel_weak_and_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_parallel_weak_and_test_app COMMAND searchlib_parallel_weak_and_test_app) diff --git a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp index 2bd560637d2..0ae3031e608 100644 --- a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp +++ b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp @@ -68,8 +68,8 @@ struct TestHeap : public WeakAndHeap { ScoresHistory history; - TestHeap(uint32_t scoresToTrack_) : WeakAndHeap(scoresToTrack_), history() {} - virtual void adjust(score_t *begin, score_t *end) override { + explicit TestHeap(uint32_t scoresToTrack_) : WeakAndHeap(scoresToTrack_), history() {} + void adjust(score_t *begin, score_t *end) override { Scores scores; for (score_t *itr = begin; itr != end; ++itr) { scores.add(*itr); @@ -86,13 +86,15 @@ struct WandTestSpec : public WandSpec HeapType heap; TermFieldMatchData rootMatchData; MatchParams matchParams; + MatchingPhase matching_phase; - WandTestSpec(uint32_t scoresToTrack, uint32_t scoresAdjustFrequency = 1, - score_t scoreThreshold = 0, double thresholdBoostFactor = 1); + explicit WandTestSpec(uint32_t scoresToTrack, uint32_t scoresAdjustFrequency = 1, + score_t scoreThreshold = 0, double thresholdBoostFactor = 1); ~WandTestSpec(); SearchIterator::UP create() { MatchData::UP childrenMatchData = createMatchData(); MatchData *tmp = childrenMatchData.get(); + bool readonly_scores_heap = (matching_phase != MatchingPhase::FIRST_PHASE); return SearchIterator::UP( new TrackedSearch("PWAND", getHistory(), ParallelWeakAndSearch::create( @@ -100,8 +102,9 @@ struct WandTestSpec : public WandSpec matchParams, RankParams(rootMatchData, std::move(childrenMatchData)), - true))); + true, readonly_scores_heap))); } + void set_second_phase() { matching_phase = MatchingPhase::SECOND_PHASE; } }; template <typename HeapType> @@ -110,11 +113,12 @@ WandTestSpec<HeapType>::WandTestSpec(uint32_t scoresToTrack, uint32_t scoresAdju : WandSpec(), heap(scoresToTrack), rootMatchData(), - matchParams(heap, scoreThreshold, thresholdBoostFactor, scoresAdjustFrequency) + matchParams(heap, scoreThreshold, thresholdBoostFactor, scoresAdjustFrequency), + matching_phase(MatchingPhase::FIRST_PHASE) {} template <typename HeapType> -WandTestSpec<HeapType>::~WandTestSpec() {} +WandTestSpec<HeapType>::~WandTestSpec() = default; using WandSpecWithTestHeap = WandTestSpec<TestHeap>; using WandSpecWithRealHeap = WandTestSpec<SharedWeakAndPriorityQueue>; @@ -137,8 +141,8 @@ SimpleResult asSimpleResult(const FakeResult &result) { SimpleResult retval; - for (size_t i = 0; i < result.inspect().size(); ++i) { - retval.addHit(result.inspect()[i].docId); + for (const auto & doc : result.inspect()) { + retval.addHit(doc.docId); } return retval; } @@ -152,26 +156,26 @@ struct WandBlueprintSpec FakeRequestContext requestContext; WandBlueprintSpec &add(const std::string &token, int32_t weight) { - tokens.push_back(std::make_pair(token, weight)); + tokens.emplace_back(token, weight); return *this; } Node::UP createNode(uint32_t scoresToTrack = 100, score_t scoreThreshold = 0, double thresholdBoostFactor = 1) const { - SimpleWandTerm *node = new SimpleWandTerm(tokens.size(), "view", 0, Weight(0), - scoresToTrack, scoreThreshold, thresholdBoostFactor); - for (size_t i = 0; i < tokens.size(); ++i) { - node->addTerm(tokens[i].first, Weight(tokens[i].second)); + auto node = std::make_unique<SimpleWandTerm>(tokens.size(), "view", 0, Weight(0), + scoresToTrack, scoreThreshold, thresholdBoostFactor); + for (const auto & token : tokens) { + node->addTerm(token.first, Weight(token.second)); } - return Node::UP(node); + return node; } Blueprint::UP blueprint(Searchable &searchable, const std::string &field, const search::query::Node &term) const { FieldSpecList fields; fields.add(FieldSpec(field, fieldId, handle)); Blueprint::UP bp = searchable.createBlueprint(requestContext, fields, term); - EXPECT_TRUE(dynamic_cast<ParallelWeakAndBlueprint*>(bp.get()) != 0); + EXPECT_TRUE(dynamic_cast<ParallelWeakAndBlueprint*>(bp.get()) != nullptr); return bp; } @@ -182,7 +186,7 @@ struct WandBlueprintSpec bp->basic_plan(true, docIdLimit); bp->fetchPostings(ExecuteInfo::FULL); SearchIterator::UP sb = bp->createSearch(*md); - EXPECT_TRUE(dynamic_cast<ParallelWeakAndSearch*>(sb.get()) != 0); + EXPECT_TRUE(dynamic_cast<ParallelWeakAndSearch*>(sb.get()) != nullptr); return sb; } @@ -197,7 +201,7 @@ struct WandBlueprintSpec bp->basic_plan(true, docIdLimit); bp->fetchPostings(ExecuteInfo::FULL); SearchIterator::UP sb = bp->createSearch(*md); - EXPECT_TRUE(dynamic_cast<ParallelWeakAndSearch*>(sb.get()) != 0); + EXPECT_TRUE(dynamic_cast<ParallelWeakAndSearch*>(sb.get()) != nullptr); return doSearch(*sb, *md->resolveTermField(handle)); } }; @@ -220,7 +224,16 @@ struct FixtureBase struct AlgoSimpleFixture : public FixtureBase { - AlgoSimpleFixture() : FixtureBase(2, 1) { + AlgoSimpleFixture() + : AlgoSimpleFixture(false) + { + } + explicit AlgoSimpleFixture(bool second_phase) + : FixtureBase(2, 1) + { + if (second_phase) { + spec.set_second_phase(); + } spec.leaf(LeafSpec("A", 1).doc(1, 1).doc(2, 2).doc(3, 3).doc(4, 4).doc(5, 5).doc(6, 6)); spec.leaf(LeafSpec("B", 4).doc(1, 1).doc(3, 3).doc(5, 5)); prepare(); @@ -258,7 +271,7 @@ struct AlgoSameScoreFixture : public FixtureBase struct AlgoScoreThresholdFixture : public FixtureBase { - AlgoScoreThresholdFixture(score_t scoreThreshold) : FixtureBase(3, 1, scoreThreshold) { + explicit AlgoScoreThresholdFixture(score_t scoreThreshold) : FixtureBase(3, 1, scoreThreshold) { spec.leaf(LeafSpec("A", 1).doc(1, 10).doc(2, 30)); spec.leaf(LeafSpec("B", 2).doc(1, 20).doc(3, 40)); prepare(); @@ -267,7 +280,7 @@ struct AlgoScoreThresholdFixture : public FixtureBase struct AlgoLargeScoresFixture : public FixtureBase { - AlgoLargeScoresFixture(score_t scoreThreshold) : FixtureBase(3, 1, scoreThreshold) { + explicit AlgoLargeScoresFixture(score_t scoreThreshold) : FixtureBase(3, 1, scoreThreshold) { spec.leaf(LeafSpec("A", 60000).doc(1, 60000).doc(2, 70000)); spec.leaf(LeafSpec("B", 70000).doc(1, 80000).doc(3, 90000)); prepare(); @@ -276,7 +289,7 @@ struct AlgoLargeScoresFixture : public FixtureBase struct AlgoExhaustPastFixture : public FixtureBase { - AlgoExhaustPastFixture(score_t scoreThreshold) : FixtureBase(3, 1, scoreThreshold) { + explicit AlgoExhaustPastFixture(score_t scoreThreshold) : FixtureBase(3, 1, scoreThreshold) { spec.leaf(LeafSpec("A", 1).doc(1, 20).doc(3, 40).doc(5, 10)); spec.leaf(LeafSpec("B", 1).doc(5, 10)); spec.leaf(LeafSpec("C", 1).doc(5, 10)); @@ -287,7 +300,7 @@ struct AlgoExhaustPastFixture : public FixtureBase TEST(ParallelWeakAndTest, require_that_algorithm_prunes_bad_hits_after_enough_good_ones_are_obtained) { - AlgoSimpleFixture f; + AlgoSimpleFixture f; // First phase FakeResult expect = FakeResult() .doc(1).score(1 * 1 + 4 * 1) .doc(2).score(1 * 2) @@ -296,6 +309,19 @@ TEST(ParallelWeakAndTest, require_that_algorithm_prunes_bad_hits_after_enough_go EXPECT_EQ(expect, f.result); } +TEST(ParallelWeakAndTest, require_that_algorithm_does_not_prune_hits_in_pater_matching_phases) +{ + AlgoSimpleFixture f(true); // Second phase + FakeResult expect = FakeResult() + .doc(1).score(1 * 1 + 4 * 1) + .doc(2).score(1 * 2) + .doc(3).score(1 * 3 + 4 * 3) + .doc(4).score(1 * 4) + .doc(5).score(1 * 5 + 4 * 5) + .doc(6).score(1 * 6); + EXPECT_EQ(expect, f.result); +} + TEST(ParallelWeakAndTest, require_that_algorithm_uses_subsearches_as_expected) { AlgoSimpleFixture f; @@ -449,11 +475,11 @@ struct BlueprintFixtureBase }; BlueprintFixtureBase::BlueprintFixtureBase() : spec(), searchable() {} -BlueprintFixtureBase::~BlueprintFixtureBase() {} +BlueprintFixtureBase::~BlueprintFixtureBase() = default; struct BlueprintHitsFixture : public BlueprintFixtureBase { - FakeResult createResult(size_t hits) { + static FakeResult createResult(size_t hits) { FakeResult result; for (size_t i = 0; i < hits; ++i) { result.doc(i + 1); @@ -479,7 +505,7 @@ struct BlueprintHitsFixture : public BlueprintFixtureBase struct ThresholdBoostFixture : public FixtureBase { FakeResult result; - ThresholdBoostFixture(double boost) : FixtureBase(1, 1, 800, boost) { + explicit ThresholdBoostFixture(double boost) : FixtureBase(1, 1, 800, boost) { spec.leaf(LeafSpec("A").doc(1, 10)); spec.leaf(LeafSpec("B").doc(2, 20)); spec.leaf(LeafSpec("C").doc(3, 30)); @@ -532,7 +558,7 @@ TEST(ParallelWeakAndTest, require_that_blueprint_picks_up_docid_limit) BlueprintFixture f; Node::UP term = f.spec.createNode(57, 67, 77.7); Blueprint::UP bp = f.blueprint(*term); - const ParallelWeakAndBlueprint * pbp = dynamic_cast<const ParallelWeakAndBlueprint *>(bp.get()); + const auto * pbp = dynamic_cast<const ParallelWeakAndBlueprint *>(bp.get()); EXPECT_EQ(0u, pbp->get_docid_limit()); bp->setDocIdLimit(1000); EXPECT_EQ(1000u, pbp->get_docid_limit()); @@ -543,7 +569,7 @@ TEST(ParallelWeakAndTest, require_that_scores_to_track_score_threshold_and_thres BlueprintFixture f; Node::UP term = f.spec.createNode(57, 67, 77.7); Blueprint::UP bp = f.blueprint(*term); - const ParallelWeakAndBlueprint * pbp = dynamic_cast<const ParallelWeakAndBlueprint *>(bp.get()); + const auto * pbp = dynamic_cast<const ParallelWeakAndBlueprint *>(bp.get()); EXPECT_EQ(57u, pbp->getScores().getScoresToTrack()); EXPECT_EQ(67u, pbp->getScoreThreshold()); EXPECT_EQ(77.7, pbp->getThresholdBoostFactor()); @@ -635,6 +661,7 @@ TEST(ParallelWeakAndTest, require_that_asString_on_blueprint_works) " strict_cost: 0\n" " sourceId: 4294967295\n" " docid_limit: 0\n" + " id: 0\n" " strict: false\n" " _weights: std::vector {\n" " [0]: 5\n" @@ -661,6 +688,7 @@ TEST(ParallelWeakAndTest, require_that_asString_on_blueprint_works) " strict_cost: 0\n" " sourceId: 4294967295\n" " docid_limit: 0\n" + " id: 0\n" " strict: false\n" " }\n" " }\n" @@ -685,7 +713,7 @@ SearchIterator::UP create_wand(bool use_dww, bool strict) { if (use_dww) { - return ParallelWeakAndSearch::create(tfmd, matchParams, weights, dict_entries, attr, strict); + return ParallelWeakAndSearch::create(tfmd, matchParams, weights, dict_entries, attr, strict, false); } // use search iterators as children MatchDataLayout layout; @@ -703,12 +731,12 @@ SearchIterator::UP create_wand(bool use_dww, childrenMatchData->resolveTermField(handles[i]))); } assert(terms.size() == dict_entries.size()); - return SearchIterator::UP(ParallelWeakAndSearch::create(terms, matchParams, RankParams(tfmd, std::move(childrenMatchData)), strict)); + return SearchIterator::UP(ParallelWeakAndSearch::create(terms, matchParams, RankParams(tfmd, std::move(childrenMatchData)), strict, false)); } class Verifier : public search::test::DwwIteratorChildrenVerifier { public: - Verifier(bool use_dww) : _use_dww(use_dww) { } + explicit Verifier(bool use_dww) : _use_dww(use_dww) { } private: SearchIterator::UP create(bool strict) const override { MatchParams match_params(_dummy_heap, _dummy_heap.getMinScore(), 1.0, 1); diff --git a/searchlib/src/tests/queryeval/predicate/CMakeLists.txt b/searchlib/src/tests/queryeval/predicate/CMakeLists.txt index 17aac2a9391..5aed1e9a4c2 100644 --- a/searchlib/src/tests/queryeval/predicate/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/predicate/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(searchlib_predicate_blueprint_test_app TEST SOURCES predicate_blueprint_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_predicate_blueprint_test_app COMMAND searchlib_predicate_blueprint_test_app) vespa_add_executable(searchlib_predicate_search_test_app TEST SOURCES predicate_search_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_predicate_search_test_app COMMAND searchlib_predicate_search_test_app) diff --git a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp index ffa2905ce0e..1ae6cff447e 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp @@ -11,7 +11,7 @@ #include <vespa/searchlib/queryeval/field_spec.h> #include <vespa/searchlib/queryeval/predicate_blueprint.h> #include <vespa/searchlib/predicate/predicate_hash.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("predicate_blueprint_test"); diff --git a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp index a69b4c7a45d..b02df263f82 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp @@ -7,7 +7,7 @@ LOG_SETUP("predicate_search_test"); #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/queryeval/predicate_search.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/arraysize.h> using search::fef::TermFieldMatchData; diff --git a/searchlib/src/tests/queryeval/profiled_iterator/CMakeLists.txt b/searchlib/src/tests/queryeval/profiled_iterator/CMakeLists.txt index 77fd0a1898b..0f291465f9c 100644 --- a/searchlib/src/tests/queryeval/profiled_iterator/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/profiled_iterator/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_queryeval_profiled_iterator_test_app TEST SOURCES profiled_iterator_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_queryeval_profiled_iterator_test_app COMMAND searchlib_queryeval_profiled_iterator_test_app) diff --git a/searchlib/src/tests/queryeval/profiled_iterator/profiled_iterator_test.cpp b/searchlib/src/tests/queryeval/profiled_iterator/profiled_iterator_test.cpp index d0942e14f7c..dbf01fac4f7 100644 --- a/searchlib/src/tests/queryeval/profiled_iterator/profiled_iterator_test.cpp +++ b/searchlib/src/tests/queryeval/profiled_iterator/profiled_iterator_test.cpp @@ -6,6 +6,8 @@ #include <vespa/vespalib/util/require.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/searchlib/queryeval/profiled_iterator.h> +#include <vespa/searchlib/queryeval/wand/weak_and_heap.h> +#include <vespa/searchlib/queryeval/wand/weak_and_search.h> #include <vespa/searchlib/queryeval/simplesearch.h> #include <vespa/searchlib/queryeval/sourceblendersearch.h> #include <vespa/searchlib/queryeval/andsearch.h> @@ -84,6 +86,19 @@ SearchIterator::UP create_iterator_tree() { t({2,4,6,8}), 5)); } +SearchIterator::UP create_weak_and() { + struct DummyHeap : WeakAndHeap { + void adjust(score_t *, score_t *) override {} + DummyHeap() : WeakAndHeap(100) {} + }; + static DummyHeap dummy_heap; + WeakAndSearch::Terms terms; + terms.emplace_back(T({1,2,3}).release(), 100, 3); + terms.emplace_back(T({5,6}).release(), 200, 2); + terms.emplace_back(T({8}).release(), 300, 1); + return WeakAndSearch::create(terms, wand::MatchParams(dummy_heap), wand::TermFrequencyScorer(), 100, true, true); +} + void collect(std::map<vespalib::string,size_t> &counts, const auto &node) { if (!node.valid()) { return; @@ -190,4 +205,25 @@ TEST(ProfiledIteratorTest, iterator_tree_can_be_profiled) { EXPECT_EQ(counts["/1/1/SimpleSearch/init"], 2); } +TEST(ProfiledIteratorTest, weak_and_can_be_profiled) { + ExecutionProfiler profiler(64); + auto root = create_weak_and(); + root = ProfiledIterator::profile(profiler, std::move(root)); + fprintf(stderr, "%s", root->asString().c_str()); + verify_result(*root, {1,2,3,5,6,8}); + Slime slime; + profiler.report(slime.setObject()); + fprintf(stderr, "%s", slime.toString().c_str()); + auto counts = collect_counts(slime.get()); + print_counts(counts); + EXPECT_EQ(counts["/WeakAndSearchLR/init"], 1); + EXPECT_EQ(counts["/0/SimpleSearch/init"], 1); + EXPECT_EQ(counts["/1/SimpleSearch/init"], 1); + EXPECT_EQ(counts["/2/SimpleSearch/init"], 1); + EXPECT_EQ(counts["/WeakAndSearchLR/seek"], 7); + EXPECT_EQ(counts["/0/SimpleSearch/seek"], 4); + EXPECT_EQ(counts["/1/SimpleSearch/seek"], 3); + EXPECT_EQ(counts["/2/SimpleSearch/seek"], 2); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/queryeval/same_element/CMakeLists.txt b/searchlib/src/tests/queryeval/same_element/CMakeLists.txt index 615a7fcac9f..268402ceaaa 100644 --- a/searchlib/src/tests/queryeval/same_element/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/same_element/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_same_element_test_app TEST SOURCES same_element_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_same_element_test_app COMMAND searchlib_same_element_test_app) diff --git a/searchlib/src/tests/queryeval/simple_phrase/CMakeLists.txt b/searchlib/src/tests/queryeval/simple_phrase/CMakeLists.txt index e01af073639..f3b4aed6ccf 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/simple_phrase/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_simple_phrase_test_app TEST SOURCES simple_phrase_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_simple_phrase_test_app COMMAND searchlib_simple_phrase_test_app) diff --git a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp index 3e779bdca14..184d21fcaae 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp +++ b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp @@ -10,7 +10,7 @@ #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/weight.h> #include <vespa/vespalib/util/testclock.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("simple_phrase_test"); @@ -46,46 +46,6 @@ struct MyTerm : public search::queryeval::SimpleLeafBlueprint { } }; -class Test : public vespalib::TestApp { - void requireThatIteratorFindsSimplePhrase(bool useBlueprint); - void requireThatIteratorFindsLongPhrase(bool useBlueprint); - void requireThatStrictIteratorFindsNextMatch(bool useBlueprint); - void requireThatPhrasesAreUnpacked(bool useBlueprint, bool unpack_normal_features, bool unpack_interleaved_features); - void requireThatTermsCanBeEvaluatedInPriorityOrder(); - void requireThatBlueprintExposesFieldWithEstimate(); - void requireThatBlueprintForcesPositionDataOnChildren(); - -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("phrasesearch_test"); - - TEST_DO(requireThatIteratorFindsSimplePhrase(false)); - TEST_DO(requireThatIteratorFindsLongPhrase(false)); - TEST_DO(requireThatStrictIteratorFindsNextMatch(false)); - TEST_DO(requireThatPhrasesAreUnpacked(false, true, false)); - TEST_DO(requireThatPhrasesAreUnpacked(false, true, true)); - TEST_DO(requireThatPhrasesAreUnpacked(false, false, false)); - TEST_DO(requireThatPhrasesAreUnpacked(false, false, true)); - TEST_DO(requireThatTermsCanBeEvaluatedInPriorityOrder()); - - TEST_DO(requireThatIteratorFindsSimplePhrase(true)); - TEST_DO(requireThatIteratorFindsLongPhrase(true)); - TEST_DO(requireThatStrictIteratorFindsNextMatch(true)); - TEST_DO(requireThatPhrasesAreUnpacked(true, true, false)); - TEST_DO(requireThatPhrasesAreUnpacked(true, true, true)); - TEST_DO(requireThatPhrasesAreUnpacked(true, false, false)); - TEST_DO(requireThatPhrasesAreUnpacked(true, false, true)); - TEST_DO(requireThatBlueprintExposesFieldWithEstimate()); - TEST_DO(requireThatBlueprintForcesPositionDataOnChildren()); - - TEST_DONE(); -} - const string field = "field"; const uint32_t fieldId = 1; const uint32_t doc_match = 42; @@ -198,73 +158,85 @@ PhraseSearchTest::PhraseSearchTest(bool expiredDoom) {} PhraseSearchTest::~PhraseSearchTest() = default; -void Test::requireThatIteratorFindsSimplePhrase(bool useBlueprint) { - PhraseSearchTest test; - test.addTerm("foo", 0).addTerm("bar", 1); +TEST("requireThatIteratorFindsSimplePhrase") { + for (bool useBlueprint: {false, true}) { + PhraseSearchTest test; + test.addTerm("foo", 0).addTerm("bar", 1); - test.fetchPostings(useBlueprint); - unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_TRUE(search->seek(doc_match)); - EXPECT_TRUE(!search->seek(doc_no_match)); + test.fetchPostings(useBlueprint); + unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_TRUE(search->seek(doc_match)); + EXPECT_TRUE(!search->seek(doc_no_match)); + } } -void Test::requireThatIteratorFindsLongPhrase(bool useBlueprint) { - PhraseSearchTest test; - test.addTerm("foo", 0).addTerm("bar", 0).addTerm("baz", 0) - .addTerm("qux", 1); - - test.fetchPostings(useBlueprint); - unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_TRUE(search->seek(doc_match)); - EXPECT_TRUE(!search->seek(doc_no_match)); +TEST("requireThatIteratorFindsLongPhrase") { + for (bool useBlueprint: {false, true}) { + PhraseSearchTest test; + test.addTerm("foo", 0).addTerm("bar", 0).addTerm("baz", 0) + .addTerm("qux", 1); + + test.fetchPostings(useBlueprint); + unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_TRUE(search->seek(doc_match)); + EXPECT_TRUE(!search->seek(doc_no_match)); + } } -void Test::requireThatStrictIteratorFindsNextMatch(bool useBlueprint) { - PhraseSearchTest test; - test.setStrict(true); - test.addTerm("foo", 0).addTerm("bar", 1); - - test.fetchPostings(useBlueprint); - unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_EQUAL(doc_match, search->getDocId()); - EXPECT_TRUE(!search->seek(doc_no_match)); - EXPECT_TRUE(search->isAtEnd()); +TEST("requireThatStrictIteratorFindsNextMatch") { + for (bool useBlueprint: {false, true}) { + PhraseSearchTest test; + test.setStrict(true); + test.addTerm("foo", 0).addTerm("bar", 1); + + test.fetchPostings(useBlueprint); + unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_EQUAL(doc_match, search->getDocId()); + EXPECT_TRUE(!search->seek(doc_no_match)); + EXPECT_TRUE(search->isAtEnd()); + } } -void Test::requireThatPhrasesAreUnpacked(bool useBlueprint, bool unpack_normal_features, bool unpack_interleaved_features) { - PhraseSearchTest test; - test.addTerm("foo", FakeResult() - .doc(doc_match).pos(1).pos(11).pos(21).field_length(30).num_occs(3)); - test.addTerm("bar", FakeResult() - .doc(doc_match).pos(2).pos(16).pos(22).field_length(30).num_occs(3)); - test.writable_term_field_match_data().setNeedNormalFeatures(unpack_normal_features); - test.writable_term_field_match_data().setNeedInterleavedFeatures(unpack_interleaved_features); - test.fetchPostings(useBlueprint); - unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); - EXPECT_TRUE(search->seek(doc_match)); - search->unpack(doc_match); - - EXPECT_EQUAL(doc_match, test.tmd().getDocId()); - if (unpack_normal_features) { - EXPECT_EQUAL(2, std::distance(test.tmd().begin(), test.tmd().end())); - EXPECT_EQUAL(1u, test.tmd().begin()->getPosition()); - EXPECT_EQUAL(21u, (test.tmd().begin() + 1)->getPosition()); - } else { - EXPECT_EQUAL(0, std::distance(test.tmd().begin(), test.tmd().end())); - } - if (unpack_interleaved_features) { - EXPECT_EQUAL(2u, test.tmd().getNumOccs()); - EXPECT_EQUAL(30u, test.tmd().getFieldLength()); - } else { - EXPECT_EQUAL(0u, test.tmd().getNumOccs()); - EXPECT_EQUAL(0u, test.tmd().getFieldLength()); +TEST("requireThatPhrasesAreUnpacked") { + for (bool useBlueprint: {false, true}) { + for (bool unpack_normal_features: {false, true}) { + for (bool unpack_interleaved_features: {false, true}) { + PhraseSearchTest test; + test.addTerm("foo", FakeResult() + .doc(doc_match).pos(1).pos(11).pos(21).field_length(30).num_occs(3)); + test.addTerm("bar", FakeResult() + .doc(doc_match).pos(2).pos(16).pos(22).field_length(30).num_occs(3)); + test.writable_term_field_match_data().setNeedNormalFeatures(unpack_normal_features); + test.writable_term_field_match_data().setNeedInterleavedFeatures(unpack_interleaved_features); + test.fetchPostings(useBlueprint); + unique_ptr<SearchIterator> search(test.createSearch(useBlueprint)); + EXPECT_TRUE(search->seek(doc_match)); + search->unpack(doc_match); + + EXPECT_EQUAL(doc_match, test.tmd().getDocId()); + if (unpack_normal_features) { + EXPECT_EQUAL(2, std::distance(test.tmd().begin(), test.tmd().end())); + EXPECT_EQUAL(1u, test.tmd().begin()->getPosition()); + EXPECT_EQUAL(21u, (test.tmd().begin() + 1)->getPosition()); + } else { + EXPECT_EQUAL(0, std::distance(test.tmd().begin(), test.tmd().end())); + } + if (unpack_interleaved_features) { + EXPECT_EQUAL(2u, test.tmd().getNumOccs()); + EXPECT_EQUAL(30u, test.tmd().getFieldLength()); + } else { + EXPECT_EQUAL(0u, test.tmd().getNumOccs()); + EXPECT_EQUAL(0u, test.tmd().getFieldLength()); + } + } + } } } -void Test::requireThatTermsCanBeEvaluatedInPriorityOrder() { +TEST("requireThatTermsCanBeEvaluatedInPriorityOrder") { vector<uint32_t> order; order.push_back(2); order.push_back(0); @@ -280,9 +252,7 @@ void Test::requireThatTermsCanBeEvaluatedInPriorityOrder() { EXPECT_TRUE(!search->seek(doc_no_match)); } -void -Test::requireThatBlueprintExposesFieldWithEstimate() -{ +TEST("requireThatBlueprintExposesFieldWithEstimate") { FieldSpec f("foo", 1, 1); SimplePhraseBlueprint phrase(f, false); ASSERT_TRUE(phrase.getState().numFields() == 1); @@ -305,9 +275,7 @@ Test::requireThatBlueprintExposesFieldWithEstimate() EXPECT_EQUAL(5u, phrase.getState().estimate().estHits); } -void -Test::requireThatBlueprintForcesPositionDataOnChildren() -{ +TEST("requireThatBlueprintForcesPositionDataOnChildren") { FieldSpec f("foo", 1, 1, true); SimplePhraseBlueprint phrase(f, false); EXPECT_TRUE(f.isFilter()); @@ -316,4 +284,4 @@ Test::requireThatBlueprintForcesPositionDataOnChildren() } // namespace -TEST_APPHOOK(Test); +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/queryeval/sourceblender/CMakeLists.txt b/searchlib/src/tests/queryeval/sourceblender/CMakeLists.txt index fb574a4ed4b..a571030630b 100644 --- a/searchlib/src/tests/queryeval/sourceblender/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/sourceblender/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_sourceblender_test_app TEST SOURCES sourceblender_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_sourceblender_test_app COMMAND searchlib_sourceblender_test_app) diff --git a/searchlib/src/tests/queryeval/sourceblender/sourceblender_test.cpp b/searchlib/src/tests/queryeval/sourceblender/sourceblender_test.cpp index b84cb02a357..b2a1f6a645a 100644 --- a/searchlib/src/tests/queryeval/sourceblender/sourceblender_test.cpp +++ b/searchlib/src/tests/queryeval/sourceblender/sourceblender_test.cpp @@ -7,15 +7,14 @@ #include <vespa/searchlib/queryeval/leaf_blueprints.h> #define ENABLE_GTEST_MIGRATION #include <vespa/searchlib/test/searchiteratorverifier.h> -#include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/attribute/fixedsourceselector.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/vespalib/gtest/gtest.h> using namespace search::queryeval; -using namespace search::fef; using namespace search; using std::make_unique; +using search::fef::MatchData; /** * Proxy search used to verify unpack pattern @@ -27,24 +26,24 @@ private: SimpleResult _unpacked; protected: - virtual void doSeek(uint32_t docid) override { + void doSeek(uint32_t docid) override { _search->seek(docid); setDocId(_search->getDocId()); } - virtual void doUnpack(uint32_t docid) override { + void doUnpack(uint32_t docid) override { _unpacked.addHit(docid); _search->unpack(docid); } public: - UnpackChecker(SearchIterator *search) : _search(search), _unpacked() {} + explicit UnpackChecker(SearchIterator *search) : _search(search), _unpacked() {} const SimpleResult &getUnpacked() const { return _unpacked; } }; class MySelector : public search::FixedSourceSelector { public: - MySelector(int defaultSource) : search::FixedSourceSelector(defaultSource, "fs") { } + explicit MySelector(int defaultSource) : search::FixedSourceSelector(defaultSource, "fs") { } MySelector & set(Source s, uint32_t docId) { setSource(s, docId); return *this; @@ -65,12 +64,12 @@ TEST(SourceBlenderTest, test_strictness) a.addHit(2).addHit(5).addHit(6).addHit(8); b.addHit(3).addHit(5).addHit(6).addHit(7); - MySelector *sel = new MySelector(5); + auto *sel = new MySelector(5); sel->set(2, 1).set(3, 2).set(5, 2).set(7, 1); - SourceBlenderBlueprint *blend_b = new SourceBlenderBlueprint(*sel); - Blueprint::UP a_b(new SimpleBlueprint(a)); - Blueprint::UP b_b(new SimpleBlueprint(b)); + auto *blend_b = new SourceBlenderBlueprint(*sel); + auto a_b = std::make_unique<SimpleBlueprint>(a); + auto b_b = std::make_unique<SimpleBlueprint>(b); a_b->setSourceId(1); b_b->setSourceId(2); blend_b->addChild(std::move(a_b)); @@ -111,16 +110,16 @@ TEST(SourceBlenderTest, test_full_sourceblender_search) c.addHit(4).addHit(11).addHit(21).addHit(32); // these are all handed over to the blender - UnpackChecker *ua = new UnpackChecker(new SimpleSearch(a)); - UnpackChecker *ub = new UnpackChecker(new SimpleSearch(b)); - UnpackChecker *uc = new UnpackChecker(new SimpleSearch(c)); + auto *ua = new UnpackChecker(new SimpleSearch(a)); + auto *ub = new UnpackChecker(new SimpleSearch(b)); + auto *uc = new UnpackChecker(new SimpleSearch(c)); auto sel = make_unique<MySelector>(5); sel->set(2, 1).set(3, 2).set(11, 2).set(21, 3).set(34, 1); SourceBlenderSearch::Children abc; - abc.push_back(SourceBlenderSearch::Child(ua, 1)); - abc.push_back(SourceBlenderSearch::Child(ub, 2)); - abc.push_back(SourceBlenderSearch::Child(uc, 3)); + abc.emplace_back(ua, 1); + abc.emplace_back(ub, 2); + abc.emplace_back(uc, 3); SearchIterator::UP blend(SourceBlenderSearch::create(sel->createIterator(), abc, true)); SimpleResult result; @@ -149,7 +148,7 @@ using search::test::SearchIteratorVerifier; class Verifier : public SearchIteratorVerifier { public: Verifier(); - ~Verifier(); + ~Verifier() override; SearchIterator::UP create(bool strict) const override { return SearchIterator::UP(SourceBlenderSearch::create(_selector.createIterator(), createChildren(strict), @@ -178,7 +177,7 @@ Verifier::Verifier() : _indexes[indexId].push_back(docId); } } -Verifier::~Verifier() {} +Verifier::~Verifier() = default; TEST(SourceBlenderTest, test_that_source_blender_iterator_adheres_to_search_terator_requirements) { diff --git a/searchlib/src/tests/queryeval/sparse_vector_benchmark/CMakeLists.txt b/searchlib/src/tests/queryeval/sparse_vector_benchmark/CMakeLists.txt index 8150cbfcc36..0f0f1709a69 100644 --- a/searchlib/src/tests/queryeval/sparse_vector_benchmark/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/sparse_vector_benchmark/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_sparse_vector_benchmark_test_app SOURCES sparse_vector_benchmark_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_sparse_vector_benchmark_test_app COMMAND searchlib_sparse_vector_benchmark_test_app BENCHMARK) diff --git a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp index 94ecd8fa539..329118fdf81 100644 --- a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp @@ -13,6 +13,7 @@ #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/wand/weak_and_search.h> #include <vespa/searchlib/queryeval/weighted_set_term_search.h> +#include <vespa/searchlib/queryeval/wand/weak_and_heap.h> #include <vespa/vespalib/util/box.h> #include <vespa/vespalib/util/stringfmt.h> @@ -135,7 +136,7 @@ constexpr vespalib::duration max_time = 1000s; //----------------------------------------------------------------------------- struct ChildFactory { - ChildFactory() {} + ChildFactory() = default; virtual std::string name() const = 0; virtual SearchIterator::UP createChild(uint32_t idx, uint32_t limit) const = 0; virtual ~ChildFactory() = default; @@ -190,8 +191,9 @@ struct ModSearchFactory : ChildFactory { //----------------------------------------------------------------------------- struct VespaWandFactory : SparseVectorFactory { + mutable SharedWeakAndPriorityQueue _scores; uint32_t n; - explicit VespaWandFactory(uint32_t n_in) noexcept : n(n_in) {} + explicit VespaWandFactory(uint32_t n_in) : _scores(n_in), n(n_in) {} std::string name() const override { return vespalib::make_string("VespaWand(%u)", n); } @@ -200,7 +202,7 @@ struct VespaWandFactory : SparseVectorFactory { for (size_t i = 0; i < childCnt; ++i) { terms.emplace_back(childFactory.createChild(i, limit), default_weight, limit / (i + 1)); } - return WeakAndSearch::create(terms, n, true); + return WeakAndSearch::create(terms, wand::MatchParams(_scores), n, true, false); } }; diff --git a/searchlib/src/tests/queryeval/termwise_eval/CMakeLists.txt b/searchlib/src/tests/queryeval/termwise_eval/CMakeLists.txt index 7e30265dc19..e112397ec73 100644 --- a/searchlib/src/tests/queryeval/termwise_eval/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/termwise_eval/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_termwise_eval_test_app TEST SOURCES termwise_eval_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_termwise_eval_test_app COMMAND searchlib_termwise_eval_test_app) diff --git a/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp b/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp index 310c6d628e3..4d84dabf834 100644 --- a/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp +++ b/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp @@ -465,7 +465,7 @@ TEST(TermwiseEvalTest, require_that_termwise_evaluation_can_be_multi_level_but_n child->addChild(UP(new MyBlueprint({3}, true, 3))); my_or.addChild(std::move(child)); for (bool strict: {true, false}) { - my_or.basic_plan(strict, 100); + my_or.null_plan(strict, 100); EXPECT_EQ(my_or.createSearch(*md)->asString(), make_termwise(OR({ TERM({1}, strict), ORz({ TERM({2}, strict), TERM({3}, strict) }, strict) }, diff --git a/searchlib/src/tests/queryeval/weak_and/CMakeLists.txt b/searchlib/src/tests/queryeval/weak_and/CMakeLists.txt index dfedbbdabb6..7b80729c95e 100644 --- a/searchlib/src/tests/queryeval/weak_and/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/weak_and/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_weak_and_test_app TEST SOURCES weak_and_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_weak_and_test_app COMMAND searchlib_weak_and_test_app) @@ -11,20 +11,20 @@ vespa_add_executable(searchlib_weak_and_test_expensive_app TEST SOURCES weak_and_test_expensive.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_weak_and_test_expensive_app NO_VALGRIND COMMAND searchlib_weak_and_test_expensive_app) vespa_add_executable(searchlib_weak_and_bench_app SOURCES weak_and_bench.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_weak_and_bench_app COMMAND searchlib_weak_and_bench_app BENCHMARK) vespa_add_executable(searchlib_parallel_weak_and_bench_app SOURCES parallel_weak_and_bench.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_parallel_weak_and_bench_app COMMAND searchlib_parallel_weak_and_bench_app BENCHMARK) diff --git a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp index 55dc3868ed4..f4de5bb0086 100644 --- a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp +++ b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp @@ -42,7 +42,7 @@ struct Stats { void unpack() noexcept { ++unpackCnt; } - void print() { + void print() const { fprintf(stderr, "Stats: hits=%zu, seeks=%zu, unpacks=%zu, skippedDocs=%zu, skippedHits=%zu\n", hitCnt, seekCnt, unpackCnt, skippedDocs, skippedHits); } @@ -100,36 +100,48 @@ struct WandFactory { }; struct VespaWandFactory : WandFactory { + mutable SharedWeakAndPriorityQueue _scores; uint32_t n; - explicit VespaWandFactory(uint32_t n_in) noexcept : n(n_in) {} + explicit VespaWandFactory(uint32_t n_in) noexcept + : _scores(n_in), + n(n_in) + {} ~VespaWandFactory() override; std::string name() const override { return make_string("VESPA WAND (n=%u)", n); } SearchIterator::UP create(const wand::Terms &terms) override { - return WeakAndSearch::create(terms, n, true); + return WeakAndSearch::create(terms, wand::MatchParams(_scores, 1, 1), n, true, false); } }; VespaWandFactory::~VespaWandFactory() = default; struct VespaArrayWandFactory : WandFactory { + mutable SharedWeakAndPriorityQueue _scores; uint32_t n; - explicit VespaArrayWandFactory(uint32_t n_in) noexcept : n(n_in) {} + explicit VespaArrayWandFactory(uint32_t n_in) + : _scores(n_in), + n(n_in) + {} ~VespaArrayWandFactory() override; std::string name() const override { return make_string("VESPA ARRAY WAND (n=%u)", n); } SearchIterator::UP create(const wand::Terms &terms) override { - return WeakAndSearch::createArrayWand(terms, wand::TermFrequencyScorer(), n, true); + return WeakAndSearch::createArrayWand(terms, wand::MatchParams(_scores, 1, 1), wand::TermFrequencyScorer(), n, true, false); } }; VespaArrayWandFactory::~VespaArrayWandFactory() = default; struct VespaHeapWandFactory : WandFactory { + mutable SharedWeakAndPriorityQueue _scores; uint32_t n; - explicit VespaHeapWandFactory(uint32_t n_in) noexcept : n(n_in) {} + explicit VespaHeapWandFactory(uint32_t n_in) + : _scores(n_in), + n(n_in) + {} ~VespaHeapWandFactory() override; std::string name() const override { return make_string("VESPA HEAP WAND (n=%u)", n); } SearchIterator::UP create(const wand::Terms &terms) override { - return WeakAndSearch::createHeapWand(terms, wand::TermFrequencyScorer(), n, true); + return WeakAndSearch::createHeapWand(terms, wand::MatchParams(_scores, 1, 1), wand::TermFrequencyScorer(), n, true, false); } }; @@ -144,20 +156,20 @@ struct VespaParallelWandFactory : public WandFactory { SearchIterator::UP create(const wand::Terms &terms) override { return ParallelWeakAndSearch::create(terms, PWMatchParams(scores, 0, 1, 1), - PWRankParams(rootMatchData, {}), true); + PWRankParams(rootMatchData, {}), true, false); } }; VespaParallelWandFactory::~VespaParallelWandFactory() = default; struct VespaParallelArrayWandFactory : public VespaParallelWandFactory { - VespaParallelArrayWandFactory(uint32_t n) noexcept : VespaParallelWandFactory(n) {} + explicit VespaParallelArrayWandFactory(uint32_t n) noexcept : VespaParallelWandFactory(n) {} ~VespaParallelArrayWandFactory() override; std::string name() const override { return make_string("VESPA ARRAY PWAND (n=%u)", scores.getScoresToTrack()); } SearchIterator::UP create(const wand::Terms &terms) override { return ParallelWeakAndSearch::createArrayWand(terms, PWMatchParams(scores, 0, 1, 1), - PWRankParams(rootMatchData, {}), true); + PWRankParams(rootMatchData, {}), true, false); } }; @@ -170,7 +182,7 @@ struct VespaParallelHeapWandFactory : public VespaParallelWandFactory { SearchIterator::UP create(const wand::Terms &terms) override { return ParallelWeakAndSearch::createHeapWand(terms, PWMatchParams(scores, 0, 1, 1), - PWRankParams(rootMatchData, {}), true); + PWRankParams(rootMatchData, {}), true, false); } }; diff --git a/searchlib/src/tests/queryeval/weak_and/weak_and_test.cpp b/searchlib/src/tests/queryeval/weak_and/weak_and_test.cpp index 689f9f085d0..3fc31e4456d 100644 --- a/searchlib/src/tests/queryeval/weak_and/weak_and_test.cpp +++ b/searchlib/src/tests/queryeval/weak_and/weak_and_test.cpp @@ -1,8 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/wand/weak_and_search.h> +#include <vespa/searchlib/queryeval/wand/weak_and_heap.h> +#include <vespa/searchlib/queryeval/matching_phase.h> #include <vespa/searchlib/queryeval/simpleresult.h> -#include <vespa/searchlib/queryeval/simplesearch.h> #include <vespa/searchlib/queryeval/test/eagerchild.h> #include <vespa/searchlib/queryeval/test/leafspec.h> #include <vespa/searchlib/queryeval/test/wandspec.h> @@ -20,18 +20,38 @@ namespace { struct MyWandSpec : public WandSpec { + SharedWeakAndPriorityQueue scores; uint32_t n; - - MyWandSpec(uint32_t n_) : WandSpec(), n(n_) {} + MatchingPhase matching_phase; + + explicit MyWandSpec(uint32_t n_in) + : WandSpec(), + scores(n_in), + n(n_in), + matching_phase(MatchingPhase::FIRST_PHASE) + {} SearchIterator *create() { - return new TrackedSearch("WAND", getHistory(), WeakAndSearch::create(getTerms(), n, true)); + bool readonly_scores_heap = (matching_phase != MatchingPhase::FIRST_PHASE); + return new TrackedSearch("WAND", getHistory(), + WeakAndSearch::create(getTerms(), wand::MatchParams(scores, 1, 1), n, true, readonly_scores_heap)); } + void set_second_phase() { matching_phase = MatchingPhase::SECOND_PHASE; } }; struct SimpleWandFixture { MyWandSpec spec; SimpleResult hits; - SimpleWandFixture() : spec(2), hits() { + SimpleWandFixture() + : SimpleWandFixture(false) + { + } + explicit SimpleWandFixture(bool second_phase) + : spec(2), + hits() + { + if (second_phase) { + spec.set_second_phase(); + } spec.leaf(LeafSpec("foo").doc(1).doc(2).doc(3).doc(4).doc(5).doc(6)); spec.leaf(LeafSpec("bar").doc(1).doc(3).doc(5)); SearchIterator::UP search(spec.create()); @@ -69,10 +89,16 @@ struct WeightOrder { TEST(WeakAndTest, require_that_wand_prunes_bad_hits_after_enough_good_ones_are_obtained) { - SimpleWandFixture f; + SimpleWandFixture f; // First phase EXPECT_EQ(SimpleResult().addHit(1).addHit(2).addHit(3).addHit(5), f.hits); } +TEST(WeakAndTest, require_that_wand_does_not_prune_hits_in_later_matching_phases) +{ + SimpleWandFixture f(true); // Second phase + EXPECT_EQ(SimpleResult().addHit(1).addHit(2).addHit(3).addHit(4).addHit(5).addHit(6), f.hits); +} + TEST(WeakAndTest, require_that_wand_uses_subsearches_as_expected) { SimpleWandFixture f; @@ -104,7 +130,8 @@ TEST(WeakAndTest, require_that_initial_docid_for_subsearches_are_taken_into_acco wand::Terms terms; terms.push_back(wand::Term(new TrackedSearch("foo", history, new EagerChild(search::endDocId)), 100, 1)); terms.push_back(wand::Term(new TrackedSearch("bar", history, new EagerChild(10)), 100, 2)); - SearchIterator::UP search(new TrackedSearch("WAND", history, WeakAndSearch::create(terms, 2, true))); + SharedWeakAndPriorityQueue scores(2); + auto search = std::make_unique<TrackedSearch>("WAND", history, WeakAndSearch::create(terms, wand::MatchParams(scores), 2, true, false)); SimpleResult hits; hits.search(*search); EXPECT_EQ(SimpleResult().addHit(10), hits); @@ -114,17 +141,26 @@ TEST(WeakAndTest, require_that_initial_docid_for_subsearches_are_taken_into_acco } class IteratorChildrenVerifier : public search::test::IteratorChildrenVerifier { +public: + IteratorChildrenVerifier(); + ~IteratorChildrenVerifier() override; private: + mutable std::vector<std::unique_ptr<SharedWeakAndPriorityQueue>> _scores; SearchIterator::UP create(bool strict) const override { wand::Terms terms; for (size_t i = 0; i < _num_children; ++i) { terms.emplace_back(createIterator(_split_lists[i], strict).release(), 100, _split_lists[i].size()); } - return SearchIterator::UP(WeakAndSearch::create(terms, -1, strict)); + static constexpr size_t LARGE_ENOUGH_HEAP_FOR_ALL = 10000; + _scores.push_back(std::make_unique<SharedWeakAndPriorityQueue>(LARGE_ENOUGH_HEAP_FOR_ALL)); + return WeakAndSearch::create(terms, wand::MatchParams(*_scores.back(), 1, 1), -1, strict, false); } }; +IteratorChildrenVerifier::IteratorChildrenVerifier() : _scores() {} +IteratorChildrenVerifier::~IteratorChildrenVerifier() = default; + TEST(WeakAndTest, verify_search_iterator_conformance) { IteratorChildrenVerifier verifier; diff --git a/searchlib/src/tests/queryeval/weak_and/weak_and_test_expensive.cpp b/searchlib/src/tests/queryeval/weak_and/weak_and_test_expensive.cpp index 54bf1e92037..0573404a3b4 100644 --- a/searchlib/src/tests/queryeval/weak_and/weak_and_test_expensive.cpp +++ b/searchlib/src/tests/queryeval/weak_and/weak_and_test_expensive.cpp @@ -16,15 +16,16 @@ void checkWandHits(WandFactory &vespa, WandFactory &rise, uint32_t step, uint32_ s1->initFullRange(); SearchIterator::UP s2 = riseSetup.create(); s2->initFullRange(); - ASSERT_TRUE(dynamic_cast<WeakAndType*>(s1.get()) != 0); - ASSERT_TRUE(dynamic_cast<WeakAndType*>(s2.get()) == 0); - ASSERT_TRUE(dynamic_cast<RiseType*>(s2.get()) != 0); - ASSERT_TRUE(dynamic_cast<RiseType*>(s1.get()) == 0); + ASSERT_TRUE(dynamic_cast<WeakAndType*>(s1.get()) != nullptr); + ASSERT_TRUE(dynamic_cast<WeakAndType*>(s2.get()) == nullptr); + ASSERT_TRUE(dynamic_cast<RiseType*>(s2.get()) != nullptr); + ASSERT_TRUE(dynamic_cast<RiseType*>(s1.get()) == nullptr); s1->seek(1); s2->seek(1); while (!s1->isAtEnd() && !s2->isAtEnd()) { + if (s1->getDocId() != s2->getDocId()) assert(true); ASSERT_EQUAL(s1->getDocId(), s2->getDocId()); if ((filter == 0) || ((s1->getDocId() % filter) != 0)) { s1->unpack(s1->getDocId()); diff --git a/searchlib/src/tests/queryeval/weak_and_heap/CMakeLists.txt b/searchlib/src/tests/queryeval/weak_and_heap/CMakeLists.txt index 1dc51921788..fc3c27a4a84 100644 --- a/searchlib/src/tests/queryeval/weak_and_heap/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/weak_and_heap/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_weak_and_heap_test_app TEST SOURCES weak_and_heap_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_weak_and_heap_test_app COMMAND searchlib_weak_and_heap_test_app) diff --git a/searchlib/src/tests/queryeval/weak_and_scorers/CMakeLists.txt b/searchlib/src/tests/queryeval/weak_and_scorers/CMakeLists.txt index 0a21b2c0148..59c0ff66161 100644 --- a/searchlib/src/tests/queryeval/weak_and_scorers/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/weak_and_scorers/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_weak_and_scorers_test_app TEST SOURCES weak_and_scorers_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_weak_and_scorers_test_app COMMAND searchlib_weak_and_scorers_test_app) diff --git a/searchlib/src/tests/queryeval/weighted_set_term/CMakeLists.txt b/searchlib/src/tests/queryeval/weighted_set_term/CMakeLists.txt index 6f7d18df9c4..c7d643225c4 100644 --- a/searchlib/src/tests/queryeval/weighted_set_term/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/weighted_set_term/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_weighted_set_term_test_app TEST SOURCES weighted_set_term_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test ) vespa_add_test(NAME searchlib_weighted_set_term_test_app COMMAND searchlib_weighted_set_term_test_app) diff --git a/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt b/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt index 58c50098371..cef37110898 100644 --- a/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_wrappers_test_app TEST SOURCES wrappers_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt index 3af65cb45af..2172163c0a4 100644 --- a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt +++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_intrinsic_blueprint_adapter_test_app TEST SOURCES intrinsic_blueprint_adapter_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_intrinsic_blueprint_adapter_test_app COMMAND searchlib_intrinsic_blueprint_adapter_test_app) diff --git a/searchlib/src/tests/ranksetup/CMakeLists.txt b/searchlib/src/tests/ranksetup/CMakeLists.txt index d5eb349a6c7..31b5e323699 100644 --- a/searchlib/src/tests/ranksetup/CMakeLists.txt +++ b/searchlib/src/tests/ranksetup/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(searchlib_ranksetup_test_app TEST SOURCES ranksetup_test.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_ranksetup_test_app COMMAND searchlib_ranksetup_test_app) diff --git a/searchlib/src/tests/ranksetup/ranksetup_test.cpp b/searchlib/src/tests/ranksetup/ranksetup_test.cpp index 348326c3936..a5e7fed5685 100644 --- a/searchlib/src/tests/ranksetup/ranksetup_test.cpp +++ b/searchlib/src/tests/ranksetup/ranksetup_test.cpp @@ -1,7 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/common/feature.h> #include <vespa/searchlib/attribute/attributeguard.h> @@ -36,6 +34,8 @@ #include <vespa/searchlib/fef/test/plugin/sum.h> #include <vespa/searchlib/fef/test/plugin/cfgvalue.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/stringfmt.h> #include <iostream> using namespace search::fef; @@ -220,9 +220,9 @@ FeatureDumper::dump() //----------------------------------------------------------------------------- // RankSetupTest //----------------------------------------------------------------------------- -class RankSetupTest : public vespalib::TestApp +class RankSetupTest : public ::testing::Test { -private: +protected: BlueprintFactory _factory; search::AttributeManager _manager; IndexEnvironment _indexEnv; @@ -230,14 +230,6 @@ private: RankEnvironment _rankEnv; DumpFeatureVisitor _visitor; - void testValueBlueprint(); - void testDoubleBlueprint(); - void testSumBlueprint(); - void testStaticRankBlueprint(); - void testChainBlueprint(); - void testCfgValueBlueprint(); - void testCompilation(); - void testRankSetup(); bool testExecution(const vespalib::string & initRank, feature_t initScore, const vespalib::string & finalRank = "", feature_t finalScore = 0.0f, uint32_t docId = 1); bool testExecution(const RankEnvironment &rankEnv, @@ -249,15 +241,52 @@ private: void checkFeatures(std::map<vespalib::string, feature_t> &exp, std::map<vespalib::string, feature_t> &actual); void testFeatureNormalization(); -public: RankSetupTest(); - ~RankSetupTest(); - int Main() override; + ~RankSetupTest() override; }; +RankSetupTest::RankSetupTest() + : _factory(), + _manager(), + _indexEnv(), + _queryEnv(), + _rankEnv(_factory, _indexEnv, _queryEnv), + _visitor() +{ + // register blueprints + setup_fef_test_plugin(_factory); + _factory.addPrototype(Blueprint::SP(new ValueBlueprint())); + _factory.addPrototype(Blueprint::SP(new RankingExpressionBlueprint())); + _factory.addPrototype(std::make_shared<SecondPhaseBlueprint>()); -void -RankSetupTest::testValueBlueprint() + // setup an original attribute manager with two attributes + search::attribute::Config cfg(search::attribute::BasicType::INT32, + search::attribute::CollectionType::SINGLE); + search::AttributeVector::SP av1 = + search::AttributeFactory::createAttribute("staticrank1", cfg); + search::AttributeVector::SP av2 = + search::AttributeFactory::createAttribute("staticrank2", cfg); + av1->addDocs(5); + av2->addDocs(5); + for (uint32_t i = 0; i < 5; ++i) { + (static_cast<search::IntegerAttribute *>(av1.get()))->update(i, i + 100); + (static_cast<search::IntegerAttribute *>(av2.get()))->update(i, i + 200); + } + av1->commit(); + av2->commit(); + _manager.add(av1); + _manager.add(av2); + + // set the index environment + _queryEnv.setIndexEnv(&_indexEnv); + + // set the manager + _queryEnv.overrideAttributeManager(&_manager); +} + +RankSetupTest::~RankSetupTest() = default; + +TEST_F(RankSetupTest, value_blueprint) { ValueBlueprint prototype; prototype.visitDumpFeatures(_indexEnv, _visitor); @@ -265,22 +294,22 @@ RankSetupTest::testValueBlueprint() Blueprint::UP bp = prototype.createInstance(); DummyDependencyHandler deps(*bp); bp->setName("value"); - EXPECT_EQUAL(bp->getName(), "value"); + EXPECT_EQ(bp->getName(), "value"); std::vector<vespalib::string> params; params.push_back("5.5"); params.push_back("10.5"); EXPECT_TRUE(bp->setup(_indexEnv, params)); - EXPECT_EQUAL(deps.input.size(), 0u); - EXPECT_EQUAL(deps.output.size(), 2u); - EXPECT_EQUAL(deps.output[0], "0"); - EXPECT_EQUAL(deps.output[1], "1"); + EXPECT_EQ(deps.input.size(), 0u); + EXPECT_EQ(deps.output.size(), 2u); + EXPECT_EQ(deps.output[0], "0"); + EXPECT_EQ(deps.output[1], "1"); vespalib::Stash stash; FeatureExecutor &fe = bp->createExecutor(_queryEnv, stash); ValueExecutor * vfe = static_cast<ValueExecutor *>(&fe); - EXPECT_EQUAL(vfe->getValues().size(), 2u); - EXPECT_EQUAL(vfe->getValues()[0], 5.5f); - EXPECT_EQUAL(vfe->getValues()[1], 10.5f); + EXPECT_EQ(vfe->getValues().size(), 2u); + EXPECT_EQ(vfe->getValues()[0], 5.5f); + EXPECT_EQ(vfe->getValues()[1], 10.5f); } { // invalid params Blueprint::UP bp = prototype.createInstance(); @@ -290,8 +319,7 @@ RankSetupTest::testValueBlueprint() } } -void -RankSetupTest::testDoubleBlueprint() +TEST_F(RankSetupTest, double_blueprint) { DoubleBlueprint prototype; prototype.visitDumpFeatures(_indexEnv, _visitor); @@ -302,17 +330,16 @@ RankSetupTest::testDoubleBlueprint() params.push_back("value(5.5).0"); params.push_back("value(10.5).0"); EXPECT_TRUE(bp->setup(_indexEnv, params)); - EXPECT_EQUAL(deps.input.size(), 2u); - EXPECT_EQUAL(deps.input[0], "value(5.5).0"); - EXPECT_EQUAL(deps.input[1], "value(10.5).0"); - EXPECT_EQUAL(deps.output.size(), 2u); - EXPECT_EQUAL(deps.output[0], "0"); - EXPECT_EQUAL(deps.output[1], "1"); + EXPECT_EQ(deps.input.size(), 2u); + EXPECT_EQ(deps.input[0], "value(5.5).0"); + EXPECT_EQ(deps.input[1], "value(10.5).0"); + EXPECT_EQ(deps.output.size(), 2u); + EXPECT_EQ(deps.output[0], "0"); + EXPECT_EQ(deps.output[1], "1"); } } -void -RankSetupTest::testSumBlueprint() +TEST_F(RankSetupTest, sum_blueprint) { SumBlueprint prototype; prototype.visitDumpFeatures(_indexEnv, _visitor); @@ -323,16 +350,15 @@ RankSetupTest::testSumBlueprint() params.push_back("value(5.5, 10.5).0"); params.push_back("value(5.5, 10.5).1"); EXPECT_TRUE(bp->setup(_indexEnv, params)); - EXPECT_EQUAL(deps.input.size(), 2u); - EXPECT_EQUAL(deps.input[0], "value(5.5, 10.5).0"); - EXPECT_EQUAL(deps.input[1], "value(5.5, 10.5).1"); - EXPECT_EQUAL(deps.output.size(), 1u); - EXPECT_EQUAL(deps.output[0], "out"); + EXPECT_EQ(deps.input.size(), 2u); + EXPECT_EQ(deps.input[0], "value(5.5, 10.5).0"); + EXPECT_EQ(deps.input[1], "value(5.5, 10.5).1"); + EXPECT_EQ(deps.output.size(), 1u); + EXPECT_EQ(deps.output[0], "out"); } } -void -RankSetupTest::testStaticRankBlueprint() +TEST_F(RankSetupTest, static_rank_blueprint) { StaticRankBlueprint prototype; { // basic test @@ -341,9 +367,9 @@ RankSetupTest::testStaticRankBlueprint() std::vector<vespalib::string> params; params.push_back("sr1"); EXPECT_TRUE(bp->setup(_indexEnv, params)); - EXPECT_EQUAL(deps.input.size(), 0u); - EXPECT_EQUAL(deps.output.size(), 1u); - EXPECT_EQUAL(deps.output[0], "out"); + EXPECT_EQ(deps.input.size(), 0u); + EXPECT_EQ(deps.output.size(), 1u); + EXPECT_EQ(deps.output[0], "out"); } { // invalid params Blueprint::UP bp = prototype.createInstance(); @@ -356,8 +382,7 @@ RankSetupTest::testStaticRankBlueprint() } } -void -RankSetupTest::testChainBlueprint() +TEST_F(RankSetupTest, chain_blueprint) { ChainBlueprint prototype; { // chaining @@ -368,8 +393,8 @@ RankSetupTest::testChainBlueprint() params.push_back("2"); params.push_back("4"); EXPECT_TRUE(bp->setup(_indexEnv, params)); - EXPECT_EQUAL(deps.input.size(), 1u); - EXPECT_EQUAL(deps.input[0], "chain(basic,1,4)"); + EXPECT_EQ(deps.input.size(), 1u); + EXPECT_EQ(deps.input[0], "chain(basic,1,4)"); } { // leaf node Blueprint::UP bp = prototype.createInstance(); @@ -379,8 +404,8 @@ RankSetupTest::testChainBlueprint() params.push_back("1"); params.push_back("4"); EXPECT_TRUE(bp->setup(_indexEnv, params)); - EXPECT_EQUAL(deps.input.size(), 1u); - EXPECT_EQUAL(deps.input[0], "value(4)"); + EXPECT_EQ(deps.input.size(), 1u); + EXPECT_EQ(deps.input[0], "value(4)"); } { // cycle Blueprint::UP bp = prototype.createInstance(); @@ -390,8 +415,8 @@ RankSetupTest::testChainBlueprint() params.push_back("1"); params.push_back("4"); EXPECT_TRUE(bp->setup(_indexEnv, params)); - EXPECT_EQUAL(deps.input.size(), 1u); - EXPECT_EQUAL(deps.input[0], "chain(cycle,4,4)"); + EXPECT_EQ(deps.input.size(), 1u); + EXPECT_EQ(deps.input[0], "chain(cycle,4,4)"); } { // invalid params Blueprint::UP bp = prototype.createInstance(); @@ -405,8 +430,7 @@ RankSetupTest::testChainBlueprint() } } -void -RankSetupTest::testCfgValueBlueprint() +TEST_F(RankSetupTest, cfg_value_blueprint) { CfgValueBlueprint prototype; IndexEnvironment indexEnv; @@ -422,25 +446,23 @@ RankSetupTest::testCfgValueBlueprint() params.push_back("foo"); EXPECT_TRUE(bp->setup(indexEnv, params)); - EXPECT_EQUAL(deps.input.size(), 0u); - EXPECT_EQUAL(deps.output.size(), 3u); - EXPECT_EQUAL(deps.output[0], "0"); - EXPECT_EQUAL(deps.output[1], "1"); - EXPECT_EQUAL(deps.output[2], "2"); + EXPECT_EQ(deps.input.size(), 0u); + EXPECT_EQ(deps.output.size(), 3u); + EXPECT_EQ(deps.output[0], "0"); + EXPECT_EQ(deps.output[1], "1"); + EXPECT_EQ(deps.output[2], "2"); vespalib::Stash stash; FeatureExecutor &fe = bp->createExecutor(_queryEnv, stash); ValueExecutor *vfe = static_cast<ValueExecutor *>(&fe); - EXPECT_EQUAL(vfe->getValues().size(), 3u); - EXPECT_EQUAL(vfe->getValues()[0], 1.0f); - EXPECT_EQUAL(vfe->getValues()[1], 2.0f); - EXPECT_EQUAL(vfe->getValues()[2], 3.0f); + EXPECT_EQ(vfe->getValues().size(), 3u); + EXPECT_EQ(vfe->getValues()[0], 1.0f); + EXPECT_EQ(vfe->getValues()[1], 2.0f); + EXPECT_EQ(vfe->getValues()[2], 3.0f); } } - -void -RankSetupTest::testCompilation() +TEST_F(RankSetupTest, compilation) { { // unknown blueprint RankSetup rs(_factory, _indexEnv); @@ -499,7 +521,7 @@ RankSetupTest::testCompilation() } } -void RankSetupTest::testRankSetup() +TEST_F(RankSetupTest, rank_setup) { using namespace search::fef::indexproperties; IndexEnvironment env; @@ -525,7 +547,8 @@ void RankSetupTest::testRankSetup() env.getProperties().add(hitcollector::ArraySize::NAME, "60"); env.getProperties().add(hitcollector::EstimatePoint::NAME, "70"); env.getProperties().add(hitcollector::EstimateLimit::NAME, "80"); - env.getProperties().add(hitcollector::RankScoreDropLimit::NAME, "90.5"); + env.getProperties().add(hitcollector::FirstPhaseRankScoreDropLimit::NAME, "90.5"); + env.getProperties().add(hitcollector::SecondPhaseRankScoreDropLimit::NAME, "91.5"); env.getProperties().add(mutate::on_match::Attribute::NAME, "a"); env.getProperties().add(mutate::on_match::Operation::NAME, "+=3"); env.getProperties().add(mutate::on_first_phase::Attribute::NAME, "b"); @@ -542,44 +565,45 @@ void RankSetupTest::testRankSetup() RankSetup rs(_factory, env); EXPECT_FALSE(rs.has_match_features()); rs.configure(); - EXPECT_EQUAL(rs.getFirstPhaseRank(), vespalib::string("firstphase")); - EXPECT_EQUAL(rs.getSecondPhaseRank(), vespalib::string("secondphase")); + EXPECT_EQ(rs.getFirstPhaseRank(), vespalib::string("firstphase")); + EXPECT_EQ(rs.getSecondPhaseRank(), vespalib::string("secondphase")); EXPECT_TRUE(rs.has_match_features()); ASSERT_TRUE(rs.get_match_features().size() == 2); - EXPECT_EQUAL(rs.get_match_features()[0], vespalib::string("match_foo")); - EXPECT_EQUAL(rs.get_match_features()[1], vespalib::string("match_bar")); + EXPECT_EQ(rs.get_match_features()[0], vespalib::string("match_foo")); + EXPECT_EQ(rs.get_match_features()[1], vespalib::string("match_bar")); ASSERT_TRUE(rs.getDumpFeatures().size() == 2); - EXPECT_EQUAL(rs.getDumpFeatures()[0], vespalib::string("foo")); - EXPECT_EQUAL(rs.getDumpFeatures()[1], vespalib::string("bar")); - EXPECT_EQUAL(rs.getNumThreadsPerSearch(), 3u); - EXPECT_EQUAL(rs.getMinHitsPerThread(), 8u); - EXPECT_EQUAL(rs.getDegradationAttribute(), "mystaticrankattr"); - EXPECT_EQUAL(rs.isDegradationOrderAscending(), true); - EXPECT_EQUAL(rs.getDegradationMaxHits(), 12345u); - EXPECT_EQUAL(rs.getDegradationSamplePercentage(), 0.9); - EXPECT_EQUAL(rs.getDegradationMaxFilterCoverage(), 0.19); - EXPECT_EQUAL(rs.getDegradationPostFilterMultiplier(), 0.7); - EXPECT_EQUAL(rs.getDiversityAttribute(), "mycategoryattr"); - EXPECT_EQUAL(rs.getDiversityMinGroups(), 37u); - EXPECT_EQUAL(rs.getDiversityCutoffFactor(), 7.1); - EXPECT_EQUAL(rs.getDiversityCutoffStrategy(), "strict"); - EXPECT_EQUAL(rs.getHeapSize(), 50u); - EXPECT_EQUAL(rs.getArraySize(), 60u); - EXPECT_EQUAL(rs.getEstimatePoint(), 70u); - EXPECT_EQUAL(rs.getEstimateLimit(), 80u); - EXPECT_EQUAL(rs.getRankScoreDropLimit(), 90.5); - EXPECT_EQUAL(rs.getMutateOnMatch()._attribute, "a"); - EXPECT_EQUAL(rs.getMutateOnMatch()._operation, "+=3"); - EXPECT_EQUAL(rs.getMutateOnFirstPhase()._attribute, "b"); - EXPECT_EQUAL(rs.getMutateOnFirstPhase()._operation, "=3"); - EXPECT_EQUAL(rs.getMutateOnSecondPhase()._attribute, "b"); - EXPECT_EQUAL(rs.getMutateOnSecondPhase()._operation, "=7"); - EXPECT_EQUAL(rs.getMutateOnSummary()._attribute, "c"); - EXPECT_EQUAL(rs.getMutateOnSummary()._operation, "-=2"); - EXPECT_EQUAL(rs.get_global_filter_lower_limit(), 0.3); - EXPECT_EQUAL(rs.get_global_filter_upper_limit(), 0.7); - EXPECT_EQUAL(rs.get_target_hits_max_adjustment_factor(), 5.0); - EXPECT_EQUAL(rs.get_fuzzy_matching_algorithm(), vespalib::FuzzyMatchingAlgorithm::DfaImplicit); + EXPECT_EQ(rs.getDumpFeatures()[0], vespalib::string("foo")); + EXPECT_EQ(rs.getDumpFeatures()[1], vespalib::string("bar")); + EXPECT_EQ(rs.getNumThreadsPerSearch(), 3u); + EXPECT_EQ(rs.getMinHitsPerThread(), 8u); + EXPECT_EQ(rs.getDegradationAttribute(), "mystaticrankattr"); + EXPECT_EQ(rs.isDegradationOrderAscending(), true); + EXPECT_EQ(rs.getDegradationMaxHits(), 12345u); + EXPECT_EQ(rs.getDegradationSamplePercentage(), 0.9); + EXPECT_EQ(rs.getDegradationMaxFilterCoverage(), 0.19); + EXPECT_EQ(rs.getDegradationPostFilterMultiplier(), 0.7); + EXPECT_EQ(rs.getDiversityAttribute(), "mycategoryattr"); + EXPECT_EQ(rs.getDiversityMinGroups(), 37u); + EXPECT_EQ(rs.getDiversityCutoffFactor(), 7.1); + EXPECT_EQ(rs.getDiversityCutoffStrategy(), "strict"); + EXPECT_EQ(rs.getHeapSize(), 50u); + EXPECT_EQ(rs.getArraySize(), 60u); + EXPECT_EQ(rs.getEstimatePoint(), 70u); + EXPECT_EQ(rs.getEstimateLimit(), 80u); + EXPECT_EQ(std::optional<feature_t>(90.5), rs.get_first_phase_rank_score_drop_limit()); + EXPECT_EQ(std::optional<feature_t>(91.5), rs.get_second_phase_rank_score_drop_limit()); + EXPECT_EQ(rs.getMutateOnMatch()._attribute, "a"); + EXPECT_EQ(rs.getMutateOnMatch()._operation, "+=3"); + EXPECT_EQ(rs.getMutateOnFirstPhase()._attribute, "b"); + EXPECT_EQ(rs.getMutateOnFirstPhase()._operation, "=3"); + EXPECT_EQ(rs.getMutateOnSecondPhase()._attribute, "b"); + EXPECT_EQ(rs.getMutateOnSecondPhase()._operation, "=7"); + EXPECT_EQ(rs.getMutateOnSummary()._attribute, "c"); + EXPECT_EQ(rs.getMutateOnSummary()._operation, "-=2"); + EXPECT_EQ(rs.get_global_filter_lower_limit(), 0.3); + EXPECT_EQ(rs.get_global_filter_upper_limit(), 0.7); + EXPECT_EQ(rs.get_target_hits_max_adjustment_factor(), 5.0); + EXPECT_EQ(rs.get_fuzzy_matching_algorithm(), vespalib::FuzzyMatchingAlgorithm::DfaImplicit); } bool @@ -604,12 +628,11 @@ RankSetupTest::testExecution(const RankEnvironment &rankEnv, const vespalib::str } RankResult rs = re.execute(docId); ok = ok && (exp == rs); - EXPECT_EQUAL(exp, rs); + EXPECT_EQ(exp, rs); return ok; } -void -RankSetupTest::testExecution() +TEST_F(RankSetupTest, execution) { { // value executor vespalib::string v = FNB().baseName("value").parameter("5.5").parameter("10.5").buildName(); @@ -699,8 +722,7 @@ RankSetupTest::testExecution() } } -void -RankSetupTest::testFeatureDump() +TEST_F(RankSetupTest, feature_dump) { { FeatureDumper dumper(_rankEnv); @@ -722,7 +744,7 @@ RankSetupTest::testFeatureDump() parameter(FNB().baseName("double").parameter("value(8)").buildName()). parameter(FNB().baseName("double").parameter("value(32)").buildName()). buildName(), 80.0f); - EXPECT_EQUAL(exp, dumper.dump()); + EXPECT_EQ(exp, dumper.dump()); } { FeatureDumper dumper(_rankEnv); @@ -732,7 +754,7 @@ RankSetupTest::testFeatureDump() RankResult exp; exp.addScore("value(50)", 50.0f); exp.addScore("value(100)", 100.0f); - EXPECT_EQUAL(exp, dumper.dump()); + EXPECT_EQ(exp, dumper.dump()); } { FeatureDumper dumper(_rankEnv); @@ -740,7 +762,7 @@ RankSetupTest::testFeatureDump() EXPECT_TRUE(dumper.setup()); RankResult exp; exp.addScore(FNB().baseName("rankingExpression").parameter("if(4<2,3,4)").buildName(), 4.0f); - EXPECT_EQUAL(exp, dumper.dump()); + EXPECT_EQ(exp, dumper.dump()); } { @@ -749,7 +771,7 @@ RankSetupTest::testFeatureDump() EXPECT_TRUE(dumper.setup()); RankResult exp; exp.addScore(FNB().baseName("rankingExpression").parameter("if(mysum(value(12),value(10))>2,3,4)").buildName(), 3.0f); - EXPECT_EQUAL(exp, dumper.dump()); + EXPECT_EQ(exp, dumper.dump()); } { // dump features indicated by visitation IndexEnvironment indexEnv; @@ -767,7 +789,7 @@ RankSetupTest::testFeatureDump() RankResult exp; exp.addScore("test_cfgvalue(foo)", 1.0); exp.addScore("test_cfgvalue(bar)", 5.0); - EXPECT_EQUAL(exp, dumper.dump()); + EXPECT_EQ(exp, dumper.dump()); } { // ignore features indicated by visitation IndexEnvironment indexEnv; @@ -786,7 +808,7 @@ RankSetupTest::testFeatureDump() EXPECT_TRUE(dumper.setup()); RankResult exp; exp.addScore("test_cfgvalue(foo)", 1.0); - EXPECT_EQUAL(exp, dumper.dump()); + EXPECT_EQ(exp, dumper.dump()); } { // Dump secondPhase feature IndexEnvironment indexEnv; @@ -799,7 +821,7 @@ RankSetupTest::testFeatureDump() EXPECT_TRUE(dumper.setup()); RankResult exp; exp.addScore("secondPhase", 4.0); - EXPECT_EQUAL(exp, dumper.dump()); + EXPECT_EQ(exp, dumper.dump()); } } @@ -807,22 +829,19 @@ void RankSetupTest::checkFeatures(std::map<vespalib::string, feature_t> &exp, std::map<vespalib::string, feature_t> &actual) { using ITR = std::map<vespalib::string, feature_t>::const_iterator; - if (!EXPECT_EQUAL(exp.size(), actual.size())) { - return; - } + ASSERT_EQ(exp.size(), actual.size()); ITR exp_itr = exp.begin(); ITR exp_end = exp.end(); ITR actual_itr = actual.begin(); ITR actual_end = actual.end(); for (; exp_itr != exp_end && actual_itr != actual_end; ++exp_itr, ++actual_itr) { - EXPECT_EQUAL(exp_itr->first, actual_itr->first); - EXPECT_APPROX(exp_itr->second, actual_itr->second, 0.001); + EXPECT_EQ(exp_itr->first, actual_itr->first); + EXPECT_NEAR(exp_itr->second, actual_itr->second, 0.001); } - EXPECT_EQUAL(exp_itr == exp_end, actual_itr == actual_end); + EXPECT_EQ(exp_itr == exp_end, actual_itr == actual_end); } -void -RankSetupTest::testFeatureNormalization() +TEST_F(RankSetupTest, feature_normalization) { BlueprintFactory factory; factory.addPrototype(Blueprint::SP(new ValueBlueprint())); @@ -855,35 +874,39 @@ RankSetupTest::testFeatureNormalization() match_program->setup(*match_data, queryEnv); summaryProgram->setup(*match_data, queryEnv); - EXPECT_APPROX(2.0, Utils::getScoreFeature(*firstPhaseProgram, 1), 0.001); - EXPECT_APPROX(4.0, Utils::getScoreFeature(*secondPhaseProgram, 1), 0.001); + EXPECT_NEAR(2.0, Utils::getScoreFeature(*firstPhaseProgram, 1), 0.001); + EXPECT_NEAR(4.0, Utils::getScoreFeature(*secondPhaseProgram, 1), 0.001); - { // rank seed features + { + SCOPED_TRACE("rank seed features"); std::map<vespalib::string, feature_t> actual = Utils::getSeedFeatures(*summaryProgram, 1); std::map<vespalib::string, feature_t> exp; exp["mysum(value(5),value(5))"] = 10.0; exp["mysum(\"value( 5 )\",\"value( 5 )\")"] = 10.0; - TEST_DO(checkFeatures(exp, actual)); + checkFeatures(exp, actual); } - { // all rank features (1. phase) + { + SCOPED_TRACE("all rank features (1. phase)"); std::map<vespalib::string, feature_t> actual = Utils::getAllFeatures(*firstPhaseProgram, 1); std::map<vespalib::string, feature_t> exp; exp["value(1)"] = 1.0; exp["value(1).0"] = 1.0; exp["mysum(value(1),value(1))"] = 2.0; exp["mysum(value(1),value(1)).out"] = 2.0; - TEST_DO(checkFeatures(exp, actual)); + checkFeatures(exp, actual); } - { // all rank features (2. phase) + { + SCOPED_TRACE("all rank features (2. phase)"); std::map<vespalib::string, feature_t> actual = Utils::getAllFeatures(*secondPhaseProgram, 1); std::map<vespalib::string, feature_t> exp; exp["value(2)"] = 2.0; exp["value(2).0"] = 2.0; exp["mysum(value(2),value(2))"] = 4.0; exp["mysum(value(2),value(2)).out"] = 4.0; - TEST_DO(checkFeatures(exp, actual)); + checkFeatures(exp, actual); } - { // all match features + { + SCOPED_TRACE("all match features"); std::map<vespalib::string, feature_t> actual = Utils::getAllFeatures(*match_program, 1); std::map<vespalib::string, feature_t> exp; exp["value(3)"] = 3.0; @@ -892,9 +915,10 @@ RankSetupTest::testFeatureNormalization() exp["mysum(value(3),value(3)).out"] = 6.0; exp["mysum(\"value( 3 )\",\"value( 3 )\")"] = 6.0; exp["mysum(\"value( 3 )\",\"value( 3 )\").out"] = 6.0; - TEST_DO(checkFeatures(exp, actual)); + checkFeatures(exp, actual); } - { // all rank features (summary) + { + SCOPED_TRACE("all rank features (summary)"); std::map<vespalib::string, feature_t> actual = Utils::getAllFeatures(*summaryProgram, 1); std::map<vespalib::string, feature_t> exp; exp["value(5)"] = 5.0; @@ -903,7 +927,7 @@ RankSetupTest::testFeatureNormalization() exp["mysum(value(5),value(5)).out"] = 10.0; exp["mysum(\"value( 5 )\",\"value( 5 )\")"] = 10.0; exp["mysum(\"value( 5 )\",\"value( 5 )\").out"] = 10.0; - TEST_DO(checkFeatures(exp, actual)); + checkFeatures(exp, actual); } } @@ -914,15 +938,17 @@ RankSetupTest::testFeatureNormalization() RankProgram::UP rankProgram = rankSetup.create_dump_program(); rankProgram->setup(*match_data, queryEnv); - { // dump seed features + { + SCOPED_TRACE("dump seed features"); std::map<vespalib::string, feature_t> actual = Utils::getSeedFeatures(*rankProgram, 1); std::map<vespalib::string, feature_t> exp; exp["mysum(value(10),value(10))"] = 20.0; exp["mysum(\"value( 10 )\",\"value( 10 )\")"] = 20.0; - TEST_DO(checkFeatures(exp, actual)); + checkFeatures(exp, actual); } - { // all dump features + { + SCOPED_TRACE("all dump features"); std::map<vespalib::string, feature_t> actual = Utils::getAllFeatures(*rankProgram, 1); std::map<vespalib::string, feature_t> exp; @@ -935,72 +961,9 @@ RankSetupTest::testFeatureNormalization() exp["mysum(\"value( 10 )\",\"value( 10 )\")"] = 20.0; exp["mysum(\"value( 10 )\",\"value( 10 )\").out"] = 20.0; - TEST_DO(checkFeatures(exp, actual)); + checkFeatures(exp, actual); } } } - -RankSetupTest::RankSetupTest() : - _factory(), - _manager(), - _indexEnv(), - _queryEnv(), - _rankEnv(_factory, _indexEnv, _queryEnv), - _visitor() -{ - // register blueprints - setup_fef_test_plugin(_factory); - _factory.addPrototype(Blueprint::SP(new ValueBlueprint())); - _factory.addPrototype(Blueprint::SP(new RankingExpressionBlueprint())); - _factory.addPrototype(std::make_shared<SecondPhaseBlueprint>()); - - // setup an original attribute manager with two attributes - search::attribute::Config cfg(search::attribute::BasicType::INT32, - search::attribute::CollectionType::SINGLE); - search::AttributeVector::SP av1 = - search::AttributeFactory::createAttribute("staticrank1", cfg); - search::AttributeVector::SP av2 = - search::AttributeFactory::createAttribute("staticrank2", cfg); - av1->addDocs(5); - av2->addDocs(5); - for (uint32_t i = 0; i < 5; ++i) { - (static_cast<search::IntegerAttribute *>(av1.get()))->update(i, i + 100); - (static_cast<search::IntegerAttribute *>(av2.get()))->update(i, i + 200); - } - av1->commit(); - av2->commit(); - _manager.add(av1); - _manager.add(av2); - - // set the index environment - _queryEnv.setIndexEnv(&_indexEnv); - - // set the manager - _queryEnv.overrideAttributeManager(&_manager); -} - -RankSetupTest::~RankSetupTest() {} - -int -RankSetupTest::Main() -{ - TEST_INIT("ranksetup_test"); - - testValueBlueprint(); - testDoubleBlueprint(); - testSumBlueprint(); - testStaticRankBlueprint(); - testChainBlueprint(); - testCfgValueBlueprint(); - - testCompilation(); - testRankSetup(); - testExecution(); - testFeatureDump(); - testFeatureNormalization(); - - TEST_DONE(); -} - -TEST_APPHOOK(RankSetupTest); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/ranksetup/verify_feature/CMakeLists.txt b/searchlib/src/tests/ranksetup/verify_feature/CMakeLists.txt index 6215658c44c..306bbf8a714 100644 --- a/searchlib/src/tests/ranksetup/verify_feature/CMakeLists.txt +++ b/searchlib/src/tests/ranksetup/verify_feature/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_verify_feature_test_app TEST SOURCES verify_feature_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_verify_feature_test_app COMMAND searchlib_verify_feature_test_app) diff --git a/searchlib/src/tests/searchcommon/attribute/config/CMakeLists.txt b/searchlib/src/tests/searchcommon/attribute/config/CMakeLists.txt index d749bff4340..2405075c659 100644 --- a/searchlib/src/tests/searchcommon/attribute/config/CMakeLists.txt +++ b/searchlib/src/tests/searchcommon/attribute/config/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchcommon_attribute_config_test_app TEST SOURCES attribute_config_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchcommon_attribute_config_test_app NO_VALGRIND COMMAND searchcommon_attribute_config_test_app) diff --git a/searchlib/src/tests/searchcommon/schema/CMakeLists.txt b/searchlib/src/tests/searchcommon/schema/CMakeLists.txt index 51144c547d4..03e354c5acb 100644 --- a/searchlib/src/tests/searchcommon/schema/CMakeLists.txt +++ b/searchlib/src/tests/searchcommon/schema/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchcommon_schema_test_app TEST SOURCES schema_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchcommon_schema_test_app NO_VALGRIND COMMAND searchcommon_schema_test_app) diff --git a/searchlib/src/tests/searchcommon/schema/schema_test.cpp b/searchlib/src/tests/searchcommon/schema/schema_test.cpp index ad36454b6d7..6e33662a83e 100644 --- a/searchlib/src/tests/searchcommon/schema/schema_test.cpp +++ b/searchlib/src/tests/searchcommon/schema/schema_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchcommon/common/schema.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/stllike/string.h> #include <fstream> @@ -18,6 +19,10 @@ using schema::CollectionType; using SIAF = Schema::ImportedAttributeField; using SIF = Schema::IndexField; +vespalib::string src_path(vespalib::stringref prefix, vespalib::stringref path) { + return prefix + TEST_PATH(path); +} + void assertField(const Schema::Field& exp, const Schema::Field& act) { @@ -144,7 +149,7 @@ TEST(SchemaTest, test_load_and_save) { // load from config -> save to file -> load from file Schema s; - SchemaConfigurer configurer(s, "dir:load-save-cfg"); + SchemaConfigurer configurer(s, src_path("dir:", "load-save-cfg")); EXPECT_EQ(3u, s.getNumIndexFields()); assertIndexField(SIF("a", SDT::STRING), s.getIndexField(0)); assertIndexField(SIF("b", SDT::INT64), s.getIndexField(1)); @@ -308,7 +313,7 @@ TEST(SchemaTest, require_that_imported_attribute_fields_are_not_saved_to_disk) TEST(SchemaTest, require_that_schema_can_be_built_with_imported_attribute_fields) { Schema s; - SchemaConfigurer configurer(s, "dir:imported-fields-cfg"); + SchemaConfigurer configurer(s, src_path("dir:", "imported-fields-cfg")); const auto &imported = s.getImportedAttributeFields(); ASSERT_EQ(2u, imported.size()); @@ -323,7 +328,7 @@ TEST(SchemaTest, require_that_schema_can_be_built_with_imported_attribute_fields TEST(SchemaTest, require_that_index_field_is_loaded_with_default_values_when_properties_are_not_set) { Schema s; - s.loadFromFile("schema-without-index-field-properties.txt"); + s.loadFromFile(TEST_PATH("schema-without-index-field-properties.txt")); const auto& index_fields = s.getIndexFields(); ASSERT_EQ(1, index_fields.size()); @@ -336,7 +341,7 @@ TEST(SchemaTest, require_that_index_field_is_loaded_with_default_values_when_pro TEST(SchemaTest, test_load_from_saved_schema_with_summary_fields) { - vespalib::string schema_name("old-schema-with-summary-fields.txt"); + vespalib::string schema_name(TEST_PATH("old-schema-with-summary-fields.txt")); Schema s; s.addIndexField(Schema::IndexField("ifoo", DataType::STRING)); s.addIndexField(Schema::IndexField("ibar", DataType::INT32)); diff --git a/searchlib/src/tests/sort/CMakeLists.txt b/searchlib/src/tests/sort/CMakeLists.txt index 44ecff7c1a4..e2f2182df42 100644 --- a/searchlib/src/tests/sort/CMakeLists.txt +++ b/searchlib/src/tests/sort/CMakeLists.txt @@ -3,14 +3,14 @@ vespa_add_executable(searchlib_sortbenchmark_app SOURCES sortbenchmark.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_sortbenchmark_app COMMAND searchlib_sortbenchmark_app BENCHMARK) vespa_add_executable(searchlib_sort_test_app SOURCES sort_test.cpp DEPENDS - searchlib + vespa_searchlib ) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND VESPA_USE_LTO) target_link_options(searchlib_sort_test_app PRIVATE "-Wno-aggressive-loop-optimizations") @@ -20,6 +20,7 @@ vespa_add_executable(searchlib_uca_stress_app SOURCES uca.cpp DEPENDS - searchlib + vespa_searchlib + GTest::gtest ) vespa_add_test(NAME searchlib_uca_stress_app COMMAND searchlib_uca_stress_app BENCHMARK) diff --git a/searchlib/src/tests/sort/sort_test.cpp b/searchlib/src/tests/sort/sort_test.cpp index cbd040f7299..f0047e6fe0e 100644 --- a/searchlib/src/tests/sort/sort_test.cpp +++ b/searchlib/src/tests/sort/sort_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/common/sortspec.h> #include <vespa/searchlib/common/converters.h> @@ -240,7 +240,7 @@ TEST("testSortSpec") TEST("testSameAsJavaOrder") { std::vector<vespalib::string> javaOrder; - std::ifstream is("javaorder.zh"); + std::ifstream is(TEST_PATH("javaorder.zh")); while (!is.eof()) { std::string line; getline(is, line); diff --git a/searchlib/src/tests/sort/sortbenchmark.cpp b/searchlib/src/tests/sort/sortbenchmark.cpp index 3a93e359efc..2f18fcfff8c 100644 --- a/searchlib/src/tests/sort/sortbenchmark.cpp +++ b/searchlib/src/tests/sort/sortbenchmark.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/sort.h> #include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/buffer.h> @@ -8,21 +8,15 @@ using vespalib::Array; using vespalib::ConstBufferRef; -class Test : public vespalib::TestApp -{ -public: +struct Test { using V = std::vector<uint32_t>; std::vector< std::vector<uint32_t> > _data; - int Main() override; void generateVectors(size_t numVectors, size_t values); V merge(); void twoWayMerge(); V cat() const; - Test(); - ~Test() override; + ~Test(); }; - -Test::Test() = default; Test::~Test() = default; void @@ -82,31 +76,27 @@ Test::cat() const return c; } -TEST_APPHOOK(Test); - -int Test::Main() -{ - TEST_INIT("sortbenchmark"); +TEST_MAIN() { size_t numVectors(11); size_t values(10000000); vespalib::string type("radix"); - if (_argc > 1) { - values = strtol(_argv[1], NULL, 0); - if (_argc > 2) { - numVectors = strtol(_argv[2], NULL, 0); - if (_argc > 2) { - type = _argv[3]; + if (argc > 1) { + values = strtol(argv[1], NULL, 0); + if (argc > 2) { + numVectors = strtol(argv[2], NULL, 0); + if (argc > 2) { + type = argv[3]; } } } - + Test test; printf("Start with %ld vectors with %ld values and type '%s'(radix, qsort, merge)\n", numVectors, values, type.c_str()); - generateVectors(numVectors, values); + test.generateVectors(numVectors, values); printf("Start cat\n"); - V v = cat(); + auto v = test.cat(); printf("Cat %ld values\n", v.size()); if (type == "merge") { - V m = merge(); + auto m = test.merge(); printf("Merged %ld values\n", m.size()); } else if (type == "qsort") { std::sort(v.begin(), v.end()); @@ -116,6 +106,4 @@ int Test::Main() S(&v[0], v.size()); printf("sorted %ld value with radix::sort\n", v.size()); } - - TEST_DONE(); } diff --git a/searchlib/src/tests/sort/uca.cpp b/searchlib/src/tests/sort/uca.cpp index d11d230142b..41f6b927990 100644 --- a/searchlib/src/tests/sort/uca.cpp +++ b/searchlib/src/tests/sort/uca.cpp @@ -1,27 +1,17 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> + #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/common/sortspec.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/array.h> #include <unicode/ustring.h> #include <unicode/coll.h> #include <fcntl.h> #include <unistd.h> -#include <vespa/log/log.h> -LOG_SETUP("uca_stress"); - using icu::Collator; -class Test : public vespalib::TestApp -{ -public: - int Main() override; - void testFromDat(); -}; - - -void Test::testFromDat() +TEST(UcaStressTest, from_dat) { size_t badnesses = 0; @@ -40,8 +30,6 @@ void Test::testFromDat() int fd = open("sort-blobs.dat", O_RDONLY); char sbuf[4]; - int num=0; - uint32_t atleast = 0; while (read(fd, sbuf, 4) == 4) { @@ -49,20 +37,20 @@ void Test::testFromDat() uint32_t len = 0; int r = read(fd, &len, 4); - EXPECT_EQUAL(4, r); + EXPECT_EQ(4, r); r = read(fd, sbuf, 4); - EXPECT_EQUAL(4, r); - EXPECT_EQUAL(midMark, sbuf); + EXPECT_EQ(4, r); + EXPECT_EQ(midMark, sbuf); if (u16buffer.size() < len) { u16buffer.resize(len); } r = read(fd, &u16buffer[0], len*2); - EXPECT_EQUAL((int)len*2, r); + EXPECT_EQ((int)len*2, r); r = read(fd, sbuf, 4); - EXPECT_EQUAL(4, r); - EXPECT_EQUAL(endMark, sbuf); + EXPECT_EQ(4, r); + EXPECT_EQ(endMark, sbuf); uint32_t wanted = coll->getSortKey(&u16buffer[0], len, NULL, 0); @@ -77,7 +65,7 @@ void Test::testFromDat() for (uint32_t pretend = 1; pretend < wanted+8; ++pretend) { memset(&u8buffer[0], 0x99, u8buffer.size()); uint32_t got = coll->getSortKey(&u16buffer[0], len, &u8buffer[0], pretend); - EXPECT_EQUAL(wanted, got); + EXPECT_EQ(wanted, got); if (u8buffer[pretend+1] != 0x99) { printf("wrote 2 bytes too far: wanted space %d, pretend allocated %d, last good=%02x, bad=%02x %02x\n", @@ -95,24 +83,13 @@ void Test::testFromDat() memset(&u8buffer[0], 0x99, u8buffer.size()); uint32_t got = coll->getSortKey(&u16buffer[0], len, &u8buffer[0], u8buffer.size()); - EXPECT_EQUAL(wanted, got); + EXPECT_EQ(wanted, got); - EXPECT_EQUAL('\0', u8buffer[got-1]); - EXPECT_EQUAL((uint8_t)0x99, u8buffer[got]); - } - if (++num >= 10000) { - TEST_FLUSH(); - num=0; + EXPECT_EQ('\0', u8buffer[got-1]); + EXPECT_EQ((uint8_t)0x99, u8buffer[got]); } } - EXPECT_EQUAL(0u, badnesses); + EXPECT_EQ(0u, badnesses); } -TEST_APPHOOK(Test); - -int Test::Main() -{ - TEST_INIT("uca_stress"); - testFromDat(); - TEST_DONE(); -} +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/sortresults/CMakeLists.txt b/searchlib/src/tests/sortresults/CMakeLists.txt index f4aa4fd65f1..c9515226e97 100644 --- a/searchlib/src/tests/sortresults/CMakeLists.txt +++ b/searchlib/src/tests/sortresults/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_sortresults_test_app TEST SOURCES sortresults_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_sortresults_test_app COMMAND searchlib_sortresults_test_app) diff --git a/searchlib/src/tests/sortspec/CMakeLists.txt b/searchlib/src/tests/sortspec/CMakeLists.txt index 3fcd0a1bb7b..d89ed1e5557 100644 --- a/searchlib/src/tests/sortspec/CMakeLists.txt +++ b/searchlib/src/tests/sortspec/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_multilevelsort_test_app TEST SOURCES multilevelsort_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_multilevelsort_test_app COMMAND searchlib_multilevelsort_test_app) diff --git a/searchlib/src/tests/sortspec/multilevelsort_test.cpp b/searchlib/src/tests/sortspec/multilevelsort_test.cpp index f3bf363645e..86e8a932510 100644 --- a/searchlib/src/tests/sortspec/multilevelsort_test.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchlib/attribute/attributemanager.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <type_traits> #include <cinttypes> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/tensor/dense_tensor_store/CMakeLists.txt b/searchlib/src/tests/tensor/dense_tensor_store/CMakeLists.txt index 04a803b12ae..ac85b73f643 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/CMakeLists.txt +++ b/searchlib/src/tests/tensor/dense_tensor_store/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_dense_tensor_store_test_app TEST SOURCES dense_tensor_store_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_dense_tensor_store_test_app COMMAND searchlib_dense_tensor_store_test_app) diff --git a/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt b/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt index 4d26eec133d..045e6fde7b1 100644 --- a/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt +++ b/searchlib/src/tests/tensor/direct_tensor_store/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_direct_tensor_store_test_app TEST SOURCES direct_tensor_store_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_direct_tensor_store_test_app COMMAND searchlib_direct_tensor_store_test_app) diff --git a/searchlib/src/tests/tensor/distance_calculator/CMakeLists.txt b/searchlib/src/tests/tensor/distance_calculator/CMakeLists.txt index 029679e2f24..d72642abe1f 100644 --- a/searchlib/src/tests/tensor/distance_calculator/CMakeLists.txt +++ b/searchlib/src/tests/tensor/distance_calculator/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_distance_calculator_test_app TEST SOURCES distance_calculator_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test GTest::GTest ) diff --git a/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt b/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt index 92ad9ae2648..17e3475a77c 100644 --- a/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt +++ b/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_distance_functions_test_app TEST SOURCES distance_functions_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_distance_functions_test_app COMMAND searchlib_distance_functions_test_app) @@ -12,5 +12,5 @@ vespa_add_executable(searchlib_distance_functions_benchmark_app TEST SOURCES distance_functions_benchmark.cpp DEPENDS - searchlib + vespa_searchlib ) diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_benchmark.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_benchmark.cpp index 14a0adac651..0cb75075f5e 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_benchmark.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_benchmark.cpp @@ -75,7 +75,11 @@ void benchmark(size_t iterations, size_t elems, const DistanceFunctionFactory & template<typename T> void benchmark(size_t iterations, size_t elems, const std::string & dist_functions) { if (dist_functions.find("euclid") != npos) { - benchmark<T>(iterations, elems, EuclideanDistanceFunctionFactory<T>()); + if constexpr ( ! std::is_same<T, BFloat16>()) { + benchmark<T>(iterations, elems, EuclideanDistanceFunctionFactory<T>()); + } else { + benchmark<BFloat16>(iterations, elems, EuclideanDistanceFunctionFactory<float>()); + } } if (dist_functions.find("angular") != npos) { if constexpr ( ! std::is_same<T, BFloat16>()) { diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index eeae12e1695..c0296548b5a 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -13,9 +13,11 @@ LOG_SETUP("distance_function_test"); using namespace search::tensor; +using search::attribute::DistanceMetric; +using vespalib::BFloat16; +using vespalib::eval::CellType; using vespalib::eval::Int8Float; using vespalib::eval::TypedCells; -using search::attribute::DistanceMetric; template <typename T> TypedCells t(const std::vector<T> &v) { return TypedCells(v); } @@ -716,6 +718,73 @@ TEST(DistanceFunctionsTest, transformed_mips_growing_norm) EXPECT_GT(-29900.0, f->calc(t(p9d))); } +template <typename FloatType> +void +expect_reference_insertion_vector(FloatType exp_dist, DistanceMetric metric, CellType cell_type) +{ + std::vector<FloatType> lhs{0.0, 1.0}; + std::vector<FloatType> rhs{0.0, 1.0}; + auto factory = make_distance_function_factory(metric, cell_type); + auto func = factory->for_insertion_vector(t(lhs)); + // Updating the insertion vector should be reflected in the calculation. + lhs[0] = 1.0; + lhs[1] = 0.0; + EXPECT_EQ(exp_dist, func->calc(t(rhs))); +} + +template <typename FloatType> +void +expect_not_reference_insertion_vector(FloatType exp_dist, DistanceMetric metric, CellType cell_type) +{ + std::vector<FloatType> lhs{1.0, 0.0}; + std::vector<FloatType> rhs{0.0, 1.0}; + auto factory = make_distance_function_factory(metric, cell_type); + auto func = factory->for_insertion_vector(t(lhs)); + // Updating the insertion vector should NOT be reflected in the calculation, as a copy has been created. + lhs[0] = 0.0; + lhs[1] = 1.0; + EXPECT_EQ(exp_dist, func->calc(t(rhs))); +} + +TEST(DistanceFunctionsTest, angular_can_reference_insertion_vector) +{ + expect_reference_insertion_vector<float>(1.0, DistanceMetric::Angular, CellType::FLOAT); + expect_reference_insertion_vector<double>(1.0, DistanceMetric::Angular, CellType::DOUBLE); + expect_reference_insertion_vector<Int8Float>(1.0, DistanceMetric::Angular, CellType::INT8); + expect_not_reference_insertion_vector<BFloat16>(1.0, DistanceMetric::Angular, CellType::BFLOAT16); +} + +TEST(DistanceFunctionsTest, prenormalized_angular_can_reference_insertion_vector) +{ + expect_reference_insertion_vector<float>(1.0, DistanceMetric::PrenormalizedAngular, CellType::FLOAT); + expect_reference_insertion_vector<double>(1.0, DistanceMetric::PrenormalizedAngular, CellType::DOUBLE); + expect_reference_insertion_vector<Int8Float>(1.0, DistanceMetric::PrenormalizedAngular, CellType::INT8); + expect_not_reference_insertion_vector<BFloat16>(1.0, DistanceMetric::PrenormalizedAngular, CellType::BFLOAT16); +} + +TEST(DistanceFunctionsTest, euclidean_can_reference_insertion_vector) +{ + expect_reference_insertion_vector<float>(2.0, DistanceMetric::Euclidean, CellType::FLOAT); + expect_reference_insertion_vector<double>(2.0, DistanceMetric::Euclidean, CellType::DOUBLE); + expect_reference_insertion_vector<Int8Float>(2.0, DistanceMetric::Euclidean, CellType::INT8); + expect_not_reference_insertion_vector<BFloat16>(2.0, DistanceMetric::Euclidean, CellType::BFLOAT16); +} + +TEST(DistanceFunctionsTest, dotproduct_can_reference_insertion_vector) +{ + expect_reference_insertion_vector<float>(0.0, DistanceMetric::Dotproduct, CellType::FLOAT); + expect_reference_insertion_vector<double>(0.0, DistanceMetric::Dotproduct, CellType::DOUBLE); + expect_reference_insertion_vector<Int8Float>(0.0, DistanceMetric::Dotproduct, CellType::INT8); + expect_not_reference_insertion_vector<BFloat16>(0.0, DistanceMetric::Dotproduct, CellType::BFLOAT16); +} + +TEST(DistanceFunctionsTest, hamming_can_reference_insertion_vector) +{ + expect_reference_insertion_vector<float>(2.0, DistanceMetric::Hamming, CellType::FLOAT); + expect_reference_insertion_vector<double>(2.0, DistanceMetric::Hamming, CellType::DOUBLE); + expect_reference_insertion_vector<Int8Float>(2.0, DistanceMetric::Hamming, CellType::INT8); + expect_not_reference_insertion_vector<BFloat16>(2.0, DistanceMetric::Hamming, CellType::BFLOAT16); +} GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/hnsw_best_neighbors/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_best_neighbors/CMakeLists.txt index 9f2f89c78fe..e1530496612 100644 --- a/searchlib/src/tests/tensor/hnsw_best_neighbors/CMakeLists.txt +++ b/searchlib/src/tests/tensor/hnsw_best_neighbors/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_hnsw_best_neighbors_test_app TEST SOURCES hnsw_best_neighbors_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_hnsw_best_neighbors_test_app COMMAND searchlib_hnsw_best_neighbors_test_app) diff --git a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt index b02f93ca0af..35333dadaf4 100644 --- a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt +++ b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_hnsw_index_test_app TEST hnsw_index_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_hnsw_index_test_app COMMAND searchlib_hnsw_index_test_app) @@ -13,6 +13,6 @@ vespa_add_executable(mt_stress_hnsw_app TEST SOURCES stress_hnsw_mt.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) diff --git a/searchlib/src/tests/tensor/hnsw_nodeid_mapping/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_nodeid_mapping/CMakeLists.txt index c53902e3632..f789093acd5 100644 --- a/searchlib/src/tests/tensor/hnsw_nodeid_mapping/CMakeLists.txt +++ b/searchlib/src/tests/tensor/hnsw_nodeid_mapping/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_hnsw_nodeid_mapping_test_app TEST SOURCES hnsw_nodeid_mapping_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_hnsw_nodeid_mapping_test_app COMMAND searchlib_hnsw_nodeid_mapping_test_app) diff --git a/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt index 206e827cce2..8c51c5f4670 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt +++ b/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(searchlib_hnsw_save_load_test_app TEST hnsw_save_load_test.cpp DEPENDS searchlib_test - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_hnsw_save_load_test_app COMMAND searchlib_hnsw_save_load_test_app) diff --git a/searchlib/src/tests/tensor/tensor_buffer_operations/CMakeLists.txt b/searchlib/src/tests/tensor/tensor_buffer_operations/CMakeLists.txt index fd1893d0c56..8794c100b70 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_operations/CMakeLists.txt +++ b/searchlib/src/tests/tensor/tensor_buffer_operations/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_tensor_buffer_operations_test_app TEST SOURCES tensor_buffer_operations_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_tensor_buffer_operations_test_app COMMAND searchlib_tensor_buffer_operations_test_app) diff --git a/searchlib/src/tests/tensor/tensor_buffer_store/CMakeLists.txt b/searchlib/src/tests/tensor/tensor_buffer_store/CMakeLists.txt index 17e90b7c1e3..941180139ad 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_store/CMakeLists.txt +++ b/searchlib/src/tests/tensor/tensor_buffer_store/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_tensor_buffer_store_test_app TEST SOURCES tensor_buffer_store_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_tensor_buffer_store_test_app COMMAND searchlib_tensor_buffer_store_test_app) diff --git a/searchlib/src/tests/tensor/tensor_buffer_type_mapper/CMakeLists.txt b/searchlib/src/tests/tensor/tensor_buffer_type_mapper/CMakeLists.txt index d1affc6eb12..76de0888beb 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_type_mapper/CMakeLists.txt +++ b/searchlib/src/tests/tensor/tensor_buffer_type_mapper/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_tensor_buffer_type_mapper_test_app TEST SOURCES tensor_buffer_type_mapper_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_tensor_buffer_type_mapper_test_app COMMAND searchlib_tensor_buffer_type_mapper_test_app) diff --git a/searchlib/src/tests/transactionlog/CMakeLists.txt b/searchlib/src/tests/transactionlog/CMakeLists.txt index af644498ec8..45a10fd18d7 100644 --- a/searchlib/src/tests/transactionlog/CMakeLists.txt +++ b/searchlib/src/tests/transactionlog/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_translogclient_test_app TEST SOURCES translogclient_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_translogclient_test_app COMMAND searchlib_translogclient_test_app) @@ -11,6 +11,6 @@ vespa_add_executable(searchlib_translog_chunks_test_app TEST SOURCES chunks_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_translog_chunks_test_app COMMAND searchlib_translog_chunks_test_app) diff --git a/searchlib/src/tests/transactionlog/chunks_test.cpp b/searchlib/src/tests/transactionlog/chunks_test.cpp index 76045786895..e057e853d0d 100644 --- a/searchlib/src/tests/transactionlog/chunks_test.cpp +++ b/searchlib/src/tests/transactionlog/chunks_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/transactionlog/chunks.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <atomic> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index 9ba9780f8ed..07f43e98cd3 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -2,7 +2,7 @@ #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchlib/test/directory_handler.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/document/util/bytebuffer.h> diff --git a/searchlib/src/tests/transactionlogstress/CMakeLists.txt b/searchlib/src/tests/transactionlogstress/CMakeLists.txt index 2ed3d133174..160f26cb795 100644 --- a/searchlib/src/tests/transactionlogstress/CMakeLists.txt +++ b/searchlib/src/tests/transactionlogstress/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_translogstress_app SOURCES translogstress.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_translogstress_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/translogstress_test.sh BENCHMARK) diff --git a/searchlib/src/tests/true/true_test.cpp b/searchlib/src/tests/true/true_test.cpp index fee248200ad..8d2ed2aeca8 100644 --- a/searchlib/src/tests/true/true_test.cpp +++ b/searchlib/src/tests/true/true_test.cpp @@ -1,14 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("true_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> -TEST_SETUP(Test) - -int -Test::Main() -{ - TEST_INIT("true_test"); +TEST("true_test") { EXPECT_TRUE(true); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/url/CMakeLists.txt b/searchlib/src/tests/url/CMakeLists.txt index 13b45d5e4bc..6b4203b260f 100644 --- a/searchlib/src/tests/url/CMakeLists.txt +++ b/searchlib/src/tests/url/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_url_test_app TEST SOURCES url_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_url_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/dotest.sh DEPENDS searchlib_url_test_app) diff --git a/searchlib/src/tests/util/CMakeLists.txt b/searchlib/src/tests/util/CMakeLists.txt index 69b1b918dbc..6cdb4817cce 100644 --- a/searchlib/src/tests/util/CMakeLists.txt +++ b/searchlib/src/tests/util/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_rawbuf_test_app TEST SOURCES rawbuf_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_rawbuf_test_app COMMAND searchlib_rawbuf_test_app) diff --git a/searchlib/src/tests/util/bufferwriter/CMakeLists.txt b/searchlib/src/tests/util/bufferwriter/CMakeLists.txt index 1e2c166813f..e250d659e98 100644 --- a/searchlib/src/tests/util/bufferwriter/CMakeLists.txt +++ b/searchlib/src/tests/util/bufferwriter/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_bufferwriter_test_app TEST SOURCES bufferwriter_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_bufferwriter_test_app COMMAND searchlib_bufferwriter_test_app) diff --git a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp index dcf4d15181b..8faec949d86 100644 --- a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp +++ b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp @@ -1,10 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/util/drainingbufferwriter.h> #include <vespa/vespalib/util/rand48.h> +#include <cassert> namespace search { @@ -20,7 +21,7 @@ public: static constexpr size_t BUFFER_SIZE = 262144; StoreBufferWriter(); - ~StoreBufferWriter(); + ~StoreBufferWriter() override; void flush() override; size_t getBytesWritten() const { return _bytesWritten; } diff --git a/searchlib/src/tests/util/folded_string_compare/CMakeLists.txt b/searchlib/src/tests/util/folded_string_compare/CMakeLists.txt index 6cf9ed8bf4d..c7422f0fe0a 100644 --- a/searchlib/src/tests/util/folded_string_compare/CMakeLists.txt +++ b/searchlib/src/tests/util/folded_string_compare/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_folded_string_compare_test_app TEST SOURCES folded_string_compare_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_folded_string_compare_test_app COMMAND searchlib_folded_string_compare_test_app) diff --git a/searchlib/src/tests/util/rawbuf_test.cpp b/searchlib/src/tests/util/rawbuf_test.cpp index cee340481f8..5f8001b6af3 100644 --- a/searchlib/src/tests/util/rawbuf_test.cpp +++ b/searchlib/src/tests/util/rawbuf_test.cpp @@ -2,7 +2,7 @@ #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("rawbuf_test"); diff --git a/searchlib/src/tests/util/searchable_stats/CMakeLists.txt b/searchlib/src/tests/util/searchable_stats/CMakeLists.txt index f8a4182a7fc..a091f32ef0f 100644 --- a/searchlib/src/tests/util/searchable_stats/CMakeLists.txt +++ b/searchlib/src/tests/util/searchable_stats/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_searchable_stats_test_app TEST SOURCES searchable_stats_test.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) vespa_add_test(NAME searchlib_searchable_stats_test_app COMMAND searchlib_searchable_stats_test_app) diff --git a/searchlib/src/tests/util/slime_output_raw_buf_adapter/CMakeLists.txt b/searchlib/src/tests/util/slime_output_raw_buf_adapter/CMakeLists.txt index 041053a2e27..ab370e34dac 100644 --- a/searchlib/src/tests/util/slime_output_raw_buf_adapter/CMakeLists.txt +++ b/searchlib/src/tests/util/slime_output_raw_buf_adapter/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchlib_slime_output_raw_buf_adapter_test_app TEST SOURCES slime_output_raw_buf_adapter_test.cpp DEPENDS - searchlib + vespa_searchlib ) vespa_add_test(NAME searchlib_slime_output_raw_buf_adapter_test_app COMMAND searchlib_slime_output_raw_buf_adapter_test_app) diff --git a/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt b/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt index 9f2d04b7918..5fb3682013b 100644 --- a/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt +++ b/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchlib_vespa-fileheader-inspect_test_app TEST SOURCES vespa-fileheader-inspect_test.cpp DEPENDS - searchlib + vespa_searchlib AFTER searchlib_vespa-fileheader-inspect_app ) diff --git a/searchlib/src/vespa/searchlib/CMakeLists.txt b/searchlib/src/vespa/searchlib/CMakeLists.txt index 849d22455d0..89aee5d26ff 100644 --- a/searchlib/src/vespa/searchlib/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/CMakeLists.txt @@ -1,6 +1,6 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. find_package(ICU 60.0 REQUIRED COMPONENTS uc i18n) -vespa_add_library(searchlib +vespa_add_library(vespa_searchlib SOURCES $<TARGET_OBJECTS:searchlib_aggregation> $<TARGET_OBJECTS:searchlib_attribute> @@ -43,4 +43,4 @@ vespa_add_library(searchlib ${VESPA_ATOMIC_LIB} ) -vespa_add_target_package_dependency(searchlib Protobuf) +vespa_add_target_package_dependency(vespa_searchlib Protobuf) diff --git a/searchlib/src/vespa/searchlib/aggregation/grouping.h b/searchlib/src/vespa/searchlib/aggregation/grouping.h index f64aa2f5a3d..77d5655e5d2 100644 --- a/searchlib/src/vespa/searchlib/aggregation/grouping.h +++ b/searchlib/src/vespa/searchlib/aggregation/grouping.h @@ -73,10 +73,8 @@ public: void postMerge(); void preAggregate(bool isOrdered); void prune(const Grouping & b); - void aggregate(DocId from, DocId to); - void aggregate(DocId docId, HitRank rank = 0); - void aggregate(const document::Document & doc, HitRank rank = 0); - void aggregate(const RankedHit * rankedHit, unsigned int len); + void aggregate(DocId docId, HitRank rank); + void aggregate(const document::Document & doc, HitRank rank); void convertToGlobalId(const IDocumentMetaStore &metaStore); void postAggregate(); void postProcess(); @@ -84,6 +82,9 @@ public: void cleanTemporary(); void configureStaticStuff(const expression::ConfigureStaticParams & params); void cleanupAttributeReferences(); + // Only used by tests + void aggregate(DocId from, DocId to); + void aggregate(const RankedHit * rankedHit, unsigned int len); }; } diff --git a/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp b/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp index 81664ce66eb..89e6e6685e9 100644 --- a/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/modifiers.cpp @@ -4,10 +4,10 @@ #include "grouping.h" #include <vespa/searchlib/expression/multiargfunctionnode.h> #include <vespa/searchlib/expression/attributenode.h> -#include <vespa/searchlib/expression/attribute_map_lookup_node.h> #include <vespa/searchlib/expression/documentfieldnode.h> #include <vespa/searchlib/expression/interpolated_document_field_lookup_node.h> #include <vespa/searchlib/expression/interpolatedlookupfunctionnode.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> using namespace search::expression; @@ -20,44 +20,32 @@ AttributeNodeReplacer::check(const vespalib::Identifiable &obj) const } void +AttributeNodeReplacer::replaceRecurse(ExpressionNode * exp, std::function<void(ExpressionNodeUP)> && modifier) { + if (exp == nullptr) return; + if (exp->inherits(AttributeNode::classId)) { + auto replacementNode = getReplacementNode(static_cast<const AttributeNode &>(*exp)); + if (replacementNode) { + modifier(std::move(replacementNode)); + } + } else { + exp->select(*this, *this); + } +} + +void AttributeNodeReplacer::execute(vespalib::Identifiable &obj) { if (obj.getClass().inherits(GroupingLevel::classId)) { - GroupingLevel & g(static_cast<GroupingLevel &>(obj)); - if (g.getExpression().getRoot()->inherits(AttributeNode::classId)) { - auto replacementNode = getReplacementNode(static_cast<const AttributeNode &>(*g.getExpression().getRoot())); - if (replacementNode) { - g.setExpression(std::move(replacementNode)); - } - } else { - g.getExpression().getRoot()->select(*this, *this); - } + auto & g(static_cast<GroupingLevel &>(obj)); + replaceRecurse(g.getExpression().getRoot(), [&g](ExpressionNodeUP replacement) { g.setExpression(std::move(replacement)); }); g.groupPrototype().select(*this, *this); } else if(obj.getClass().inherits(AggregationResult::classId)) { - AggregationResult & a(static_cast<AggregationResult &>(obj)); - ExpressionNode * e(a.getExpression()); - if (e) { - if (e->inherits(AttributeNode::classId)) { - auto replacementNode = getReplacementNode(static_cast<const AttributeNode &>(*e)); - if (replacementNode) { - a.setExpression(std::move(replacementNode)); - } - } else { - e->select(*this, *this); - } - } + auto & a(static_cast<AggregationResult &>(obj)); + replaceRecurse(a.getExpression(), [&a](ExpressionNodeUP replacement) { a.setExpression(std::move(replacement)); }); } else if(obj.getClass().inherits(MultiArgFunctionNode::classId)) { MultiArgFunctionNode::ExpressionNodeVector & v(static_cast<MultiArgFunctionNode &>(obj).expressionNodeVector()); - for(size_t i(0), m(v.size()); i < m; i++) { - ExpressionNode::CP & e(v[i]); - if (e->inherits(AttributeNode::classId)) { - auto replacementNode = getReplacementNode(static_cast<const AttributeNode &>(*e)); - if (replacementNode) { - e = std::move(replacementNode); - } - } else { - e->select(*this, *this); - } + for (auto & e : v) { + replaceRecurse(e.get(), [&e](ExpressionNodeUP replacement) noexcept { e = std::move(replacement); }); } } } @@ -72,6 +60,14 @@ Attribute2DocumentAccessor::getReplacementNode(const AttributeNode &attributeNod return std::make_unique<DocumentFieldNode>(attributeNode.getAttributeName()); } +std::unique_ptr<ExpressionNode> +NonAttribute2DocumentAccessor::getReplacementNode(const expression::AttributeNode &attributeNode) { + if (_attrCtx.getAttribute(attributeNode.getAttributeName()) == nullptr) { + return Attribute2DocumentAccessor::getReplacementNode(attributeNode); + } + return {}; +} + } // this function was added by ../../forcelink.sh diff --git a/searchlib/src/vespa/searchlib/aggregation/modifiers.h b/searchlib/src/vespa/searchlib/aggregation/modifiers.h index 934e7111ced..9b0f5d8186d 100644 --- a/searchlib/src/vespa/searchlib/aggregation/modifiers.h +++ b/searchlib/src/vespa/searchlib/aggregation/modifiers.h @@ -4,28 +4,43 @@ #include <vespa/vespalib/objects/objectoperation.h> #include <vespa/vespalib/objects/objectpredicate.h> #include <memory> +#include <functional> namespace search::expression { + class ExpressionNode; + class AttributeNode; +} -class ExpressionNode; -class AttributeNode; - +namespace search::attribute { + class IAttributeContext; } namespace search::aggregation { class AttributeNodeReplacer : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { +protected: + using ExpressionNodeUP = std::unique_ptr<expression::ExpressionNode>; private: + void replaceRecurse(expression::ExpressionNode * exp, std::function<void(ExpressionNodeUP)> && modifier); void execute(vespalib::Identifiable &obj) override; bool check(const vespalib::Identifiable &obj) const override; - virtual std::unique_ptr<search::expression::ExpressionNode> getReplacementNode(const search::expression::AttributeNode &attributeNode) = 0; + virtual ExpressionNodeUP getReplacementNode(const expression::AttributeNode &attributeNode) = 0; }; class Attribute2DocumentAccessor : public AttributeNodeReplacer { +protected: + ExpressionNodeUP getReplacementNode(const expression::AttributeNode &attributeNode) override; +}; + +class NonAttribute2DocumentAccessor : public Attribute2DocumentAccessor +{ +public: + explicit NonAttribute2DocumentAccessor(const attribute::IAttributeContext &attrCtx) noexcept : _attrCtx(attrCtx) {} private: - std::unique_ptr<search::expression::ExpressionNode> getReplacementNode(const search::expression::AttributeNode &attributeNode) override; + ExpressionNodeUP getReplacementNode(const expression::AttributeNode &attributeNode) override; + const attribute::IAttributeContext &_attrCtx; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 70b86bf22a1..1f8b0cf28d3 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -11,7 +11,6 @@ #include "in_term_search.h" #include "multi_term_or_filter_search.h" #include "predicate_attribute.h" -#include <vespa/eval/eval/value.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcommon/attribute/hit_estimate_flow_stats_adapter.h> #include <vespa/searchlib/common/location.h> @@ -82,6 +81,7 @@ using search::queryeval::FieldSpecBase; using search::queryeval::FieldSpecBaseList; using search::queryeval::FilterWrapper; using search::queryeval::IRequestContext; +using search::queryeval::MatchingPhase; using search::queryeval::NoUnpack; using search::queryeval::OrLikeSearch; using search::queryeval::OrSearch; @@ -443,7 +443,8 @@ private: class DirectWandBlueprint : public queryeval::ComplexLeafBlueprint { private: - mutable queryeval::SharedWeakAndPriorityQueue _scores; + using WeakAndPriorityQueue = queryeval::WeakAndPriorityQueue; + std::unique_ptr<WeakAndPriorityQueue> _scores; const queryeval::wand::score_t _scoreThreshold; double _thresholdBoostFactor; const uint32_t _scoresAdjustFrequency; @@ -451,19 +452,23 @@ private: std::vector<IDirectPostingStore::LookupResult> _terms; const IDocidWithWeightPostingStore &_attr; vespalib::datastore::EntryRef _dictionary_snapshot; + MatchingPhase _matching_phase; + public: DirectWandBlueprint(const FieldSpec &field, const IDocidWithWeightPostingStore &attr, uint32_t scoresToTrack, - queryeval::wand::score_t scoreThreshold, double thresholdBoostFactor, size_t size_hint) + queryeval::wand::score_t scoreThreshold, double thresholdBoostFactor, size_t size_hint, + bool thread_safe) : ComplexLeafBlueprint(field), - _scores(scoresToTrack), + _scores(WeakAndPriorityQueue::createHeap(scoresToTrack, thread_safe)), _scoreThreshold(scoreThreshold), _thresholdBoostFactor(thresholdBoostFactor), - _scoresAdjustFrequency(queryeval::DEFAULT_PARALLEL_WAND_SCORES_ADJUST_FREQUENCY), + _scoresAdjustFrequency(queryeval::wand::DEFAULT_PARALLEL_WAND_SCORES_ADJUST_FREQUENCY), _weights(), _terms(), _attr(attr), - _dictionary_snapshot(_attr.get_dictionary_snapshot()) + _dictionary_snapshot(_attr.get_dictionary_snapshot()), + _matching_phase(MatchingPhase::FIRST_PHASE) { _weights.reserve(size_hint); _terms.reserve(size_hint); @@ -496,7 +501,7 @@ public: using OrFlow = search::queryeval::OrFlow; using MyAdapter = attribute::DirectPostingStoreFlowStatsAdapter; double child_est = OrFlow::estimate_of(MyAdapter(docid_limit), _terms); - double my_est = abs_to_rel_est(_scores.getScoresToTrack(), docid_limit); + double my_est = abs_to_rel_est(_scores->getScoresToTrack(), docid_limit); double est = (child_est + my_est) / 2.0; return {est, OrFlow::cost_of(MyAdapter(docid_limit), _terms, false), OrFlow::cost_of(MyAdapter(docid_limit), _terms, true) + queryeval::flow::heap_cost(est, _terms.size())}; @@ -507,14 +512,15 @@ public: if (_terms.empty()) { return std::make_unique<queryeval::EmptySearch>(); } + bool readonly_scores_heap = (_matching_phase != MatchingPhase::FIRST_PHASE); return queryeval::ParallelWeakAndSearch::create(*tfmda[0], - queryeval::ParallelWeakAndSearch::MatchParams(_scores, _scoreThreshold, - _thresholdBoostFactor, _scoresAdjustFrequency) - .setDocIdLimit(get_docid_limit()), - _weights, _terms, _attr, strict()); + queryeval::ParallelWeakAndSearch::MatchParams(*_scores, _scoreThreshold, _thresholdBoostFactor, + _scoresAdjustFrequency, get_docid_limit()), + _weights, _terms, _attr, strict(), readonly_scores_heap); } std::unique_ptr<SearchIterator> createFilterSearch(FilterConstraint constraint) const override; bool always_needs_unpack() const override { return true; } + void set_matching_phase(MatchingPhase matching_phase) noexcept override; }; DirectWandBlueprint::~DirectWandBlueprint() = default; @@ -534,6 +540,27 @@ DirectWandBlueprint::createFilterSearch(FilterConstraint constraint) const } } +void +DirectWandBlueprint::set_matching_phase(MatchingPhase matching_phase) noexcept +{ + _matching_phase = matching_phase; + if (matching_phase != MatchingPhase::FIRST_PHASE) { + /* + * During first phase matching, the scores heap is adjusted by + * the iterators. The minimum score is increased when the + * scores heap is full while handling a matching document with + * a higher score than the worst existing one. + * + * During later matching phases, only the original minimum + * score is used, and the heap is not updated by the + * iterators. This ensures that all documents considered a hit + * by the first phase matching will also be considered as hits + * by the later matching phases. + */ + _scores->set_min_score(_scoreThreshold); + } +} + bool AttributeFieldBlueprint::getRange(vespalib::string &from, vespalib::string &to) const { if (_type == INT) { @@ -712,15 +739,12 @@ public: void visit(query::WandTerm &n) override { if (has_always_btree_iterators_with_docid_and_weight()) { - auto *bp = new DirectWandBlueprint(_field, *_dwwps, - n.getTargetNumHits(), n.getScoreThreshold(), n.getThresholdBoostFactor(), - n.getNumTerms()); + auto *bp = new DirectWandBlueprint(_field, *_dwwps, n.getTargetNumHits(), n.getScoreThreshold(), + n.getThresholdBoostFactor(), n.getNumTerms(), is_search_multi_threaded()); createDirectMultiTerm(bp, n); } else { - auto *bp = new ParallelWeakAndBlueprint(_field, - n.getTargetNumHits(), - n.getScoreThreshold(), - n.getThresholdBoostFactor()); + auto *bp = new ParallelWeakAndBlueprint(_field, n.getTargetNumHits(), n.getScoreThreshold(), + n.getThresholdBoostFactor(), is_search_multi_threaded()); createShallowWeightedSet(bp, n, _field, _attr.isIntegerType()); } } diff --git a/searchlib/src/vespa/searchlib/attribute/attributecontext.h b/searchlib/src/vespa/searchlib/attribute/attributecontext.h index a02e05abe4f..bd98031ee66 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributecontext.h +++ b/searchlib/src/vespa/searchlib/attribute/attributecontext.h @@ -29,7 +29,7 @@ private: const IAttributeVector *getAttribute(AttributeMap & map, const string & name, bool stableEnum) const; const IAttributeVector *getAttributeMtSafe(AttributeMap & map, const string & name, bool stableEnum) const; public: - AttributeContext(const IAttributeManager & manager); + explicit AttributeContext(const IAttributeManager & manager); ~AttributeContext() override; // Implements IAttributeContext diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 4f1d3a3a72c..f86f4993444 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -4,7 +4,7 @@ #include "allocatedbitvector.h" #include "partialbitvector.h" #include <vespa/searchlib/util/file_settings.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/thread_bundle.h> #include <vespa/vespalib/util/size_literals.h> @@ -18,7 +18,7 @@ LOG_SETUP(".searchlib.common.bitvector"); using vespalib::make_string; using vespalib::IllegalArgumentException; -using vespalib::hwaccelrated::IAccelrated; +using vespalib::hwaccelerated::IAccelerated; using vespalib::Optimized; using vespalib::alloc::Alloc; @@ -44,7 +44,7 @@ struct BitVector::OrParts : vespalib::Runnable _byte_size((size + 7)/8) {} void run() override { - const auto & accelrator = IAccelrated::getAccelerator(); + const auto & accelrator = IAccelerated::getAccelerator(); BitVector * master = _vectors[0]; Word * destination = master->getWordIndex(_offset); for (uint32_t i(1); i < _vectors.size(); i++) { @@ -224,7 +224,7 @@ BitVector::countInterval(Range range_in) const ++endw; } if (startw < endw) { - res += IAccelrated::getAccelerator().populationCount(bitValues + startw, endw - startw); + res += IAccelerated::getAccelerator().populationCount(bitValues + startw, endw - startw); } if (partialEnd) { res += Optimized::popCount(load(bitValues[endw]) & ~endBits(last)); @@ -242,12 +242,12 @@ BitVector::orWith(const BitVector & right) if (right.size() < size()) { ssize_t commonBytes = numActiveBytes(range.start(), range.end()) - sizeof(Word); if (commonBytes > 0) { - IAccelrated::getAccelerator().orBit(getWordIndex(range.start()), right.getWordIndex(range.start()), commonBytes); + IAccelerated::getAccelerator().orBit(getWordIndex(range.start()), right.getWordIndex(range.start()), commonBytes); } Index last(range.end() - 1); store(getWordIndex(last)[0], getWordIndex(last)[0] | (load(right.getWordIndex(last)[0]) & ~endBits(last))); } else { - IAccelrated::getAccelerator().orBit(getWordIndex(range.start()), right.getWordIndex(range.start()), getActiveBytes()); + IAccelerated::getAccelerator().orBit(getWordIndex(range.start()), right.getWordIndex(range.start()), getActiveBytes()); } repairEnds(); invalidateCachedCount(); @@ -276,7 +276,7 @@ BitVector::andWith(const BitVector & right) } uint32_t commonBytes = std::min(getActiveBytes(), numActiveBytes(getStartIndex(), right.size())); - IAccelrated::getAccelerator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); + IAccelerated::getAccelerator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); if (right.size() < size()) { clearInterval(right.size(), size()); } @@ -295,12 +295,12 @@ BitVector::andNotWith(const BitVector& right) if (right.size() < size()) { ssize_t commonBytes = numActiveBytes(range.start(), range.end()) - sizeof(Word); if (commonBytes > 0) { - IAccelrated::getAccelerator().andNotBit(getWordIndex(range.start()), right.getWordIndex(range.start()), commonBytes); + IAccelerated::getAccelerator().andNotBit(getWordIndex(range.start()), right.getWordIndex(range.start()), commonBytes); } Index last(range.end() - 1); store(getWordIndex(last)[0], getWordIndex(last)[0] & ~(load(right.getWordIndex(last)[0]) & ~endBits(last))); } else { - IAccelrated::getAccelerator().andNotBit(getWordIndex(range.start()), right.getWordIndex(range.start()), getActiveBytes()); + IAccelerated::getAccelerator().andNotBit(getWordIndex(range.start()), right.getWordIndex(range.start()), getActiveBytes()); } repairEnds(); @@ -309,7 +309,7 @@ BitVector::andNotWith(const BitVector& right) void BitVector::notSelf() { - IAccelrated::getAccelerator().notBit(getActiveStart(), getActiveBytes()); + IAccelerated::getAccelerator().notBit(getActiveStart(), getActiveBytes()); setGuardBit(); invalidateCachedCount(); } diff --git a/searchlib/src/vespa/searchlib/engine/search_protocol_proto.h b/searchlib/src/vespa/searchlib/engine/search_protocol_proto.h index 3c5aa34ad2a..aafa253e7ff 100644 --- a/searchlib/src/vespa/searchlib/engine/search_protocol_proto.h +++ b/searchlib/src/vespa/searchlib/engine/search_protocol_proto.h @@ -2,4 +2,4 @@ #pragma once -#include "search_protocol.pb.h" +#include <vespa/searchlib/engine/search_protocol.pb.h> diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 4736dbecb86..b468d653032 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -16,7 +16,6 @@ vespa_add_library(searchlib_features OBJECT distance_calculator_bundle.cpp distancefeature.cpp distancetopathfeature.cpp - documenttestutils.cpp dotproductfeature.cpp element_completeness_feature.cpp element_similarity_feature.cpp @@ -26,6 +25,8 @@ vespa_add_library(searchlib_features OBJECT fieldmatchfeature.cpp fieldtermmatchfeature.cpp firstphasefeature.cpp + first_phase_rank_feature.cpp + first_phase_rank_lookup.cpp flow_completeness_feature.cpp foreachfeature.cpp freshnessfeature.cpp diff --git a/searchlib/src/vespa/searchlib/features/documenttestutils.cpp b/searchlib/src/vespa/searchlib/features/documenttestutils.cpp deleted file mode 100644 index 5962cb32573..00000000000 --- a/searchlib/src/vespa/searchlib/features/documenttestutils.cpp +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - - -#include "utils.h" -#include <vespa/searchlib/fef/itablemanager.h> -#include <vespa/searchlib/fef/properties.h> -#include <vespa/searchlib/fef/itermdata.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/stllike/asciistream.h> - -#include <cmath> -#include <ostream> - -#include <vespa/vespalib/util/issue.h> -using vespalib::Issue; - -#include <vespa/log/log.h> -LOG_SETUP(".features.utils"); -using namespace search::fef; - -namespace search::features::util { - -feature_t -lookupConnectedness(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) -{ - if (termId == 0) { - return fallback; // no previous term - } - - const ITermData * data = env.getTerm(termId); - const ITermData * prev = env.getTerm(termId - 1); - if (data == nullptr || prev == nullptr) { - return fallback; // default value - } - return lookupConnectedness(env, data->getUniqueId(), prev->getUniqueId(), fallback); -} - -feature_t -lookupConnectedness(const search::fef::IQueryEnvironment& env, - uint32_t currUniqueId, uint32_t prevUniqueId, feature_t fallback) -{ - // Connectedness of 0.5 between term with unique id 2 and term with unique id 1 is represented as: - // [vespa.term.2.connexity: "1", vespa.term.2.connexity: "0.5"] - vespalib::asciistream os; - os << "vespa.term." << currUniqueId << ".connexity"; - Property p = env.getProperties().lookup(os.str()); - if (p.size() == 2) { - // we have a defined connectedness with the previous term - if (strToNum<uint32_t>(p.getAt(0)) == prevUniqueId) { - return strToNum<feature_t>(p.getAt(1)); - } - } - return fallback; -} - -feature_t -lookupSignificance(const search::fef::IQueryEnvironment& env, const ITermData& term, feature_t fallback) -{ - // Significance of 0.5 for term with unique id 1 is represented as: - // [vespa.term.1.significance: "0.5"] - vespalib::asciistream os; - os << "vespa.term." << term.getUniqueId() << ".significance"; - Property p = env.getProperties().lookup(os.str()); - if (p.found()) { - return strToNum<feature_t>(p.get()); - } - return fallback; -} - -feature_t -lookupSignificance(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) -{ - const ITermData* term = env.getTerm(termId); - if (term == nullptr) { - return fallback; - } - return lookupSignificance(env, *term, fallback); -} - -double -getRobertsonSparckJonesWeight(double docCount, double docsInCorpus) -{ - return std::log((docsInCorpus - docCount + 0.5)/(docCount + 0.5)); -} - -static const double N = 1000000.0; - -feature_t -getSignificance(double docFreq) -{ - if (docFreq < (1.0/N)) { - docFreq = 1.0/N; - } - if (docFreq > 1.0) { - docFreq = 1.0; - } - double d = std::log(docFreq)/std::log(1.0/N); - return 0.5 + 0.5 * d; -#if 0 - double n = docFreq * N; - n = (n == 0) ? 1 : (n > N ? N : n); - double a = getRobertsonSparckJonesWeight(1, N + 1); - double b = getRobertsonSparckJonesWeight(N + 1, N + 1); - double w = getRobertsonSparckJonesWeight(n, N + 1); - return ((w - b)/(a - b)); -#endif -} - -feature_t -getSignificance(const search::fef::ITermData& termData) -{ - using FRA = search::fef::ITermFieldRangeAdapter; - double df = 0; - for (FRA iter(termData); iter.valid(); iter.next()) { - df = std::max(df, iter.get().getDocFreq()); - } - - feature_t signif = getSignificance(df); - LOG(debug, "getSignificance %e %f [ %e %f ] = %e", df, df, df * N, df * N, signif); - return signif; -} - -const search::fef::Table * -lookupTable(const search::fef::IIndexEnvironment & env, const vespalib::string & featureName, - const vespalib::string & table, const vespalib::string & fieldName, const vespalib::string & fallback) -{ - vespalib::string tn1 = env.getProperties().lookup(featureName, table).get(fallback); - vespalib::string tn2 = env.getProperties().lookup(featureName, table, fieldName).get(tn1); - const search::fef::Table * retval = env.getTableManager().getTable(tn2); - if (retval == nullptr) { - LOG(warning, "Could not find the %s '%s' to be used for field '%s' in feature '%s'", - table.c_str(), tn2.c_str(), fieldName.c_str(), featureName.c_str()); - } - return retval; -} - -const search::fef::ITermData * -getTermByLabel(const search::fef::IQueryEnvironment &env, const vespalib::string &label) -{ - // Labeling the query item with unique id '5' with the label 'foo' - // is represented as: [vespa.label.foo.id: "5"] - vespalib::asciistream os; - os << "vespa.label." << label << ".id"; - Property p = env.getProperties().lookup(os.str()); - if (!p.found()) { - return 0; - } - uint32_t uid = strToNum<uint32_t>(p.get()); - if (uid == 0) { - Issue::report("Query label '%s' was attached to invalid unique id: '%s'", - label.c_str(), p.get().c_str()); - return 0; - } - for (uint32_t i(0), m(env.getNumTerms()); i < m; ++i) { - const ITermData *term = env.getTerm(i); - if (term->getUniqueId() == uid) { - return term; - } - } - Issue::report("Query label '%s' was attached to non-existing unique id: '%s'", - label.c_str(), p.get().c_str()); - return 0; -} - -} diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index f3e3a3545fa..a957ab61fb2 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -20,7 +20,7 @@ using namespace search::fef; using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::TypedCells; using vespalib::Issue; -using vespalib::hwaccelrated::IAccelrated; +using vespalib::hwaccelerated::IAccelerated; namespace search::features { namespace dotproduct::wset { @@ -213,7 +213,7 @@ namespace dotproduct::array { template <typename BaseType> DotProductExecutorBase<BaseType>::DotProductExecutorBase(const V & queryVector) : FeatureExecutor(), - _multiplier(IAccelrated::getAccelerator()), + _multiplier(IAccelerated::getAccelerator()), _queryVector(queryVector) { } diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h index 8f81819e9ec..d16e2bd9276 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h @@ -7,7 +7,7 @@ #include <vespa/searchcommon/attribute/i_multi_value_read_view.h> #include <vespa/searchcommon/attribute/multivalue.h> #include <vespa/searchlib/fef/blueprint.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <vespa/vespalib/stllike/hash_map.hpp> namespace search::fef { class Property; } @@ -168,7 +168,7 @@ class DotProductExecutorBase : public fef::FeatureExecutor { public: using V = std::vector<BaseType>; private: - const vespalib::hwaccelrated::IAccelrated & _multiplier; + const vespalib::hwaccelerated::IAccelerated & _multiplier; V _queryVector; virtual vespalib::ConstArrayRef<BaseType> getAttributeValues(uint32_t docid) = 0; public: diff --git a/searchlib/src/vespa/searchlib/features/first_phase_rank_feature.cpp b/searchlib/src/vespa/searchlib/features/first_phase_rank_feature.cpp new file mode 100644 index 00000000000..5c8a9a391ff --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/first_phase_rank_feature.cpp @@ -0,0 +1,71 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "first_phase_rank_feature.h" +#include "valuefeature.h" +#include <vespa/vespalib/util/stash.h> + +namespace search::features { + +FirstPhaseRankExecutor::FirstPhaseRankExecutor(const FirstPhaseRankLookup& lookup) + : FeatureExecutor(), + _lookup(lookup) +{ +} +FirstPhaseRankExecutor::~FirstPhaseRankExecutor() = default; + +void +FirstPhaseRankExecutor::execute(uint32_t docid) +{ + outputs().set_number(0, _lookup.lookup(docid)); +} + +FirstPhaseRankBlueprint::FirstPhaseRankBlueprint() + : Blueprint("firstPhaseRank") +{ +} + +FirstPhaseRankBlueprint::~FirstPhaseRankBlueprint() = default; + +void +FirstPhaseRankBlueprint::visitDumpFeatures(const fef::IIndexEnvironment&, fef::IDumpFeatureVisitor&) const +{ +} + +std::unique_ptr<fef::Blueprint> +FirstPhaseRankBlueprint::createInstance() const +{ + return std::make_unique<FirstPhaseRankBlueprint>(); +} + +fef::ParameterDescriptions +FirstPhaseRankBlueprint::getDescriptions() const +{ + return fef::ParameterDescriptions().desc(); +} + +bool +FirstPhaseRankBlueprint::setup(const fef::IIndexEnvironment&, const fef::ParameterList&) +{ + describeOutput("score", "The first phase rank."); + return true; +} + +void +FirstPhaseRankBlueprint::prepareSharedState(const fef::IQueryEnvironment&, fef::IObjectStore& store) const +{ + FirstPhaseRankLookup::make_shared_state(store); +} + +fef::FeatureExecutor& +FirstPhaseRankBlueprint::createExecutor(const fef::IQueryEnvironment& env, vespalib::Stash& stash) const +{ + const auto* lookup = FirstPhaseRankLookup::get_shared_state(env.getObjectStore()); + if (lookup != nullptr) { + return stash.create<FirstPhaseRankExecutor>(*lookup); + } else { + std::vector<feature_t> values{std::numeric_limits<feature_t>::max()}; + return stash.create<ValueExecutor>(values); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/features/first_phase_rank_feature.h b/searchlib/src/vespa/searchlib/features/first_phase_rank_feature.h new file mode 100644 index 00000000000..f90ea26f859 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/first_phase_rank_feature.h @@ -0,0 +1,40 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "first_phase_rank_lookup.h" +#include <vespa/searchlib/fef/blueprint.h> +#include <vespa/searchlib/fef/featureexecutor.h> + +namespace search::features { + +class FirstPhaseRankLookup; + +/* + * Executor for first phase rank feature that outputs the first phase rank + * for the given docid on this search node (1.0, 2.0, 3.0, etc.). + */ +class FirstPhaseRankExecutor : public fef::FeatureExecutor { + const FirstPhaseRankLookup& _lookup; +public: + FirstPhaseRankExecutor(const FirstPhaseRankLookup& lookup); + ~FirstPhaseRankExecutor() override; + void execute(uint32_t docid) override; +}; + +/* + * Blueprint for first phase rank feature. + */ +class FirstPhaseRankBlueprint : public fef::Blueprint { +public: + FirstPhaseRankBlueprint(); + ~FirstPhaseRankBlueprint() override; + void visitDumpFeatures(const fef::IIndexEnvironment& env, fef::IDumpFeatureVisitor& visitor) const override; + std::unique_ptr<fef::Blueprint> createInstance() const override; + fef::ParameterDescriptions getDescriptions() const override; + bool setup(const fef::IIndexEnvironment& env, const fef::ParameterList& params) override; + void prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const override; + fef::FeatureExecutor& createExecutor(const fef::IQueryEnvironment& env, vespalib::Stash& stash) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/first_phase_rank_lookup.cpp b/searchlib/src/vespa/searchlib/features/first_phase_rank_lookup.cpp new file mode 100644 index 00000000000..2dfaabb8326 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/first_phase_rank_lookup.cpp @@ -0,0 +1,67 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "first_phase_rank_lookup.h" +#include <vespa/searchlib/fef/objectstore.h> +#include <cassert> +#include <limits> + +using search::fef::AnyWrapper; + +namespace search::features { + +namespace { + +const vespalib::string key = "firstPhaseRankLookup"; + +} + +FirstPhaseRankLookup::FirstPhaseRankLookup() + : _map() +{ +} + +FirstPhaseRankLookup::FirstPhaseRankLookup(FirstPhaseRankLookup&&) = default; + +FirstPhaseRankLookup::~FirstPhaseRankLookup() = default; + +feature_t +FirstPhaseRankLookup::lookup(uint32_t docid) const noexcept +{ + auto itr = _map.find(docid); + if (itr != _map.end()) [[likely]] { + return itr->second; + } else { + return std::numeric_limits<feature_t>::max(); + } +} + +void +FirstPhaseRankLookup::add(uint32_t docid, uint32_t rank) +{ + auto insres = _map.insert(std::make_pair(docid, rank)); + assert(insres.second); +} + +void +FirstPhaseRankLookup::make_shared_state(fef::IObjectStore& store) +{ + if (store.get(key) == nullptr) { + store.add(key, std::make_unique<AnyWrapper<FirstPhaseRankLookup>>(FirstPhaseRankLookup())); + } +} + +FirstPhaseRankLookup* +FirstPhaseRankLookup::get_mutable_shared_state(fef::IObjectStore& store) +{ + auto* wrapper = dynamic_cast<AnyWrapper<FirstPhaseRankLookup>*>(store.get_mutable(key)); + return (wrapper == nullptr) ? nullptr : &wrapper->getValue(); +} + +const FirstPhaseRankLookup* +FirstPhaseRankLookup::get_shared_state(const fef::IObjectStore& store) +{ + const auto* wrapper = dynamic_cast<const AnyWrapper<FirstPhaseRankLookup>*>(store.get(key)); + return (wrapper == nullptr) ? nullptr : &wrapper->getValue(); +} + +} diff --git a/searchlib/src/vespa/searchlib/features/first_phase_rank_lookup.h b/searchlib/src/vespa/searchlib/features/first_phase_rank_lookup.h new file mode 100644 index 00000000000..83d89ed2dd1 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/first_phase_rank_lookup.h @@ -0,0 +1,32 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/common/feature.h> +#include <vespa/vespalib/stllike/hash_map.h> + +namespace search::fef { class IObjectStore; } + +namespace search::features { + +/* + * This class contains a mapping from docids used by second phase to + * first phase rank. + */ +class FirstPhaseRankLookup { + vespalib::hash_map<uint32_t, uint32_t> _map; +public: + FirstPhaseRankLookup(); + FirstPhaseRankLookup(const FirstPhaseRankLookup&) = delete; + FirstPhaseRankLookup(FirstPhaseRankLookup&&); + ~FirstPhaseRankLookup(); + FirstPhaseRankLookup& operator=(const FirstPhaseRankLookup&) = delete; + FirstPhaseRankLookup& operator=(FirstPhaseRankLookup&&) = delete; + feature_t lookup(uint32_t docid) const noexcept; + void add(uint32_t docid, uint32_t rank); + static void make_shared_state(fef::IObjectStore& store); + static FirstPhaseRankLookup* get_mutable_shared_state(fef::IObjectStore& store); + static const FirstPhaseRankLookup* get_shared_state(const fef::IObjectStore& store); +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index bdffbd1c6aa..d65459817f0 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -22,6 +22,7 @@ #include "fieldmatchfeature.h" #include "fieldtermmatchfeature.h" #include "firstphasefeature.h" +#include "first_phase_rank_feature.h" #include "flow_completeness_feature.h" #include "foreachfeature.h" #include "freshnessfeature.h" @@ -91,6 +92,7 @@ void setup_search_features(fef::IBlueprintRegistry & registry) registry.addPrototype(std::make_shared<FieldMatchBlueprint>()); registry.addPrototype(std::make_shared<FieldTermMatchBlueprint>()); registry.addPrototype(std::make_shared<FirstPhaseBlueprint>()); + registry.addPrototype(std::make_shared<FirstPhaseRankBlueprint>()); registry.addPrototype(std::make_shared<FlowCompletenessBlueprint>()); registry.addPrototype(std::make_shared<ForeachBlueprint>()); registry.addPrototype(std::make_shared<FreshnessBlueprint>()); diff --git a/searchlib/src/vespa/searchlib/features/utils.cpp b/searchlib/src/vespa/searchlib/features/utils.cpp index 92758c58262..0c3bbcb0ffa 100644 --- a/searchlib/src/vespa/searchlib/features/utils.cpp +++ b/searchlib/src/vespa/searchlib/features/utils.cpp @@ -1,7 +1,20 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "utils.hpp" +#include <vespa/searchlib/fef/itablemanager.h> +#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/itermdata.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/issue.h> #include <charconv> +#include <cmath> +#include <ostream> + +#include <vespa/log/log.h> +LOG_SETUP(".features.utils"); + +using vespalib::Issue; +using namespace search::fef; namespace search::features::util { @@ -40,4 +53,146 @@ template <> int16_t strToNum<int16_t>(vespalib::stringref str) { return strToIn template <> int32_t strToNum<int32_t>(vespalib::stringref str) { return strToInt<int32_t>(str); } template <> int64_t strToNum<int64_t>(vespalib::stringref str) { return strToInt<int64_t>(str); } +feature_t +lookupConnectedness(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) +{ + if (termId == 0) { + return fallback; // no previous term + } + + const ITermData * data = env.getTerm(termId); + const ITermData * prev = env.getTerm(termId - 1); + if (data == nullptr || prev == nullptr) { + return fallback; // default value + } + return lookupConnectedness(env, data->getUniqueId(), prev->getUniqueId(), fallback); +} + +feature_t +lookupConnectedness(const search::fef::IQueryEnvironment& env, + uint32_t currUniqueId, uint32_t prevUniqueId, feature_t fallback) +{ + // Connectedness of 0.5 between term with unique id 2 and term with unique id 1 is represented as: + // [vespa.term.2.connexity: "1", vespa.term.2.connexity: "0.5"] + vespalib::asciistream os; + os << "vespa.term." << currUniqueId << ".connexity"; + Property p = env.getProperties().lookup(os.str()); + if (p.size() == 2) { + // we have a defined connectedness with the previous term + if (strToNum<uint32_t>(p.getAt(0)) == prevUniqueId) { + return strToNum<feature_t>(p.getAt(1)); + } + } + return fallback; +} + +feature_t +lookupSignificance(const search::fef::IQueryEnvironment& env, const ITermData& term, feature_t fallback) +{ + // Significance of 0.5 for term with unique id 1 is represented as: + // [vespa.term.1.significance: "0.5"] + vespalib::asciistream os; + os << "vespa.term." << term.getUniqueId() << ".significance"; + Property p = env.getProperties().lookup(os.str()); + if (p.found()) { + return strToNum<feature_t>(p.get()); + } + return fallback; +} + +feature_t +lookupSignificance(const search::fef::IQueryEnvironment& env, uint32_t termId, feature_t fallback) +{ + const ITermData* term = env.getTerm(termId); + if (term == nullptr) { + return fallback; + } + return lookupSignificance(env, *term, fallback); +} + +double +getRobertsonSparckJonesWeight(double docCount, double docsInCorpus) +{ + return std::log((docsInCorpus - docCount + 0.5)/(docCount + 0.5)); +} + +static const double N = 1000000.0; + +feature_t +getSignificance(double docFreq) +{ + if (docFreq < (1.0/N)) { + docFreq = 1.0/N; + } + if (docFreq > 1.0) { + docFreq = 1.0; + } + double d = std::log(docFreq)/std::log(1.0/N); + return 0.5 + 0.5 * d; +#if 0 + double n = docFreq * N; + n = (n == 0) ? 1 : (n > N ? N : n); + double a = getRobertsonSparckJonesWeight(1, N + 1); + double b = getRobertsonSparckJonesWeight(N + 1, N + 1); + double w = getRobertsonSparckJonesWeight(n, N + 1); + return ((w - b)/(a - b)); +#endif +} + +feature_t +getSignificance(const search::fef::ITermData& termData) +{ + using FRA = search::fef::ITermFieldRangeAdapter; + double df = 0; + for (FRA iter(termData); iter.valid(); iter.next()) { + df = std::max(df, iter.get().getDocFreq()); + } + + feature_t signif = getSignificance(df); + LOG(debug, "getSignificance %e %f [ %e %f ] = %e", df, df, df * N, df * N, signif); + return signif; +} + +const search::fef::Table * +lookupTable(const search::fef::IIndexEnvironment & env, const vespalib::string & featureName, + const vespalib::string & table, const vespalib::string & fieldName, const vespalib::string & fallback) +{ + vespalib::string tn1 = env.getProperties().lookup(featureName, table).get(fallback); + vespalib::string tn2 = env.getProperties().lookup(featureName, table, fieldName).get(tn1); + const search::fef::Table * retval = env.getTableManager().getTable(tn2); + if (retval == nullptr) { + LOG(warning, "Could not find the %s '%s' to be used for field '%s' in feature '%s'", + table.c_str(), tn2.c_str(), fieldName.c_str(), featureName.c_str()); + } + return retval; +} + +const search::fef::ITermData * +getTermByLabel(const search::fef::IQueryEnvironment &env, const vespalib::string &label) +{ + // Labeling the query item with unique id '5' with the label 'foo' + // is represented as: [vespa.label.foo.id: "5"] + vespalib::asciistream os; + os << "vespa.label." << label << ".id"; + Property p = env.getProperties().lookup(os.str()); + if (!p.found()) { + return 0; + } + uint32_t uid = strToNum<uint32_t>(p.get()); + if (uid == 0) { + Issue::report("Query label '%s' was attached to invalid unique id: '%s'", + label.c_str(), p.get().c_str()); + return 0; + } + for (uint32_t i(0), m(env.getNumTerms()); i < m; ++i) { + const ITermData *term = env.getTerm(i); + if (term->getUniqueId() == uid) { + return term; + } + } + Issue::report("Query label '%s' was attached to non-existing unique id: '%s'", + label.c_str(), p.get().c_str()); + return 0; +} + } diff --git a/searchlib/src/vespa/searchlib/features/utils.h b/searchlib/src/vespa/searchlib/features/utils.h index b7b84013630..e90592fbe5d 100644 --- a/searchlib/src/vespa/searchlib/features/utils.h +++ b/searchlib/src/vespa/searchlib/features/utils.h @@ -71,22 +71,6 @@ inline feature_t getAsFeature<vespalib::stringref>(vespalib::stringref value) { return vespalib::hash2d(value); } - -/** - * This method inputs a value to cap to the range [capFloor, capCeil] and then normalize this - * value to the unit range [0, 1]. - * - * @param val The value to unit normalize. - * @param capFloor The minimum value of the cap range. - * @param capCeil The maximum value of the cap range. - * @return The unit normalized value. - */ -template <typename T> -T unitNormalize(const T &val, const T &capFloor, const T &capCeil) -{ - return (std::max(capFloor, std::min(capCeil, val)) - capFloor) / (capCeil - capFloor); -} - /** * Returns the normalized strength with which the given term is connected to the previous term in the query. * Uses the property map of the query environment to lookup this data. diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index 1f88c34bef3..3ca53d3f777 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -36,14 +36,20 @@ lookupStringVector(const Properties &props, const vespalib::string &name, return defaultValue; } -double -lookupDouble(const Properties &props, const vespalib::string &name, double defaultValue) +std::optional<double> +lookup_opt_double(const Properties &props, vespalib::stringref name, std::optional<double> default_value) { Property p = props.lookup(name); if (p.found()) { return vespalib::locale::c::strtod(p.get().c_str(), nullptr); } - return defaultValue; + return default_value; +} + +double +lookupDouble(const Properties &props, const vespalib::string &name, double defaultValue) +{ + return lookup_opt_double(props, name, defaultValue).value(); } uint32_t @@ -180,7 +186,7 @@ namespace onsummary { namespace temporary { const vespalib::string WeakAndRange::NAME("vespa.weakand.range"); -const double WeakAndRange::DEFAULT_VALUE(0.0); +const double WeakAndRange::DEFAULT_VALUE(1.0); double WeakAndRange::lookup(const Properties &props) @@ -683,19 +689,34 @@ EstimateLimit::lookup(const Properties &props) return lookupUint32(props, NAME, DEFAULT_VALUE); } -const vespalib::string RankScoreDropLimit::NAME("vespa.hitcollector.rankscoredroplimit"); -const feature_t RankScoreDropLimit::DEFAULT_VALUE(-std::numeric_limits<feature_t>::quiet_NaN()); +const vespalib::string FirstPhaseRankScoreDropLimit::NAME("vespa.hitcollector.rankscoredroplimit"); +const std::optional<feature_t> FirstPhaseRankScoreDropLimit::DEFAULT_VALUE(std::nullopt); -feature_t -RankScoreDropLimit::lookup(const Properties &props) +std::optional<feature_t> +FirstPhaseRankScoreDropLimit::lookup(const Properties &props) { return lookup(props, DEFAULT_VALUE); } -feature_t -RankScoreDropLimit::lookup(const Properties &props, feature_t defaultValue) +std::optional<feature_t> +FirstPhaseRankScoreDropLimit::lookup(const Properties &props, std::optional<feature_t> default_value) { - return lookupDouble(props, NAME, defaultValue); + return lookup_opt_double(props, NAME, default_value); +} + +const vespalib::string SecondPhaseRankScoreDropLimit::NAME("vespa.hitcollector.secondphase.rankscoredroplimit"); +const std::optional<feature_t> SecondPhaseRankScoreDropLimit::DEFAULT_VALUE(std::nullopt); + +std::optional<feature_t> +SecondPhaseRankScoreDropLimit::lookup(const Properties &props) +{ + return lookup_opt_double(props, NAME, DEFAULT_VALUE); +} + +std::optional<feature_t> +SecondPhaseRankScoreDropLimit::lookup(const Properties &props, std::optional<feature_t> default_value) +{ + return lookup_opt_double(props, NAME, default_value); } } // namspace hitcollector diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index d047eb13347..afaf7ac1a61 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -5,6 +5,7 @@ #include <vespa/searchlib/common/feature.h> #include <vespa/vespalib/fuzzy/fuzzy_matching_algorithm.h> #include <vespa/vespalib/stllike/string.h> +#include <optional> #include <vector> namespace search::fef { class Properties; } @@ -562,16 +563,28 @@ namespace hitcollector { }; /** - * Property for the rank score drop limit used in parallel query evaluation. - * Drop a hit if the rank score <= drop limit. + * Property for the first phase rank score drop limit used in parallel + * query evaluation. + * Drop a hit if the first phase rank score <= drop limit. **/ - struct RankScoreDropLimit { + struct FirstPhaseRankScoreDropLimit { static const vespalib::string NAME; - static const feature_t DEFAULT_VALUE; - static feature_t lookup(const Properties &props); - static feature_t lookup(const Properties &props, feature_t defaultValue); + static const std::optional<feature_t> DEFAULT_VALUE; + static std::optional<feature_t> lookup(const Properties &props); + static std::optional<feature_t> lookup(const Properties &props, std::optional<feature_t> default_value); }; + /** + * Property for the second phase rank score drop limit used in + * parallel query evaluation. Drop a hit if the score (reranked or + * rescored) <= drop limit. + **/ + struct SecondPhaseRankScoreDropLimit { + static const vespalib::string NAME; + static const std::optional<feature_t> DEFAULT_VALUE; + static std::optional<feature_t> lookup(const Properties &props); + static std::optional<feature_t> lookup(const Properties &props, std::optional<double> default_value); + }; } // namespace hitcollector diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.cpp b/searchlib/src/vespa/searchlib/fef/objectstore.cpp index 3e5baf49116..a90702a88a6 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.cpp +++ b/searchlib/src/vespa/searchlib/fef/objectstore.cpp @@ -35,4 +35,11 @@ ObjectStore::get(const vespalib::string & key) const return (found != _objectMap.end()) ? found->second : NULL; } +Anything * +ObjectStore::get_mutable(const vespalib::string& key) +{ + auto found = _objectMap.find(key); + return (found != _objectMap.end()) ? found->second : nullptr; +} + } diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.h b/searchlib/src/vespa/searchlib/fef/objectstore.h index 9d1671e521c..d2d768ee338 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.h +++ b/searchlib/src/vespa/searchlib/fef/objectstore.h @@ -24,6 +24,7 @@ class AnyWrapper : public Anything public: explicit AnyWrapper(T value) : _value(std::move(value)) { } const T & getValue() const { return _value; } + T& getValue() { return _value; } static const T & getValue(const Anything & any) { return static_cast<const AnyWrapper &>(any).getValue(); } private: T _value; @@ -38,6 +39,7 @@ public: virtual ~IObjectStore() = default; virtual void add(const vespalib::string & key, Anything::UP value) = 0; virtual const Anything * get(const vespalib::string & key) const = 0; + virtual Anything* get_mutable(const vespalib::string& key) = 0; }; /** @@ -50,6 +52,7 @@ public: ~ObjectStore() override; void add(const vespalib::string & key, Anything::UP value) override; const Anything * get(const vespalib::string & key) const override; + Anything* get_mutable(const vespalib::string & key) override; private: using ObjectMap = vespalib::hash_map<vespalib::string, Anything *>; ObjectMap _objectMap; diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index ba5abb35141..f62dda66b76 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -50,7 +50,8 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _degradationMaxFilterCoverage(1.0), _degradationSamplePercentage(0.2), _degradationPostFilterMultiplier(1.0), - _rankScoreDropLimit(0), + _first_phase_rank_score_drop_limit(), + _second_phase_rank_score_drop_limit(), _match_features(), _summaryFeatures(), _dumpFeatures(), @@ -120,7 +121,8 @@ RankSetup::configure() setDiversityCutoffStrategy(matchphase::DiversityCutoffStrategy::lookup(_indexEnv.getProperties())); setEstimatePoint(hitcollector::EstimatePoint::lookup(_indexEnv.getProperties())); setEstimateLimit(hitcollector::EstimateLimit::lookup(_indexEnv.getProperties())); - setRankScoreDropLimit(hitcollector::RankScoreDropLimit::lookup(_indexEnv.getProperties())); + set_first_phase_rank_score_drop_limit(hitcollector::FirstPhaseRankScoreDropLimit::lookup(_indexEnv.getProperties())); + set_second_phase_rank_score_drop_limit(hitcollector::SecondPhaseRankScoreDropLimit::lookup(_indexEnv.getProperties())); setSoftTimeoutEnabled(softtimeout::Enabled::lookup(_indexEnv.getProperties())); setSoftTimeoutTailCost(softtimeout::TailCost::lookup(_indexEnv.getProperties())); set_global_filter_lower_limit(matching::GlobalFilterLowerLimit::lookup(_indexEnv.getProperties())); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index f20ecd4b42b..b5309b128e2 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -9,6 +9,7 @@ #include "rank_program.h" #include <vespa/searchlib/common/stringmap.h> #include <vespa/vespalib/fuzzy/fuzzy_matching_algorithm.h> +#include <optional> namespace search::fef { @@ -59,7 +60,8 @@ private: double _degradationMaxFilterCoverage; double _degradationSamplePercentage; double _degradationPostFilterMultiplier; - feature_t _rankScoreDropLimit; + std::optional<feature_t> _first_phase_rank_score_drop_limit; + std::optional<feature_t> _second_phase_rank_score_drop_limit; std::vector<vespalib::string> _match_features; std::vector<vespalib::string> _summaryFeatures; std::vector<vespalib::string> _dumpFeatures; @@ -332,18 +334,22 @@ public: uint32_t getEstimateLimit() const { return _estimateLimit; } /** - * Sets the rank score drop limit to be used in parallel query evaluation. + * Sets the first phase rank score drop limit to be used in parallel query evaluation. * - * @param rankScoreDropLimit the rank score drop limit + * @param value the first phase rank score drop limit **/ - void setRankScoreDropLimit(feature_t rankScoreDropLimit) { _rankScoreDropLimit = rankScoreDropLimit; } + void set_first_phase_rank_score_drop_limit(std::optional<feature_t> value) { _first_phase_rank_score_drop_limit = value; } /** * Returns the rank score drop limit to be used in parallel query evaluation. * * @return the rank score drop limit **/ - feature_t getRankScoreDropLimit() const { return _rankScoreDropLimit; } + std::optional<feature_t> get_first_phase_rank_score_drop_limit() const noexcept { return _first_phase_rank_score_drop_limit; } + + void set_second_phase_rank_score_drop_limit(std::optional<feature_t> value) { _second_phase_rank_score_drop_limit = value; } + + std::optional<feature_t> get_second_phase_rank_score_drop_limit() const noexcept { return _second_phase_rank_score_drop_limit; } /** * This method may be used to indicate that certain features diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index 116e677d439..3da68db4c34 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -8,6 +8,7 @@ #include "queryvisitor.h" #include "string_term_vector.h" #include "termnodes.h" +#include <cassert> namespace search::query { @@ -29,8 +30,8 @@ public: private: void visitNodes(const std::vector<Node *> &nodes) { - for (size_t i = 0; i < nodes.size(); ++i) { - nodes[i]->accept(*this); + for (auto node : nodes) { + node->accept(*this); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index 51fe2d12637..31799b6935c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(searchlib_queryeval OBJECT emptysearch.cpp equiv_blueprint.cpp equivsearch.cpp + exact_nearest_neighbor_iterator.cpp executeinfo.cpp fake_requestcontext.cpp fake_result.cpp @@ -21,6 +22,7 @@ vespa_add_library(searchlib_queryeval OBJECT fake_searchable.cpp field_spec.cpp filter_wrapper.cpp + first_phase_rescorer.cpp flow.cpp full_search.cpp get_weight_from_node.cpp @@ -37,7 +39,6 @@ vespa_add_library(searchlib_queryeval OBJECT multibitvectoriterator.cpp multisearch.cpp nearest_neighbor_blueprint.cpp - nearest_neighbor_iterator.cpp nearsearch.cpp nns_index_iterator.cpp orsearch.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 412a5973ad8..f86edfc3faa 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -122,6 +122,7 @@ Blueprint::Blueprint() noexcept _flow_stats(0.0, 0.0, 0.0), _sourceId(0xffffffff), _docid_limit(0), + _id(0), _strict(false), _frozen(false) { @@ -141,6 +142,13 @@ Blueprint::resolve_strict(InFlow &in_flow) noexcept _strict = in_flow.strict(); } +uint32_t +Blueprint::enumerate(uint32_t next_id) noexcept +{ + set_id(next_id++); + return next_id; +} + void Blueprint::each_node_post_order(const std::function<void(Blueprint&)> &f) { @@ -254,8 +262,8 @@ create_op_filter(const Blueprint::Children &children, bool strict, Blueprint::Fi MultiSearch::Children list; std::unique_ptr<SearchIterator> spare; list.reserve(children.size()); - for (size_t i = 0; i < children.size(); ++i) { - auto filter = children[i]->createFilterSearch(constraint); + for (const auto & child : children) { + auto filter = child->createFilterSearch(constraint); auto matches_any = filter->matches_any(); if (should_short_circuit<Op>(matches_any)) { return filter; @@ -411,6 +419,7 @@ Blueprint::visitMembers(vespalib::ObjectVisitor &visitor) const visitor.visitFloat("strict_cost", strict_cost()); visitor.visitInt("sourceId", _sourceId); visitor.visitInt("docid_limit", _docid_limit); + visitor.visitInt("id", _id); visitor.visitBool("strict", _strict); } @@ -450,11 +459,21 @@ IntermediateBlueprint::setDocIdLimit(uint32_t limit) noexcept } } +uint32_t +IntermediateBlueprint::enumerate(uint32_t next_id) noexcept +{ + set_id(next_id++); + for (Blueprint::UP &child: _children) { + next_id = child->enumerate(next_id); + } + return next_id; +} + void IntermediateBlueprint::each_node_post_order(const std::function<void(Blueprint&)> &f) { - for (Blueprint::UP &child : _children) { - f(*child); + for (Blueprint::UP &child: _children) { + child->each_node_post_order(f); } f(*this); } @@ -623,9 +642,9 @@ IntermediateBlueprint::sort(InFlow in_flow) sort(_children, in_flow); } auto flow = my_flow(in_flow); - for (size_t i = 0; i < _children.size(); ++i) { - _children[i]->sort(InFlow(flow.strict(), flow.flow())); - flow.add(_children[i]->estimate()); + for (const auto & child : _children) { + child->sort(InFlow(flow.strict(), flow.flow())); + flow.add(child->estimate()); } } @@ -644,8 +663,8 @@ IntermediateBlueprint::createSearch(fef::MatchData &md) const { MultiSearch::Children subSearches; subSearches.reserve(_children.size()); - for (size_t i = 0; i < _children.size(); ++i) { - subSearches.push_back(_children[i]->createSearch(md)); + for (const auto & child : _children) { + subSearches.push_back(child->createSearch(md)); } return createIntermediateSearch(std::move(subSearches), md); } @@ -693,23 +712,30 @@ void IntermediateBlueprint::fetchPostings(const ExecuteInfo &execInfo) { auto flow = my_flow(InFlow(strict(), execInfo.hit_rate())); - for (size_t i = 0; i < _children.size(); ++i) { + for (const auto & child : _children) { double nextHitRate = flow.flow(); - Blueprint & child = *_children[i]; - child.fetchPostings(ExecuteInfo::create(nextHitRate, execInfo)); - flow.add(child.estimate()); + child->fetchPostings(ExecuteInfo::create(nextHitRate, execInfo)); + flow.add(child->estimate()); } } void IntermediateBlueprint::freeze() { - for (Blueprint::UP &child: _children) { + for (auto &child: _children) { child->freeze(); } freeze_self(); } +void +IntermediateBlueprint::set_matching_phase(MatchingPhase matching_phase) noexcept +{ + for (auto &child : _children) { + child->set_matching_phase(matching_phase); + } +} + namespace { bool @@ -784,6 +810,11 @@ LeafBlueprint::freeze() freeze_self(); } +void +LeafBlueprint::set_matching_phase(MatchingPhase) noexcept +{ +} + SearchIterator::UP LeafBlueprint::createSearch(fef::MatchData &md) const { diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index a443f34f856..5e156853ffb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -7,6 +7,7 @@ #include "unpackinfo.h" #include "executeinfo.h" #include "global_filter.h" +#include "matching_phase.h" #include "multisearch.h" #include <vespa/searchlib/common/bitvector.h> @@ -240,6 +241,7 @@ private: FlowStats _flow_stats; uint32_t _sourceId; uint32_t _docid_limit; + uint32_t _id; bool _strict; bool _frozen; @@ -288,6 +290,10 @@ public: virtual void setDocIdLimit(uint32_t limit) noexcept { _docid_limit = limit; } uint32_t get_docid_limit() const noexcept { return _docid_limit; } + void set_id(uint32_t value) noexcept { _id = value; } + uint32_t id() const noexcept { return _id; } + virtual uint32_t enumerate(uint32_t next_id) noexcept; + bool strict() const noexcept { return _strict; } virtual void each_node_post_order(const std::function<void(Blueprint&)> &f); @@ -382,6 +388,8 @@ public: virtual void freeze() = 0; bool frozen() const { return _frozen; } + virtual void set_matching_phase(MatchingPhase matching_phase) noexcept = 0; + virtual SearchIteratorUP createSearch(fef::MatchData &md) const = 0; virtual SearchIteratorUP createFilterSearch(FilterConstraint constraint) const = 0; static SearchIteratorUP create_and_filter(const Children &children, bool strict, FilterConstraint constraint); @@ -480,6 +488,7 @@ public: ~IntermediateBlueprint() override; void setDocIdLimit(uint32_t limit) noexcept final; + uint32_t enumerate(uint32_t next_id) noexcept override; void each_node_post_order(const std::function<void(Blueprint&)> &f) override; void optimize(Blueprint* &self, OptimizePass pass) final; @@ -506,6 +515,7 @@ public: void visitMembers(vespalib::ObjectVisitor &visitor) const override; void fetchPostings(const ExecuteInfo &execInfo) override; void freeze() final; + void set_matching_phase(MatchingPhase matching_phase) noexcept override; UnpackInfo calculateUnpackInfo(const fef::MatchData & md) const; IntermediateBlueprint * asIntermediate() noexcept final { return this; } @@ -552,6 +562,7 @@ public: const State &getState() const final { return _state; } void fetchPostings(const ExecuteInfo &execInfo) override; void freeze() final; + void set_matching_phase(MatchingPhase matching_phase) noexcept override; SearchIteratorUP createSearch(fef::MatchData &md) const override; const LeafBlueprint * asLeaf() const noexcept final { return this; } diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp index 27ff0d235a3..c4aea7deae8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp @@ -22,6 +22,11 @@ CreateBlueprintVisitorHelper::CreateBlueprintVisitorHelper(Searchable &searchabl CreateBlueprintVisitorHelper::~CreateBlueprintVisitorHelper() = default; +bool +CreateBlueprintVisitorHelper::is_search_multi_threaded() const noexcept { + return getRequestContext().thread_bundle().size() > 1; +} + attribute::SearchContextParams CreateBlueprintVisitorHelper::createContextParams() const { return attribute::SearchContextParams().metaStoreReadGuard(_requestContext.getMetaStoreReadGuard()); @@ -104,7 +109,8 @@ void CreateBlueprintVisitorHelper::visitWandTerm(query::WandTerm &n) { createWeightedSet(std::make_unique<ParallelWeakAndBlueprint>(_field, n.getTargetNumHits(), - n.getScoreThreshold(), n.getThresholdBoostFactor()), + n.getScoreThreshold(), n.getThresholdBoostFactor(), + is_search_multi_threaded()), n); } diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h index 98f62fa3249..ec163260dc3 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h @@ -29,6 +29,7 @@ protected: const IRequestContext & getRequestContext() const { return _requestContext; } attribute::SearchContextParams createContextParams() const; attribute::SearchContextParams createContextParams(bool isFilter) const; + bool is_search_multi_threaded() const noexcept; public: CreateBlueprintVisitorHelper(Searchable &searchable, const FieldSpec &field, const IRequestContext & requestContext); ~CreateBlueprintVisitorHelper() override; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/exact_nearest_neighbor_iterator.cpp index c76fe3363e4..e5e4a0edee1 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/exact_nearest_neighbor_iterator.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "nearest_neighbor_iterator.h" +#include "exact_nearest_neighbor_iterator.h" #include "global_filter.h" #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/tensor/distance_calculator.h> @@ -20,16 +20,17 @@ namespace search::queryeval { * Currently always does brute-force scanning, which is very expensive. **/ template <bool strict, bool has_filter, bool has_single_subspace> -class NearestNeighborImpl final : public NearestNeighborIterator +class ExactNearestNeighborImpl final : public ExactNearestNeighborIterator { public: - explicit NearestNeighborImpl(Params params_in) - : NearestNeighborIterator(std::move(params_in)), - _lastScore(0.0) + explicit ExactNearestNeighborImpl(bool readonly_distance_heap, Params params_in) + : ExactNearestNeighborIterator(std::move(params_in)), + _lastScore(0.0), + _readonly_distance_heap(readonly_distance_heap) { } - ~NearestNeighborImpl() override; + ~ExactNearestNeighborImpl() override; void doSeek(uint32_t docId) override { double distanceLimit = params().distanceHeap.distanceLimit(); @@ -54,7 +55,9 @@ public: void doUnpack(uint32_t docId) override { double score = params().distance_calc->function().to_rawscore(_lastScore); params().tfmd.setRawScore(docId, score); - params().distanceHeap.used(_lastScore); + if (!_readonly_distance_heap) { + params().distanceHeap.used(_lastScore); + } } Trinary is_strict() const override { return strict ? Trinary::True : Trinary::False ; } @@ -65,49 +68,51 @@ private: } double _lastScore; + const bool _readonly_distance_heap; }; template <bool strict, bool has_filter, bool has_single_subspace> -NearestNeighborImpl<strict, has_filter, has_single_subspace>::~NearestNeighborImpl() = default; +ExactNearestNeighborImpl<strict, has_filter, has_single_subspace>::~ExactNearestNeighborImpl() = default; namespace { template <bool strict, bool has_filter> -std::unique_ptr<NearestNeighborIterator> -resolve_single_subspace(NearestNeighborIterator::Params params) +std::unique_ptr<ExactNearestNeighborIterator> +resolve_single_subspace(bool readonly_distance_heap, ExactNearestNeighborIterator::Params params) { if (params.distance_calc->has_single_subspace()) { - using NNI = NearestNeighborImpl<strict, has_filter, true>; - return std::make_unique<NNI>(std::move(params)); + using NNI = ExactNearestNeighborImpl<strict, has_filter, true>; + return std::make_unique<NNI>(readonly_distance_heap, std::move(params)); } else { - using NNI = NearestNeighborImpl<strict, has_filter, false>; - return std::make_unique<NNI>(std::move(params)); + using NNI = ExactNearestNeighborImpl<strict, has_filter, false>; + return std::make_unique<NNI>(readonly_distance_heap, std::move(params)); } } template <bool has_filter> -std::unique_ptr<NearestNeighborIterator> -resolve_strict(bool strict, NearestNeighborIterator::Params params) +std::unique_ptr<ExactNearestNeighborIterator> +resolve_strict(bool strict, bool readonly_distance_heap, ExactNearestNeighborIterator::Params params) { if (strict) { - return resolve_single_subspace<true, has_filter>(std::move(params)); + return resolve_single_subspace<true, has_filter>(readonly_distance_heap, std::move(params)); } else { - return resolve_single_subspace<false, has_filter>(std::move(params)); + return resolve_single_subspace<false, has_filter>(readonly_distance_heap, std::move(params)); } } } // namespace <unnamed> -std::unique_ptr<NearestNeighborIterator> -NearestNeighborIterator::create(bool strict, fef::TermFieldMatchData &tfmd, - std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, - NearestNeighborDistanceHeap &distanceHeap, const GlobalFilter &filter) +std::unique_ptr<ExactNearestNeighborIterator> +ExactNearestNeighborIterator::create(bool strict, fef::TermFieldMatchData &tfmd, + std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, + NearestNeighborDistanceHeap &distanceHeap, const GlobalFilter &filter, + bool readonly_distance_heap) { Params params(tfmd, std::move(distance_calc), distanceHeap, filter); if (filter.is_active()) { - return resolve_strict<true>(strict, std::move(params)); + return resolve_strict<true>(strict, readonly_distance_heap, std::move(params)); } else { - return resolve_strict<false>(strict, std::move(params)); + return resolve_strict<false>(strict, readonly_distance_heap, std::move(params)); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/exact_nearest_neighbor_iterator.h index 177c732a44d..2937c0c4abf 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/exact_nearest_neighbor_iterator.h @@ -16,7 +16,7 @@ namespace search::queryeval { class GlobalFilter; -class NearestNeighborIterator : public SearchIterator +class ExactNearestNeighborIterator : public SearchIterator { public: using ITensorAttribute = search::tensor::ITensorAttribute; @@ -39,16 +39,17 @@ public: {} }; - explicit NearestNeighborIterator(Params params_in) + explicit ExactNearestNeighborIterator(Params params_in) : _params(std::move(params_in)) {} - static std::unique_ptr<NearestNeighborIterator> create( + static std::unique_ptr<ExactNearestNeighborIterator> create( bool strict, fef::TermFieldMatchData &tfmd, std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, NearestNeighborDistanceHeap &distanceHeap, - const GlobalFilter &filter); + const GlobalFilter &filter, + bool readonly_distance_heap); const Params& params() const { return _params; } private: diff --git a/searchlib/src/vespa/searchlib/queryeval/first_phase_rescorer.cpp b/searchlib/src/vespa/searchlib/queryeval/first_phase_rescorer.cpp new file mode 100644 index 00000000000..a7b1e3a7c92 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/first_phase_rescorer.cpp @@ -0,0 +1,38 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "first_phase_rescorer.h" + +namespace search::queryeval { + +FirstPhaseRescorer::FirstPhaseRescorer(const std::pair<Scores,Scores>& ranges) + : _scale(1.0), + _adjust(0.0) +{ + if (need_rescore(ranges)) { + auto& first_phase_scores = ranges.first; + auto& second_phase_scores = ranges.second; + // scale and adjust the first phase score according to the + // first phase and second phase heap score values to avoid that + // a score from the first phase is larger than second_phase_scores.low + double first_phase_range = first_phase_scores.high - first_phase_scores.low; + if (first_phase_range < 1.0) { + first_phase_range = 1.0; + } + double second_phase_range = second_phase_scores.high - second_phase_scores.low; + if (second_phase_range < 1.0) { + second_phase_range = 1.0; + } + _scale = second_phase_range / first_phase_range; + _adjust = first_phase_scores.low * _scale - second_phase_scores.low; + } +} + +bool +FirstPhaseRescorer::need_rescore(const std::pair<Scores,Scores>& ranges) +{ + auto& first_phase_scores = ranges.first; + auto& second_phase_scores = ranges.second; + return (first_phase_scores.low > second_phase_scores.low); +} + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/first_phase_rescorer.h b/searchlib/src/vespa/searchlib/queryeval/first_phase_rescorer.h new file mode 100644 index 00000000000..301e2aa78d0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/first_phase_rescorer.h @@ -0,0 +1,25 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "scores.h" +#include <cstdint> + +namespace search::queryeval { + +/* + * Rescore hits not selected for second phase to prevent them from getting + * a better score than hits selected for second phase ranking. + */ +class FirstPhaseRescorer { + double _scale; + double _adjust; +public: + FirstPhaseRescorer(const std::pair<Scores,Scores>& ranges); + static bool need_rescore(const std::pair<Scores,Scores>& ranges); + double rescore(uint32_t, double score) const noexcept { + return ((score * _scale) - _adjust); + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp index bf7f44f0e7a..01587ef485a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "hitcollector.h" +#include "first_phase_rescorer.h" #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/sort.h> #include <cassert> @@ -43,9 +44,7 @@ HitCollector::HitCollector(uint32_t numDocs, uint32_t maxHitsSize) _unordered(false), _docIdVector(), _bitVector(), - _reRankedHits(), - _scale(1.0), - _adjust(0) + _reRankedHits() { if (_maxHitsSize > 0) { _collector = std::make_unique<RankedHitCollector>(*this); @@ -71,7 +70,7 @@ HitCollector::RankedHitCollector::collect(uint32_t docId, feature_t score) } hc._hits.emplace_back(docId, score); } else { - collectAndChangeCollector(docId, score); + collectAndChangeCollector(docId, score); // note - self-destruct. } } @@ -101,11 +100,10 @@ HitCollector::RankedHitCollector::collectAndChangeCollector(uint32_t docId, feat if (hc._maxDocIdVectorSize > hc._maxHitsSize) { // start using docid vector hc._docIdVector.reserve(hc._maxDocIdVectorSize); - uint32_t iSize = hc._hits.size(); - for (uint32_t i = 0; i < iSize; ++i) { - hc._docIdVector.push_back(hc._hits[i].first); + for (const auto& hit : hc._hits) { + hc._docIdVector.push_back(hit.first); } - if ((iSize > 0) && (docId < hc._docIdVector.back())) { + if (!hc._docIdVector.empty() && (docId < hc._docIdVector.back())) { hc._unordered = true; } hc._docIdVector.push_back(docId); @@ -114,9 +112,8 @@ HitCollector::RankedHitCollector::collectAndChangeCollector(uint32_t docId, feat // start using bit vector hc._bitVector = BitVector::create(hc._numDocs); hc._bitVector->invalidateCachedCount(); - uint32_t iSize = hc._hits.size(); - for (uint32_t i = 0; i < iSize; ++i) { - hc._bitVector->setBit(hc._hits[i].first); + for (const auto& hit : _hc._hits) { + hc._bitVector->setBit(hit.first); } hc._bitVector->setBit(docId); newCollector = std::make_unique<BitVectorCollector<true>>(hc); @@ -125,7 +122,7 @@ HitCollector::RankedHitCollector::collectAndChangeCollector(uint32_t docId, feat std::make_heap(hc._hits.begin(), hc._hits.end(), ScoreComparator()); hc._hitsSortOrder = SortOrder::HEAP; this->considerForHitVector(docId, score); - hc._collector = std::move(newCollector); + hc._collector = std::move(newCollector); // note - self-destruct. } template<bool CollectRankedHit> @@ -145,7 +142,7 @@ HitCollector::DocIdCollector<CollectRankedHit>::collect(uint32_t docId, feature_ } hc._docIdVector.push_back(docId); } else { - collectAndChangeCollector(docId); + collectAndChangeCollector(docId); // note - self-destruct. } } @@ -157,9 +154,8 @@ HitCollector::DocIdCollector<CollectRankedHit>::collectAndChangeCollector(uint32 // start using bit vector instead of docid array. hc._bitVector = BitVector::create(hc._numDocs); hc._bitVector->invalidateCachedCount(); - uint32_t iSize = static_cast<uint32_t>(hc._docIdVector.size()); - for (uint32_t i = 0; i < iSize; ++i) { - hc._bitVector->setBit(hc._docIdVector[i]); + for (auto docid : hc._docIdVector) { + hc._bitVector->setBit(docid); } std::vector<uint32_t> emptyVector; emptyVector.swap(hc._docIdVector); @@ -191,91 +187,231 @@ HitCollector::setRanges(const std::pair<Scores, Scores> &ranges) namespace { +struct NoRescorer +{ + static double rescore(uint32_t, double score) noexcept { return score; } +}; + +template <typename Rescorer> +class RerankRescorer { + Rescorer _rescorer; + using HitVector = std::vector<HitCollector::Hit>; + using Iterator = typename HitVector::const_iterator; + Iterator _reranked_cur; + Iterator _reranked_end; +public: + RerankRescorer(const Rescorer& rescorer, + const HitVector& reranked_hits) + : _rescorer(rescorer), + _reranked_cur(reranked_hits.begin()), + _reranked_end(reranked_hits.end()) + { + } + + double rescore(uint32_t docid, double score) noexcept { + if (_reranked_cur != _reranked_end && _reranked_cur->first == docid) { + double result = _reranked_cur->second; + ++_reranked_cur; + return result; + } else { + return _rescorer.rescore(docid, score); + } + } +}; + +class SimpleHitAdder { +protected: + ResultSet& _rs; +public: + SimpleHitAdder(ResultSet& rs) + : _rs(rs) + { + } + void add(uint32_t docid, double rank_value) { + _rs.push_back({docid, rank_value}); + } +}; + +class ConditionalHitAdder : public SimpleHitAdder { +protected: + double _second_phase_rank_drop_limit; +public: + ConditionalHitAdder(ResultSet& rs, double second_phase_rank_drop_limit) + : SimpleHitAdder(rs), + _second_phase_rank_drop_limit(second_phase_rank_drop_limit) + { + } + void add(uint32_t docid, double rank_value) { + if (rank_value > _second_phase_rank_drop_limit) { + _rs.push_back({docid, rank_value}); + } + } +}; + +class TrackingConditionalHitAdder : public ConditionalHitAdder { + std::vector<uint32_t>& _dropped; +public: + TrackingConditionalHitAdder(ResultSet& rs, double second_phase_rank_drop_limit, std::vector<uint32_t>& dropped) + : ConditionalHitAdder(rs, second_phase_rank_drop_limit), + _dropped(dropped) + { + } + void add(uint32_t docid, double rank_value) { + if (rank_value > _second_phase_rank_drop_limit) { + _rs.push_back({docid, rank_value}); + } else { + _dropped.emplace_back(docid); + } + } +}; + +template <typename HitAdder, typename Rescorer> void -mergeHitsIntoResultSet(const std::vector<HitCollector::Hit> &hits, ResultSet &result) +add_rescored_hits(HitAdder hit_adder, const std::vector<HitCollector::Hit>& hits, Rescorer rescorer) { - uint32_t rhCur(0); - uint32_t rhEnd(result.getArrayUsed()); - for (const auto &hit : hits) { - while (rhCur != rhEnd && result[rhCur].getDocId() != hit.first) { - // just set the iterators right - ++rhCur; + for (auto& hit : hits) { + hit_adder.add(hit.first, rescorer.rescore(hit.first, hit.second)); + } +} + +template <typename HitAdder, typename Rescorer> +void +add_rescored_hits(HitAdder hit_adder, const std::vector<HitCollector::Hit>& hits, const std::vector<HitCollector::Hit>& reranked_hits, Rescorer rescorer) +{ + if (reranked_hits.empty()) { + add_rescored_hits(hit_adder, hits, rescorer); + } else { + add_rescored_hits(hit_adder, hits, RerankRescorer(rescorer, reranked_hits)); + } +} + +template <typename Rescorer> +void +add_rescored_hits(ResultSet& rs, const std::vector<HitCollector::Hit>& hits, const std::vector<HitCollector::Hit>& reranked_hits, std::optional<double> second_phase_rank_drop_limit, std::vector<uint32_t>* dropped, Rescorer rescorer) +{ + if (second_phase_rank_drop_limit.has_value()) { + if (dropped != nullptr) { + add_rescored_hits(TrackingConditionalHitAdder(rs, second_phase_rank_drop_limit.value(), *dropped), hits, reranked_hits, rescorer); + } else { + add_rescored_hits(ConditionalHitAdder(rs, second_phase_rank_drop_limit.value()), hits, reranked_hits, rescorer); } - assert(rhCur != rhEnd); // the hits should be a subset of the hits in ranked hit array. - result[rhCur]._rankValue = hit.second; + } else { + add_rescored_hits(SimpleHitAdder(rs), hits, reranked_hits, rescorer); + } +} + +template <typename HitAdder, typename Rescorer> +void +mixin_rescored_hits(HitAdder hit_adder, const std::vector<HitCollector::Hit>& hits, const std::vector<uint32_t>& docids, double default_value, Rescorer rescorer) +{ + auto hits_cur = hits.begin(); + auto hits_end = hits.end(); + for (auto docid : docids) { + if (hits_cur != hits_end && docid == hits_cur->first) { + hit_adder.add(docid, rescorer.rescore(docid, hits_cur->second)); + ++hits_cur; + } else { + hit_adder.add(docid, default_value); + } + } +} + +template <typename HitAdder, typename Rescorer> +void +mixin_rescored_hits(HitAdder hit_adder, const std::vector<HitCollector::Hit>& hits, const std::vector<uint32_t>& docids, double default_value, const std::vector<HitCollector::Hit>& reranked_hits, Rescorer rescorer) +{ + if (reranked_hits.empty()) { + mixin_rescored_hits(hit_adder, hits, docids, default_value, rescorer); + } else { + mixin_rescored_hits(hit_adder, hits, docids, default_value, RerankRescorer(rescorer, reranked_hits)); + } +} + +template <typename Rescorer> +void +mixin_rescored_hits(ResultSet& rs, const std::vector<HitCollector::Hit>& hits, const std::vector<uint32_t>& docids, double default_value, const std::vector<HitCollector::Hit>& reranked_hits, std::optional<double> second_phase_rank_drop_limit, std::vector<uint32_t>* dropped, Rescorer rescorer) +{ + if (second_phase_rank_drop_limit.has_value()) { + if (dropped != nullptr) { + mixin_rescored_hits(TrackingConditionalHitAdder(rs, second_phase_rank_drop_limit.value(), *dropped), hits, docids, default_value, reranked_hits, rescorer); + } else { + mixin_rescored_hits(ConditionalHitAdder(rs, second_phase_rank_drop_limit.value()), hits, docids, default_value, reranked_hits, rescorer); + } + } else { + mixin_rescored_hits(SimpleHitAdder(rs), hits, docids, default_value, reranked_hits, rescorer); + } +} + +void +add_bitvector_to_dropped(std::vector<uint32_t>& dropped, vespalib::ConstArrayRef<RankedHit> hits, const BitVector& bv) +{ + auto hits_cur = hits.begin(); + auto hits_end = hits.end(); + auto docid = bv.getFirstTrueBit(); + auto docid_limit = bv.size(); + while (docid < docid_limit) { + if (hits_cur != hits_end && hits_cur->getDocId() == docid) { + ++hits_cur; + } else { + dropped.emplace_back(docid); + } + docid = bv.getNextTrueBit(docid + 1); } } } std::unique_ptr<ResultSet> -HitCollector::getResultSet(HitRank default_value) +HitCollector::get_result_set(std::optional<double> second_phase_rank_drop_limit, std::vector<uint32_t>* dropped) { - bool needReScore = false; - Scores &initHeapScores = _ranges.first; - Scores &finalHeapScores = _ranges.second; - if (initHeapScores.low > finalHeapScores.low) { - // scale and adjust the score according to the range - // of the initial and final heap score values to avoid that - // a score from the first phase is larger than finalHeapScores.low - feature_t initRange = initHeapScores.high - initHeapScores.low; - if (initRange < 1.0) initRange = 1.0f; - feature_t finalRange = finalHeapScores.high - finalHeapScores.low; - if (finalRange < 1.0) finalRange = 1.0f; - _scale = finalRange / initRange; - _adjust = initHeapScores.low * _scale - finalHeapScores.low; - needReScore = true; + /* + * Use default_rank_value (i.e. -HUGE_VAL) when hit collector saves + * rank scores, otherwise use zero_rank_value (i.e. 0.0). + */ + auto default_value = save_rank_scores() ? search::default_rank_value : search::zero_rank_value; + + bool needReScore = FirstPhaseRescorer::need_rescore(_ranges); + FirstPhaseRescorer rescorer(_ranges); + + if (dropped != nullptr) { + dropped->clear(); } // destroys the heap property or score sort order sortHitsByDocId(); auto rs = std::make_unique<ResultSet>(); - if ( ! _collector->isDocIdCollector() ) { - unsigned int iSize = _hits.size(); - rs->allocArray(iSize); + if ( ! _collector->isDocIdCollector() || + (second_phase_rank_drop_limit.has_value() && + (_bitVector || dropped == nullptr))) { + rs->allocArray(_hits.size()); + auto* dropped_or_null = dropped; + if (second_phase_rank_drop_limit.has_value() && _bitVector) { + dropped_or_null = nullptr; + } if (needReScore) { - for (uint32_t i = 0; i < iSize; ++i) { - rs->push_back(RankedHit(_hits[i].first, getReScore(_hits[i].second))); - } + add_rescored_hits(*rs, _hits, _reRankedHits, second_phase_rank_drop_limit, dropped_or_null, rescorer); } else { - for (uint32_t i = 0; i < iSize; ++i) { - rs->push_back(RankedHit(_hits[i].first, _hits[i].second)); - } + add_rescored_hits(*rs, _hits, _reRankedHits, second_phase_rank_drop_limit, dropped_or_null, NoRescorer()); } } else { if (_unordered) { std::sort(_docIdVector.begin(), _docIdVector.end()); } - unsigned int iSize = _hits.size(); - unsigned int jSize = _docIdVector.size(); - rs->allocArray(jSize); - uint32_t i = 0; + rs->allocArray(_docIdVector.size()); if (needReScore) { - for (uint32_t j = 0; j < jSize; ++j) { - uint32_t docId = _docIdVector[j]; - if (i < iSize && docId == _hits[i].first) { - rs->push_back(RankedHit(docId, getReScore(_hits[i].second))); - ++i; - } else { - rs->push_back(RankedHit(docId, default_value)); - } - } + mixin_rescored_hits(*rs, _hits, _docIdVector, default_value, _reRankedHits, second_phase_rank_drop_limit, dropped, rescorer); } else { - for (uint32_t j = 0; j < jSize; ++j) { - uint32_t docId = _docIdVector[j]; - if (i < iSize && docId == _hits[i].first) { - rs->push_back(RankedHit(docId, _hits[i].second)); - ++i; - } else { - rs->push_back(RankedHit(docId, default_value)); - } - } + mixin_rescored_hits(*rs, _hits, _docIdVector, default_value, _reRankedHits, second_phase_rank_drop_limit, dropped, NoRescorer()); } } - if (!_reRankedHits.empty()) { - mergeHitsIntoResultSet(_reRankedHits, *rs); + if (second_phase_rank_drop_limit.has_value() && _bitVector) { + if (dropped != nullptr) { + assert(dropped->empty()); + add_bitvector_to_dropped(*dropped, {rs->getArray(), rs->getArrayUsed()}, *_bitVector); + } + _bitVector.reset(); } if (_bitVector) { @@ -285,4 +421,10 @@ HitCollector::getResultSet(HitRank default_value) return rs; } +std::unique_ptr<ResultSet> +HitCollector::getResultSet() +{ + return get_result_set(std::nullopt, nullptr); +} + } diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index 94ffe619bab..c23fb0a6ef6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -8,6 +8,7 @@ #include <vespa/searchlib/common/resultset.h> #include <vespa/vespalib/util/sort.h> #include <algorithm> +#include <optional> #include <vector> namespace search::queryeval { @@ -35,8 +36,6 @@ private: std::vector<Hit> _reRankedHits; std::pair<Scores, Scores> _ranges; - feature_t _scale; - feature_t _adjust; struct ScoreComparator { bool operator() (const Hit & lhs, const Hit & rhs) const noexcept { @@ -120,12 +119,11 @@ private: void collect(uint32_t docId, feature_t score) override; }; - HitRank getReScore(feature_t score) const { - return ((score * _scale) - _adjust); - } VESPA_DLL_LOCAL void sortHitsByScore(size_t topn); VESPA_DLL_LOCAL void sortHitsByDocId(); + bool save_rank_scores() const noexcept { return _maxHitsSize != 0; } + public: HitCollector(const HitCollector &) = delete; HitCollector &operator=(const HitCollector &) = delete; @@ -169,15 +167,17 @@ public: const std::pair<Scores, Scores> &getRanges() const { return _ranges; } void setRanges(const std::pair<Scores, Scores> &ranges); + std::unique_ptr<ResultSet> + get_result_set(std::optional<double> second_phase_rank_drop_limit, std::vector<uint32_t>* dropped); + /** * Returns a result set based on the content of this collector. * Invoking this method will destroy the heap property of the * ranked hits and the match data heap. * - * @param auto pointer to the result set - * @param default_value rank value to be used for results without rank value + * @return unique pointer to the result set **/ - std::unique_ptr<ResultSet> getResultSet(HitRank default_value = default_rank_value); + std::unique_ptr<ResultSet> getResultSet(); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 5b8fa79b8af..72feeaebc13 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -26,7 +26,7 @@ size_t lookup_create_source(std::vector<std::unique_ptr<CombineType> > &sources, return i; } } - sources.push_back(std::unique_ptr<CombineType>(new CombineType())); + sources.push_back(std::make_unique<CombineType>()); sources.back()->setSourceId(child_source); sources.back()->setDocIdLimit(docid_limit); return (sources.size() - 1); @@ -419,6 +419,14 @@ WeakAndBlueprint::my_flow(InFlow in_flow) const return AnyFlow::create<OrFlow>(in_flow); } +WeakAndBlueprint::WeakAndBlueprint(uint32_t n, float idf_range, bool thread_safe) + : _scores(WeakAndPriorityQueue::createHeap(n, thread_safe)), + _n(n), + _idf_range(idf_range), + _weights(), + _matching_phase(MatchingPhase::FIRST_PHASE) +{} + WeakAndBlueprint::~WeakAndBlueprint() = default; FlowStats @@ -478,13 +486,15 @@ WeakAndBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, assert(_weights.size() == childCnt()); for (size_t i = 0; i < sub_searches.size(); ++i) { // TODO: pass ownership with unique_ptr - terms.emplace_back(sub_searches[i].release(), - _weights[i], + terms.emplace_back(sub_searches[i].release(), _weights[i], getChild(i).getState().estimate().estHits); } + bool readonly_scores_heap = (_matching_phase != MatchingPhase::FIRST_PHASE); return (_idf_range == 0.0) - ? WeakAndSearch::create(terms, wand::TermFrequencyScorer(), _n, strict()) - : WeakAndSearch::create(terms, wand::Bm25TermFrequencyScorer(get_docid_limit(), _idf_range), _n, strict()); + ? WeakAndSearch::create(terms, wand::MatchParams(*_scores), wand::TermFrequencyScorer(), _n, strict(), + readonly_scores_heap) + : WeakAndSearch::create(terms, wand::MatchParams(*_scores), wand::Bm25TermFrequencyScorer(get_docid_limit(), _idf_range), _n, strict(), + readonly_scores_heap); } SearchIterator::UP @@ -493,6 +503,28 @@ WeakAndBlueprint::createFilterSearch(FilterConstraint constraint) const return create_atmost_or_filter(get_children(), strict(), constraint); } +void +WeakAndBlueprint::set_matching_phase(MatchingPhase matching_phase) noexcept +{ + _matching_phase = matching_phase; + if (matching_phase != MatchingPhase::FIRST_PHASE) { + /* + * During first phase matching, the scores heap is adjusted by + * the iterators. The minimum score is increased when the + * scores heap is full while handling a matching document with + * a higher score than the worst existing one. + * + * During later matching phases, only the original minimum + * score is used, and the heap is not updated by the + * iterators. This ensures that all documents considered a hit + * by the first phase matching will also be considered as hits + * by the later matching phases. + */ + _scores->set_min_score(1); + } +} + + //----------------------------------------------------------------------------- AnyFlow @@ -503,7 +535,7 @@ NearBlueprint::my_flow(InFlow in_flow) const FlowStats NearBlueprint::calculate_flow_stats(uint32_t) const { - double est = AndFlow::estimate_of(get_children()); + double est = AndFlow::estimate_of(get_children()); return {est, AndFlow::cost_of(get_children(), false) + childCnt() * est, AndFlow::cost_of(get_children(), true) + childCnt() * est}; @@ -549,7 +581,7 @@ NearBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, tfmda.add(cs.field(j).resolve(md)); } } - return SearchIterator::UP(new NearSearch(std::move(sub_searches), tfmda, _window, strict())); + return std::make_unique<NearSearch>(std::move(sub_searches), tfmda, _window, strict()); } SearchIterator::UP @@ -612,7 +644,7 @@ ONearBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, } // could sort sub_searches here // but then strictness inheritance would also need to be fixed - return SearchIterator::UP(new ONearSearch(std::move(sub_searches), tfmda, _window, strict())); + return std::make_unique<ONearSearch>(std::move(sub_searches), tfmda, _window, strict()); } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index 913370caae1..016ee67cac8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -4,6 +4,7 @@ #include "blueprint.h" #include "multisearch.h" +#include <vespa/searchlib/queryeval/wand/weak_and_heap.h> namespace search::queryeval { @@ -88,9 +89,11 @@ private: class WeakAndBlueprint : public IntermediateBlueprint { private: + std::unique_ptr<WeakAndPriorityQueue> _scores; uint32_t _n; float _idf_range; std::vector<uint32_t> _weights; + MatchingPhase _matching_phase; AnyFlow my_flow(InFlow in_flow) const override; public: @@ -106,15 +109,16 @@ public: fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(FilterConstraint constraint) const override; - explicit WeakAndBlueprint(uint32_t n) noexcept : WeakAndBlueprint(n, 0.0) {} - WeakAndBlueprint(uint32_t n, float idf_range) noexcept : _n(n), _idf_range(idf_range), _weights() {} + explicit WeakAndBlueprint(uint32_t n) : WeakAndBlueprint(n, 0.0, true) {} + WeakAndBlueprint(uint32_t n, float idf_range, bool thread_safe); ~WeakAndBlueprint() override; void addTerm(Blueprint::UP bp, uint32_t weight) { addChild(std::move(bp)); _weights.push_back(weight); } - uint32_t getN() const { return _n; } - const std::vector<uint32_t> &getWeights() const { return _weights; } + uint32_t getN() const noexcept { return _n; } + const std::vector<uint32_t> &getWeights() const noexcept { return _weights; } + void set_matching_phase(MatchingPhase matching_phase) noexcept override; }; //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h index f05bf8e1adc..7c4e874601e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h +++ b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h @@ -61,6 +61,11 @@ public: } std::unique_ptr<BitVector> get_hits(uint32_t begin_id, uint32_t end_id) const; void or_hits_into(BitVector &result, uint32_t begin_id) const; + void transform_children(auto f) { + for (size_t i = 0; i < _children.size(); ++i) { + _children[i] = f(std::move(_children[i]), i); + } + } }; using SearchIteratorPack = SearchIteratorPackT<uint16_t>; diff --git a/searchlib/src/vespa/searchlib/queryeval/matching_phase.h b/searchlib/src/vespa/searchlib/queryeval/matching_phase.h new file mode 100644 index 00000000000..cf475099c58 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/matching_phase.h @@ -0,0 +1,18 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::queryeval { + +/* + * The different matching phases when evaluating a query. + */ +enum class MatchingPhase { + FIRST_PHASE, + SECOND_PHASE, + MATCH_FEATURES, + SUMMARY_FEATURES, + DUMP_FEATURES +}; + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp index e90156868fb..cbb5a43fa47 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp @@ -4,19 +4,19 @@ #include "andsearch.h" #include "andnotsearch.h" #include "sourceblendersearch.h" -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> namespace search::queryeval { using vespalib::Trinary; -using vespalib::hwaccelrated::IAccelrated; +using vespalib::hwaccelerated::IAccelerated; using Meta = MultiBitVectorBase::Meta; namespace { struct And { using Word = BitWord::Word; - void operator () (const IAccelrated & accel, size_t offset, const std::vector<Meta> & src, void *dest) noexcept { + void operator () (const IAccelerated & accel, size_t offset, const std::vector<Meta> & src, void *dest) noexcept { accel.and128(offset, src, dest); } static constexpr bool isAnd() noexcept { return true; } @@ -24,7 +24,7 @@ struct And { struct Or { using Word = BitWord::Word; - void operator () (const IAccelrated & accel, size_t offset, const std::vector<Meta> & src, void *dest) noexcept { + void operator () (const IAccelerated & accel, size_t offset, const std::vector<Meta> & src, void *dest) noexcept { accel.or128(offset, src, dest); } static constexpr bool isAnd() noexcept { return false; } @@ -52,7 +52,7 @@ template <typename Update> MultiBitVector<Update>::MultiBitVector(size_t reserved) : MultiBitVectorBase(reserved), _update(), - _accel(IAccelrated::getAccelerator()), + _accel(IAccelerated::getAccelerator()), _lastWords() { static_assert(sizeof(_lastWords) == 128, "Lastwords should have 128 byte size"); @@ -234,10 +234,9 @@ SearchIterator::UP MultiBitVectorIteratorBase::optimize(SearchIterator::UP parentIt) { if (parentIt->isSourceBlender()) { - auto & parent(static_cast<SourceBlenderSearch &>(*parentIt)); - for (size_t i(0); i < parent.getNumChildren(); i++) { - parent.setChild(i, optimize(parent.steal(i))); - } + parentIt->transform_children([](auto child, size_t){ + return optimize(std::move(child)); + }); } else if (parentIt->isMultiSearch()) { parentIt = optimizeMultiSearch(std::move(parentIt)); } diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h index 0ecf9d85b92..67a4e116398 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h @@ -6,7 +6,7 @@ #include "unpackinfo.h" #include <vespa/searchlib/common/bitword.h> -namespace vespalib::hwaccelrated { class IAccelrated; } +namespace vespalib::hwaccelerated { class IAccelerated; } namespace search::queryeval { @@ -46,10 +46,10 @@ private: VESPA_DLL_LOCAL bool updateLastValueCold(uint32_t docId) noexcept __attribute__((noinline)); VESPA_DLL_LOCAL void fetchChunk(uint32_t docId) noexcept __attribute__((noinline)); - using IAccelrated = vespalib::hwaccelrated::IAccelrated; + using IAccelerated = vespalib::hwaccelerated::IAccelerated; Update _update; - const IAccelrated & _accel; + const IAccelerated & _accel; alignas(64) Word _lastWords[16]; static constexpr size_t NumWordsInBatch = sizeof(_lastWords) / sizeof(Word); }; diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp index 80ceb107b7c..18c1ee7deb5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp @@ -55,10 +55,10 @@ MultiSearch::initRange(uint32_t beginid, uint32_t endid) } void -MultiSearch::disclose_children(std::vector<UP*> &dst) +MultiSearch::transform_children(std::function<SearchIterator::UP(SearchIterator::UP, size_t)> f) { - for (auto &child: _children) { - dst.push_back(&child); + for (size_t i = 0; i < _children.size(); ++i) { + _children[i] = f(std::move(_children[i]), i); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.h b/searchlib/src/vespa/searchlib/queryeval/multisearch.h index 88ed3aaf310..8f1b4070a03 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.h @@ -41,7 +41,7 @@ public: void insert(size_t index, SearchIterator::UP search); virtual bool needUnpack(size_t index) const { (void) index; return true; } void initRange(uint32_t beginId, uint32_t endId) override; - void disclose_children(std::vector<UP*> &dst) override; + void transform_children(std::function<SearchIterator::UP(SearchIterator::UP, size_t)> f) override; protected: MultiSearch(); void doUnpack(uint32_t docid) override; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index da1e29875be..951855b43bb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -2,7 +2,7 @@ #include "nearest_neighbor_blueprint.h" #include "emptysearch.h" -#include "nearest_neighbor_iterator.h" +#include "exact_nearest_neighbor_iterator.h" #include "nns_index_iterator.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> @@ -64,7 +64,8 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _global_filter_set(false), _global_filter_hits(), _global_filter_hit_ratio(), - _doom(doom) + _doom(doom), + _matching_phase(MatchingPhase::FIRST_PHASE) { if (distance_threshold < std::numeric_limits<double>::max()) { _distance_threshold = _distance_calc->function().convert_threshold(distance_threshold); @@ -143,9 +144,10 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData default: ; } - return NearestNeighborIterator::create(strict(), tfmd, - std::make_unique<search::tensor::DistanceCalculator>(_attr_tensor, _query_tensor), - _distance_heap, *_global_filter); + return ExactNearestNeighborIterator::create(strict(), tfmd, + std::make_unique<search::tensor::DistanceCalculator>(_attr_tensor, _query_tensor), + _distance_heap, *_global_filter, + _matching_phase != MatchingPhase::FIRST_PHASE); } void @@ -183,6 +185,27 @@ NearestNeighborBlueprint::always_needs_unpack() const return true; } +void +NearestNeighborBlueprint::set_matching_phase(MatchingPhase matching_phase) noexcept +{ + _matching_phase = matching_phase; + if (matching_phase != MatchingPhase::FIRST_PHASE) { + /* + * During first phase matching, the distance heap is adjusted + * by the iterators. The distance threshold is lowered when + * the distance heap is full while handling a matching + * document with a lower distance than the worst existing one. + * + * During later matching phases, only the original distance + * threshold is used, and the heap is not updated by the + * iterators. This ensures that all documents considered a hit + * by the first phase matching will also be considered as hits + * by the later matching phases. + */ + _distance_heap.set_distance_threshold(_distance_threshold); + } +} + std::ostream& operator<<(std::ostream& out, NearestNeighborBlueprint::Algorithm algorithm) { diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index 95a8a6a6afe..bd40b9e030c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -47,6 +47,7 @@ private: std::optional<uint32_t> _global_filter_hits; std::optional<double> _global_filter_hit_ratio; const vespalib::Doom& _doom; + MatchingPhase _matching_phase; void perform_top_k(const search::tensor::NearestNeighborIndex* nns_index); public: @@ -80,6 +81,7 @@ public: } void visitMembers(vespalib::ObjectVisitor& visitor) const override; bool always_needs_unpack() const override; + void set_matching_phase(MatchingPhase matching_phase) noexcept override; }; std::ostream& diff --git a/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.cpp index ae0cc09b3ab..a8f1c892262 100644 --- a/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/objects/visit.hpp> #include <vespa/vespalib/util/classname.h> #include <vespa/vespalib/util/stringfmt.h> +#include "wand/weak_and_search.h" #include <typeindex> @@ -46,17 +47,6 @@ std::unique_ptr<SearchIterator> create(Profiler &profiler, ctor_token); } -void handle_leaf_node(Profiler &profiler, SearchIterator &leaf, const vespalib::string &path) { - if (leaf.isSourceBlender()) { - auto &source_blender = static_cast<SourceBlenderSearch&>(leaf); - for (size_t i = 0; i < source_blender.getNumChildren(); ++i) { - auto child = source_blender.steal(i); - child = ProfiledIterator::profile(profiler, std::move(child), fmt("%s%zu/", path.c_str(), i)); - source_blender.setChild(i, std::move(child)); - } - } -} - } void @@ -111,26 +101,12 @@ ProfiledIterator::visitMembers(vespalib::ObjectVisitor &visitor) const } std::unique_ptr<SearchIterator> -ProfiledIterator::profile(Profiler &profiler, std::unique_ptr<SearchIterator> root, const vespalib::string &root_path) +ProfiledIterator::profile(Profiler &profiler, std::unique_ptr<SearchIterator> node, const vespalib::string &path) { - std::vector<UP*> links({&root}); - std::vector<vespalib::string> paths({root_path}); - for (size_t offset = 0; offset < links.size(); ++offset) { - UP &link = *(links[offset]); - vespalib::string path = paths[offset]; - size_t first_child = links.size(); - link->disclose_children(links); - size_t num_children = links.size() - first_child; - if (num_children == 0) { - handle_leaf_node(profiler, *link, path); - } else { - for (size_t i = 0; i < num_children; ++i) { - paths.push_back(fmt("%s%zu/", path.c_str(), i)); - } - } - link = create(profiler, path, std::move(link), ctor_tag{}); - } - return root; + node->transform_children([&](auto child, size_t i){ + return profile(profiler, std::move(child), fmt("%s%zu/", path.c_str(), i)); + }); + return create(profiler, path, std::move(node), ctor_tag{}); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.h b/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.h index 4c9853d89c4..dc276d266cf 100644 --- a/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/profiled_iterator.h @@ -53,8 +53,8 @@ public: Trinary matches_any() const override { return _search->matches_any(); } const PostingInfo *getPostingInfo() const override { return _search->getPostingInfo(); } static std::unique_ptr<SearchIterator> profile(Profiler &profiler, - std::unique_ptr<SearchIterator> root, - const vespalib::string &root_path = "/"); + std::unique_ptr<SearchIterator> node, + const vespalib::string &path = "/"); }; } // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/scores.h b/searchlib/src/vespa/searchlib/queryeval/scores.h index 19976a2831b..83b1ba9d406 100644 --- a/searchlib/src/vespa/searchlib/queryeval/scores.h +++ b/searchlib/src/vespa/searchlib/queryeval/scores.h @@ -24,6 +24,9 @@ struct Scores { high = score; } } + bool operator==(const Scores& rhs) const { + return low == rhs.low && high == rhs.high; + } }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp index e979a810799..ab6238c5ea4 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp @@ -118,7 +118,7 @@ SearchIterator::visitMembers(vespalib::ObjectVisitor &visitor) const } void -SearchIterator::disclose_children(std::vector<UP*> &) +SearchIterator::transform_children(std::function<SearchIterator::UP(SearchIterator::UP, size_t)>) { } diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h index f8124a161e2..455b1b67f4b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/trinary.h> #include <memory> #include <vector> +#include <functional> namespace vespalib { class ObjectVisitor; } namespace vespalib::slime { @@ -20,6 +21,8 @@ namespace search::attribute { class ISearchContext; } namespace search::queryeval { +struct WeakAndSearch; + /** * This is the abstract superclass of all search objects. Each search * object act as an iterator over documents that are results for the @@ -357,6 +360,8 @@ public: */ virtual bool isMultiSearch() const { return false; } + virtual WeakAndSearch *as_weak_and() noexcept { return nullptr; } + /** * This is used for adding an extra filter. If it is accepted it will return an empty UP. * If not you will get in in return. Currently it will only be accepted by a @@ -378,11 +383,11 @@ public: // number of matches: (False <= Undefined <= True) virtual Trinary matches_any() const { return Trinary::Undefined; } - // Disclose children by giving out references to owning - // pointers. This allows re-wiring from the outside, which is - // needed for deep decoration used by match profiling. Only - // disclose children that are treated as generic SearchIterators. - virtual void disclose_children(std::vector<UP*> &dst); + // Transform all children using the given function. The number + // passed with the child to be transformed should match the index + // of the child in the originating blueprint. This is used for + // deep decoration when doing match profiling. + virtual void transform_children(std::function<SearchIterator::UP(SearchIterator::UP, size_t)> f); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp index d2461e25fa9..f35a6fb32ee 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp @@ -161,9 +161,14 @@ SourceBlenderSearch::~SourceBlenderSearch() } void -SourceBlenderSearch::setChild(size_t index, SearchIterator::UP child) { - assert(_sources[_children[index]] == nullptr); - _sources[_children[index]] = child.release(); +SourceBlenderSearch::transform_children(std::function<SearchIterator::UP(SearchIterator::UP, size_t)> f) +{ + for (size_t i = 0; i < _children.size(); ++i) { + SearchIterator::UP ptr(_sources[_children[i]]); + _sources[_children[i]] = nullptr; + ptr = f(std::move(ptr), i); + _sources[_children[i]] = ptr.release(); + } } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h index cbe4666e955..679e3d35186 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h @@ -72,13 +72,7 @@ public: const Children &children, bool strict); ~SourceBlenderSearch() override; - size_t getNumChildren() const { return _children.size(); } - SearchIterator::UP steal(size_t index) { - SearchIterator::UP retval(_sources[_children[index]]); - _sources[_children[index]] = nullptr; - return retval; - } - void setChild(size_t index, SearchIterator::UP child); + void transform_children(std::function<SearchIterator::UP(SearchIterator::UP, size_t)> f) override; void initRange(uint32_t beginId, uint32_t endId) override; }; diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index 4c55496822b..75574080164 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -1,45 +1,27 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "parallel_weak_and_blueprint.h" -#include "wand_parts.h" #include "parallel_weak_and_search.h" #include <vespa/searchlib/queryeval/field_spec.hpp> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/queryeval/flow_tuning.h> -#include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.hpp> #include <algorithm> namespace search::queryeval { -ParallelWeakAndBlueprint::ParallelWeakAndBlueprint(FieldSpecBase field, - uint32_t scoresToTrack, - score_t scoreThreshold, - double thresholdBoostFactor) +ParallelWeakAndBlueprint::ParallelWeakAndBlueprint(FieldSpecBase field, uint32_t scoresToTrack, + score_t scoreThreshold, double thresholdBoostFactor, + bool thread_safe) : ComplexLeafBlueprint(field), - _scores(scoresToTrack), + _scores(WeakAndPriorityQueue::createHeap(scoresToTrack, thread_safe)), _scoreThreshold(scoreThreshold), _thresholdBoostFactor(thresholdBoostFactor), - _scoresAdjustFrequency(DEFAULT_PARALLEL_WAND_SCORES_ADJUST_FREQUENCY), + _scoresAdjustFrequency(wand::DEFAULT_PARALLEL_WAND_SCORES_ADJUST_FREQUENCY), _layout(), _weights(), - _terms() -{ -} - -ParallelWeakAndBlueprint::ParallelWeakAndBlueprint(FieldSpecBase field, - uint32_t scoresToTrack, - score_t scoreThreshold, - double thresholdBoostFactor, - uint32_t scoresAdjustFrequency) - : ComplexLeafBlueprint(field), - _scores(scoresToTrack), - _scoreThreshold(scoreThreshold), - _thresholdBoostFactor(thresholdBoostFactor), - _scoresAdjustFrequency(scoresAdjustFrequency), - _layout(), - _weights(), - _terms() + _terms(), + _matching_phase(MatchingPhase::FIRST_PHASE) { } @@ -84,7 +66,7 @@ ParallelWeakAndBlueprint::calculate_flow_stats(uint32_t docid_limit) const term->update_flow_stats(docid_limit); } double child_est = OrFlow::estimate_of(_terms); - double my_est = abs_to_rel_est(_scores.getScoresToTrack(), docid_limit); + double my_est = abs_to_rel_est(_scores->getScoresToTrack(), docid_limit); double est = (child_est + my_est) / 2.0; return {est, OrFlow::cost_of(_terms, false), OrFlow::cost_of(_terms, true) + flow::heap_cost(est, _terms.size())}; @@ -106,14 +88,12 @@ ParallelWeakAndBlueprint::createLeafSearch(const search::fef::TermFieldMatchData childState.estimate().estHits, childState.field(0).resolve(*childrenMatchData)); } - return SearchIterator::UP - (ParallelWeakAndSearch::create(terms, - ParallelWeakAndSearch::MatchParams(_scores, - _scoreThreshold, - _thresholdBoostFactor, - _scoresAdjustFrequency).setDocIdLimit(get_docid_limit()), - ParallelWeakAndSearch::RankParams(*tfmda[0], - std::move(childrenMatchData)), strict())); + bool readonly_scores_heap = (_matching_phase != MatchingPhase::FIRST_PHASE); + return ParallelWeakAndSearch::create(terms, + ParallelWeakAndSearch::MatchParams(*_scores, _scoreThreshold, _thresholdBoostFactor, + _scoresAdjustFrequency, get_docid_limit()), + ParallelWeakAndSearch::RankParams(*tfmda[0],std::move(childrenMatchData)), + strict(), readonly_scores_heap); } std::unique_ptr<SearchIterator> @@ -137,6 +117,27 @@ ParallelWeakAndBlueprint::always_needs_unpack() const } void +ParallelWeakAndBlueprint::set_matching_phase(MatchingPhase matching_phase) noexcept +{ + _matching_phase = matching_phase; + if (matching_phase != MatchingPhase::FIRST_PHASE) { + /* + * During first phase matching, the scores heap is adjusted by + * the iterators. The minimum score is increased when the + * scores heap is full while handling a matching document with + * a higher score than the worst existing one. + * + * During later matching phases, only the original minimum + * score is used, and the heap is not updated by the + * iterators. This ensures that all documents considered a hit + * by the first phase matching will also be considered as hits + * by the later matching phases. + */ + _scores->set_min_score(_scoreThreshold); + } +} + +void ParallelWeakAndBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const { LeafBlueprint::visitMembers(visitor); diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h index 4a55bf14095..bd47d127d97 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h @@ -11,8 +11,6 @@ namespace search::queryeval { -const uint32_t DEFAULT_PARALLEL_WAND_SCORES_ADJUST_FREQUENCY = 4; - /** * Blueprint for the parallel weak and search operator. */ @@ -21,32 +19,25 @@ class ParallelWeakAndBlueprint : public ComplexLeafBlueprint private: using score_t = wand::score_t; - mutable SharedWeakAndPriorityQueue _scores; - const wand::score_t _scoreThreshold; - double _thresholdBoostFactor; - const uint32_t _scoresAdjustFrequency; - fef::MatchDataLayout _layout; - std::vector<int32_t> _weights; - std::vector<Blueprint::UP> _terms; + std::unique_ptr<WeakAndPriorityQueue> _scores; + const wand::score_t _scoreThreshold; + double _thresholdBoostFactor; + const uint32_t _scoresAdjustFrequency; + fef::MatchDataLayout _layout; + std::vector<int32_t> _weights; + std::vector<Blueprint::UP> _terms; + MatchingPhase _matching_phase; public: ParallelWeakAndBlueprint(const ParallelWeakAndBlueprint &) = delete; ParallelWeakAndBlueprint &operator=(const ParallelWeakAndBlueprint &) = delete; - ParallelWeakAndBlueprint(FieldSpecBase field, - uint32_t scoresToTrack, - score_t scoreThreshold, - double thresholdBoostFactor); - ParallelWeakAndBlueprint(FieldSpecBase field, - uint32_t scoresToTrack, - score_t scoreThreshold, - double thresholdBoostFactor, - uint32_t scoresAdjustFrequency); + ParallelWeakAndBlueprint(FieldSpecBase field, uint32_t scoresToTrack, + score_t scoreThreshold, double thresholdBoostFactor, + bool thread_safe); ~ParallelWeakAndBlueprint() override; - const WeakAndHeap &getScores() const { return _scores; } - + const WeakAndHeap &getScores() const { return *_scores; } score_t getScoreThreshold() const { return _scoreThreshold; } - double getThresholdBoostFactor() const { return _thresholdBoostFactor; } // Used by create visitor @@ -70,6 +61,7 @@ public: void visitMembers(vespalib::ObjectVisitor &visitor) const override; void fetchPostings(const ExecuteInfo &execInfo) override; bool always_needs_unpack() const override; + void set_matching_phase(MatchingPhase matching_phase) noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp index 9e887b9d0f7..d5093408622 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp @@ -32,6 +32,7 @@ private: score_t _boostedThreshold; const MatchParams _matchParams; std::vector<score_t> _localScores; + const bool _readonly_scores_heap; void updateThreshold(score_t newThreshold) { if (newThreshold > _threshold) { @@ -67,7 +68,8 @@ private: public: ParallelWeakAndSearchImpl(fef::TermFieldMatchData &tfmd, VectorizedTerms &&terms, - const MatchParams &matchParams) + const MatchParams &matchParams, + bool readonly_scores_heap) : _tfmd(tfmd), _terms(std::move(terms)), _heaps(DocIdOrder(_terms.docId()), _terms.size()), @@ -75,8 +77,10 @@ public: _threshold(matchParams.scoreThreshold), _boostedThreshold(_threshold * matchParams.thresholdBoostFactor), _matchParams(matchParams), - _localScores() + _localScores(), + _readonly_scores_heap(readonly_scores_heap) { + _localScores.reserve(_matchParams.scoresAdjustFrequency); } size_t get_num_terms() const override { return _terms.size(); } int32_t get_term_weight(size_t idx) const override { return _terms.weight(idx); } @@ -93,10 +97,12 @@ public: } void doUnpack(uint32_t docid) override { score_t score = _algo.get_full_score(_terms, _heaps, DotProductScorer()); - _localScores.push_back(score); - if (_localScores.size() == _matchParams.scoresAdjustFrequency) { - _matchParams.scores.adjust(&_localScores[0], &_localScores[0] + _localScores.size()); - _localScores.clear(); + if (!_readonly_scores_heap) { + _localScores.push_back(score); + if (_localScores.size() == _matchParams.scoresAdjustFrequency) { + _matchParams.scores.adjust(&_localScores[0], &_localScores[0] + _localScores.size()); + _localScores.clear(); + } } _tfmd.setRawScore(docid, score); } @@ -129,7 +135,8 @@ template <typename FutureHeap, typename PastHeap, bool IS_STRICT> SearchIterator::UP createWand(const wand::Terms &terms, const ParallelWeakAndSearch::MatchParams &matchParams, - ParallelWeakAndSearch::RankParams &&rankParams) + ParallelWeakAndSearch::RankParams &&rankParams, + bool readonly_scores_heap) { using WandType = ParallelWeakAndSearchImpl<VectorizedIteratorTerms, FutureHeap, PastHeap, IS_STRICT>; if (should_monitor_wand()) { @@ -143,7 +150,7 @@ createWand(const wand::Terms &terms, DotProductScorer(), matchParams.docIdLimit, std::move(rankParams.childrenMatchData)), - matchParams), + matchParams, readonly_scores_heap), false); return std::make_unique<MonitoringDumpIterator>(std::move(monitoringIterator)); } @@ -152,7 +159,7 @@ createWand(const wand::Terms &terms, DotProductScorer(), matchParams.docIdLimit, std::move(rankParams.childrenMatchData)), - matchParams); + matchParams, readonly_scores_heap); } } // namespace search::queryeval::wand::<unnamed> @@ -163,33 +170,36 @@ SearchIterator::UP ParallelWeakAndSearch::createArrayWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, - bool strict) + bool strict, + bool readonly_scores_heap) { return strict - ? wand::createWand<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>(terms, matchParams, std::move(rankParams)) - : wand::createWand<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>(terms, matchParams, std::move(rankParams)); + ? wand::createWand<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>(terms, matchParams, std::move(rankParams), readonly_scores_heap) + : wand::createWand<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>(terms, matchParams, std::move(rankParams), readonly_scores_heap); } SearchIterator::UP ParallelWeakAndSearch::createHeapWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, - bool strict) + bool strict, + bool readonly_scores_heap) { return strict - ? wand::createWand<vespalib::LeftHeap, vespalib::RightHeap, true>(terms, matchParams, std::move(rankParams)) - : wand::createWand<vespalib::LeftHeap, vespalib::RightHeap, false>(terms, matchParams, std::move(rankParams)); + ? wand::createWand<vespalib::LeftHeap, vespalib::RightHeap, true>(terms, matchParams, std::move(rankParams), readonly_scores_heap) + : wand::createWand<vespalib::LeftHeap, vespalib::RightHeap, false>(terms, matchParams, std::move(rankParams), readonly_scores_heap); } SearchIterator::UP ParallelWeakAndSearch::create(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, - bool strict) + bool strict, + bool readonly_scores_heap) { return (terms.size() < 128) - ? createArrayWand(terms, matchParams, std::move(rankParams), strict) - : createHeapWand(terms, matchParams, std::move(rankParams), strict); + ? createArrayWand(terms, matchParams, std::move(rankParams), strict, readonly_scores_heap) + : createHeapWand(terms, matchParams, std::move(rankParams), strict, readonly_scores_heap); } //----------------------------------------------------------------------------- @@ -197,19 +207,19 @@ ParallelWeakAndSearch::create(const Terms &terms, namespace { template <typename VectorizedTerms, typename FutureHeap, typename PastHeap> -SearchIterator::UP create_helper(search::fef::TermFieldMatchData &tfmd, VectorizedTerms &&terms, const MatchParams ¶ms, bool strict) { +SearchIterator::UP create_helper(search::fef::TermFieldMatchData &tfmd, VectorizedTerms &&terms, const MatchParams ¶ms, bool strict, bool readonly_scores_heap) { if (strict) { - return std::make_unique<wand::ParallelWeakAndSearchImpl<VectorizedTerms, FutureHeap, PastHeap, true>>(tfmd, std::move(terms), params); + return std::make_unique<wand::ParallelWeakAndSearchImpl<VectorizedTerms, FutureHeap, PastHeap, true>>(tfmd, std::forward<VectorizedTerms>(terms), params, readonly_scores_heap); } else { - return std::make_unique<wand::ParallelWeakAndSearchImpl<VectorizedTerms, FutureHeap, PastHeap, false>>(tfmd, std::move(terms), params); + return std::make_unique<wand::ParallelWeakAndSearchImpl<VectorizedTerms, FutureHeap, PastHeap, false>>(tfmd, std::forward<VectorizedTerms>(terms), params, readonly_scores_heap); } } template <typename VectorizedTerms> -SearchIterator::UP create_helper(search::fef::TermFieldMatchData &tfmd, VectorizedTerms &&terms, const MatchParams ¶ms, bool strict, bool use_array) { +SearchIterator::UP create_helper(search::fef::TermFieldMatchData &tfmd, VectorizedTerms &&terms, const MatchParams ¶ms, bool strict, bool readonly_scores_heap, bool use_array) { return (use_array) - ? create_helper<VectorizedTerms, vespalib::LeftArrayHeap, vespalib::RightArrayHeap>(tfmd, std::move(terms), params, strict) - : create_helper<VectorizedTerms, vespalib::LeftHeap, vespalib::RightHeap>(tfmd, std::move(terms), params, strict); + ? create_helper<VectorizedTerms, vespalib::LeftArrayHeap, vespalib::RightArrayHeap>(tfmd, std::forward<VectorizedTerms>(terms), params, strict, readonly_scores_heap) + : create_helper<VectorizedTerms, vespalib::LeftHeap, vespalib::RightHeap>(tfmd, std::forward<VectorizedTerms>(terms), params, strict, readonly_scores_heap); } } // namespace search::queryeval::<unnamed> @@ -220,12 +230,13 @@ ParallelWeakAndSearch::create(search::fef::TermFieldMatchData &tfmd, const std::vector<int32_t> &weights, const std::vector<IDirectPostingStore::LookupResult> &dict_entries, const IDocidWithWeightPostingStore &attr, - bool strict) + bool strict, + bool readonly_scores_heap) { assert(weights.size() == dict_entries.size()); if (!wand::should_monitor_wand()) { wand::VectorizedAttributeTerms terms(weights, dict_entries, attr, wand::DotProductScorer(), matchParams.docIdLimit); - return create_helper(tfmd, std::move(terms), matchParams, strict, (weights.size() < 128)); + return create_helper(tfmd, std::move(terms), matchParams, strict, readonly_scores_heap, (weights.size() < 128)); } else { // reverse-wrap direct iterators into old API to be compatible with monitoring fef::MatchDataLayout layout; @@ -244,7 +255,7 @@ ParallelWeakAndSearch::create(search::fef::TermFieldMatchData &tfmd, dict_entries[i].posting_size, childrenMatchData->resolveTermField(handles[i])); } - return create(terms, matchParams, RankParams(tfmd, std::move(childrenMatchData)), strict); + return create(terms, matchParams, RankParams(tfmd, std::move(childrenMatchData)), strict, readonly_scores_heap); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.h b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.h index bd173ab41eb..130bd46f470 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.h @@ -20,27 +20,25 @@ struct ParallelWeakAndSearch : public SearchIterator /** * Params used to tweak the behavior of the WAND algorithm. */ - struct MatchParams + struct MatchParams : wand::MatchParams { - WeakAndHeap &scores; - score_t scoreThreshold; - double thresholdBoostFactor; - uint32_t scoresAdjustFrequency; - docid_t docIdLimit; - MatchParams(WeakAndHeap &scores_, - score_t scoreThreshold_, - double thresholdBoostFactor_, - uint32_t scoresAdjustFrequency_) - : scores(scores_), - scoreThreshold(scoreThreshold_), - thresholdBoostFactor(thresholdBoostFactor_), - scoresAdjustFrequency(scoresAdjustFrequency_), - docIdLimit(0) + const double thresholdBoostFactor; + const docid_t docIdLimit; + MatchParams(WeakAndHeap &scores_in, + score_t scoreThreshold_in, + double thresholdBoostFactor_in, + uint32_t scoresAdjustFrequency_in, + uint32_t docIdLimit_in) noexcept + : wand::MatchParams(scores_in, scoreThreshold_in, scoresAdjustFrequency_in), + thresholdBoostFactor(thresholdBoostFactor_in), + docIdLimit(docIdLimit_in) + {} + MatchParams(WeakAndHeap &scores_in, + score_t scoreThreshold_in, + double thresholdBoostFactor_in, + uint32_t scoresAdjustFrequency_in) noexcept + : MatchParams(scores_in, scoreThreshold_in, thresholdBoostFactor_in, scoresAdjustFrequency_in, 0) {} - MatchParams &setDocIdLimit(docid_t value) { - docIdLimit = value; - return *this; - } }; /** @@ -51,7 +49,7 @@ struct ParallelWeakAndSearch : public SearchIterator fef::TermFieldMatchData &rootMatchData; fef::MatchData::UP childrenMatchData; RankParams(fef::TermFieldMatchData &rootMatchData_, - fef::MatchData::UP &&childrenMatchData_) + fef::MatchData::UP &&childrenMatchData_) noexcept : rootMatchData(rootMatchData_), childrenMatchData(std::move(childrenMatchData_)) {} @@ -64,16 +62,15 @@ struct ParallelWeakAndSearch : public SearchIterator virtual score_t get_max_score(size_t idx) const = 0; virtual const MatchParams &getMatchParams() const = 0; - static SearchIterator::UP createArrayWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); - static SearchIterator::UP createHeapWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); - static SearchIterator::UP create(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); + static SearchIterator::UP createArrayWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict, bool readonly_scores_heap); + static SearchIterator::UP createHeapWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict, bool readonly_scores_heap); + static SearchIterator::UP create(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict, bool readonly_scores_heap); - static SearchIterator::UP create(fef::TermFieldMatchData &tmd, - const MatchParams &matchParams, + static SearchIterator::UP create(fef::TermFieldMatchData &tmd, const MatchParams &matchParams, const std::vector<int32_t> &weights, const std::vector<IDirectPostingStore::LookupResult> &dict_entries, - const IDocidWithWeightPostingStore &attr, - bool strict); + const IDocidWithWeightPostingStore &attr, bool strict, + bool readonly_scores_heap); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h index 88f0c9288f9..f0a359b98df 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h @@ -14,6 +14,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <cmath> +namespace search::queryeval { class WeakAndHeap; } namespace search::queryeval::wand { //----------------------------------------------------------------------------- @@ -22,9 +23,29 @@ using score_t = int64_t; using docid_t = uint32_t; using ref_t = uint16_t; +const uint32_t DEFAULT_PARALLEL_WAND_SCORES_ADJUST_FREQUENCY = 4; + //----------------------------------------------------------------------------- /** + * Params used to tweak the behavior of the WAND algorithm. + */ +struct MatchParams +{ + WeakAndHeap &scores; + score_t scoreThreshold; + const uint32_t scoresAdjustFrequency; + MatchParams(WeakAndHeap &scores_in) noexcept + : MatchParams(scores_in, 1, DEFAULT_PARALLEL_WAND_SCORES_ADJUST_FREQUENCY) + {} + MatchParams(WeakAndHeap &scores_in, score_t scoreThreshold_in, uint32_t scoresAdjustFrequency_in) noexcept + : scores(scores_in), + scoreThreshold(scoreThreshold_in), + scoresAdjustFrequency(scoresAdjustFrequency_in) + {} +}; + +/** * Wrapper used to specify underlying terms during setup **/ struct Term { @@ -34,12 +55,18 @@ struct Term { uint32_t estHits; fef::TermFieldMatchData *matchData; score_t maxScore = 0.0; // <- only used by rise wand test + size_t old_idx = 0; // reverse-mapping used by iterator profiling Term(SearchIterator *s, int32_t w, uint32_t e, fef::TermFieldMatchData *tfmd) noexcept : search(s), weight(w), estHits(e), matchData(tfmd) {} Term() noexcept : Term(nullptr, 0, 0, nullptr){} Term(SearchIterator *s, int32_t w, uint32_t e) noexcept : Term(s, w, e, nullptr) {} Term(SearchIterator::UP s, int32_t w, uint32_t e) noexcept : Term(s.release(), w, e, nullptr) {} + Term copy_from(size_t idx) const noexcept { + Term result = *this; + result.old_idx = idx; + return result; + } }; using Terms = std::vector<Term>; //----------------------------------------------------------------------------- @@ -242,6 +269,13 @@ public: void unpack(uint16_t ref, uint32_t docid) { iteratorPack().unpack(ref, docid); } void visit_members(vespalib::ObjectVisitor &visitor) const; const Terms &input_terms() const { return _terms; } + void transform_children(auto f) { + iteratorPack().transform_children([&](auto itr, size_t idx){ + auto ret = f(std::move(itr), _terms[idx].old_idx); + _terms[idx].search = ret.get(); + return ret; + }); + } }; template <typename Scorer> @@ -250,7 +284,7 @@ VectorizedIteratorTerms::VectorizedIteratorTerms(const Terms &t, const Scorer & : _terms() { std::vector<ref_t> order = init_state<Scorer>(TermInput(t), scorer, docIdLimit); - _terms = assemble([&t](ref_t ref){ return t[ref]; }, order); + _terms = assemble([&t](ref_t ref){ return t[ref].copy_from(ref); }, order); iteratorPack() = SearchIteratorPack(assemble([&t](ref_t ref){ return t[ref].search; }, order), assemble([&t](ref_t ref){ return t[ref].matchData; }, order), std::move(childrenMatchData)); diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.cpp index d4b92fd67e6..53ebb33e1ea 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.cpp @@ -4,23 +4,28 @@ namespace search::queryeval { -SharedWeakAndPriorityQueue::SharedWeakAndPriorityQueue(uint32_t scoresToTrack) : +WeakAndPriorityQueue::WeakAndPriorityQueue(uint32_t scoresToTrack) : WeakAndHeap(scoresToTrack), - _bestScores(), - _lock() -{ - _bestScores.reserve(scoresToTrack); -} + _bestScores() +{ } -SharedWeakAndPriorityQueue::~SharedWeakAndPriorityQueue() = default; +WeakAndPriorityQueue::~WeakAndPriorityQueue() = default; + +std::unique_ptr<WeakAndPriorityQueue> +WeakAndPriorityQueue::createHeap(uint32_t scoresToTrack, bool thread_safe) { + if (thread_safe) { + return std::make_unique<queryeval::SharedWeakAndPriorityQueue>(scoresToTrack); + } + return std::make_unique<WeakAndPriorityQueue>(scoresToTrack); +} void -SharedWeakAndPriorityQueue::adjust(score_t *begin, score_t *end) +WeakAndPriorityQueue::adjust(score_t *begin, score_t *end) { if (getScoresToTrack() == 0) { return; } - std::lock_guard guard(_lock); + for (score_t *itr = begin; itr != end; ++itr) { score_t score = *itr; if (!is_full()) { @@ -35,4 +40,17 @@ SharedWeakAndPriorityQueue::adjust(score_t *begin, score_t *end) } } +SharedWeakAndPriorityQueue::SharedWeakAndPriorityQueue(uint32_t scoresToTrack) + : WeakAndPriorityQueue(scoresToTrack), + _lock() +{ } + +SharedWeakAndPriorityQueue::~SharedWeakAndPriorityQueue() = default; + +void +SharedWeakAndPriorityQueue::adjust(score_t *begin, score_t *end) { + std::lock_guard guard(_lock); + WeakAndPriorityQueue::adjust(begin, end); +} + } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h index f1c90f5e6ac..b2287a3b79c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h @@ -17,13 +17,13 @@ namespace search::queryeval { class WeakAndHeap { public: using score_t = wand::score_t; - WeakAndHeap(uint32_t scoresToTrack) : + explicit WeakAndHeap(uint32_t scoresToTrack) noexcept : _minScore((scoresToTrack == 0) ? std::numeric_limits<score_t>::max() : 0), _scoresToTrack(scoresToTrack) { } - virtual ~WeakAndHeap() {} + virtual ~WeakAndHeap() = default; /** * Consider the given scores for insertion into the underlying structure. * The implementation may change the given score array to speed up execution. @@ -33,11 +33,13 @@ public: /** * The number of scores this heap is tracking. **/ - uint32_t getScoresToTrack() const { return _scoresToTrack; } + uint32_t getScoresToTrack() const noexcept { return _scoresToTrack; } - score_t getMinScore() const { return _minScore.load(std::memory_order_relaxed); } + score_t getMinScore() const noexcept { return _minScore.load(std::memory_order_relaxed); } protected: - void setMinScore(score_t minScore) { _minScore.store(minScore, std::memory_order_relaxed); } + void setMinScore(score_t minScore) noexcept { + _minScore.store(minScore, std::memory_order_relaxed); + } private: std::atomic<score_t> _minScore; const uint32_t _scoresToTrack; @@ -47,19 +49,29 @@ private: * An implementation using an underlying priority queue to keep track of the N * best hits that can be shared among multiple search iterators. */ -class SharedWeakAndPriorityQueue : public WeakAndHeap +class WeakAndPriorityQueue : public WeakAndHeap { private: using Scores = vespalib::PriorityQueue<score_t>; Scores _bestScores; - std::mutex _lock; - bool is_full() const { return (_bestScores.size() >= getScoresToTrack()); } + bool is_full() const noexcept { return (_bestScores.size() >= getScoresToTrack()); } +public: + explicit WeakAndPriorityQueue(uint32_t scoresToTrack); + ~WeakAndPriorityQueue() override; + Scores &getScores() noexcept { return _bestScores; } + void adjust(score_t *begin, score_t *end) override; + static std::unique_ptr<WeakAndPriorityQueue> createHeap(uint32_t scoresToTrack, bool thread_safe); + void set_min_score(score_t min_score) noexcept { setMinScore(min_score); } +}; +class SharedWeakAndPriorityQueue final : public WeakAndPriorityQueue +{ +private: + std::mutex _lock; public: - SharedWeakAndPriorityQueue(uint32_t scoresToTrack); - ~SharedWeakAndPriorityQueue(); - Scores &getScores() { return _bestScores; } + explicit SharedWeakAndPriorityQueue(uint32_t scoresToTrack); + ~SharedWeakAndPriorityQueue() override; void adjust(score_t *begin, score_t *end) override; }; diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp index cf3fd44ad4f..6f0a4b8f825 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "weak_and_search.h" -#include "wand_parts.h" +#include "weak_and_heap.h" #include <vespa/searchlib/queryeval/orsearch.h> #include <vespa/vespalib/util/left_right_heap.h> #include <vespa/vespalib/util/priority_queue.h> @@ -20,8 +20,10 @@ private: DualHeap<FutureHeap, PastHeap> _heaps; Algorithm _algo; score_t _threshold; // current score threshold - Scores _scores; // best n scores + MatchParams _matchParams; + std::vector<score_t> _localScores; const uint32_t _n; + const bool _readonly_scores_heap; void seek_strict(uint32_t docid) { _algo.set_candidate(_terms, _heaps, docid); @@ -40,24 +42,36 @@ private: } } } + void updateThreshold(score_t newThreshold) { + if (newThreshold > _threshold) { + _threshold = newThreshold; + } + } public: template<typename Scorer> - WeakAndSearchLR(const Terms &terms, const Scorer & scorer, uint32_t n) + WeakAndSearchLR(const Terms &terms, const MatchParams & matchParams, const Scorer & scorer, uint32_t n, bool readonly_scores_heap) : _terms(terms, scorer, 0, {}), _heaps(DocIdOrder(_terms.docId()), _terms.size()), _algo(), - _threshold(1), - _scores(), - _n(n) + _threshold(matchParams.scoreThreshold), + _matchParams(matchParams), + _localScores(), + _n(n), + _readonly_scores_heap(readonly_scores_heap) { + _localScores.reserve(_matchParams.scoresAdjustFrequency); } size_t get_num_terms() const override { return _terms.size(); } int32_t get_term_weight(size_t idx) const override { return _terms.weight(idx); } score_t get_max_score(size_t idx) const override { return _terms.maxScore(idx); } const Terms &getTerms() const override { return _terms.input_terms(); } uint32_t getN() const override { return _n; } + void transform_children(std::function<SearchIterator::UP(SearchIterator::UP, size_t)> f) override { + _terms.transform_children(std::move(f)); + } void doSeek(uint32_t docid) override { + updateThreshold(_matchParams.scores.getMinScore()); if (IS_STRICT) { seek_strict(docid); } else { @@ -66,12 +80,13 @@ public: } void doUnpack(uint32_t docid) override { _algo.find_matching_terms(_terms, _heaps); - _scores.push(_algo.get_upper_bound()); - if (_scores.size() > _n) { - _scores.pop_front(); - } - if (_scores.size() == _n) { - _threshold = _scores.front(); + if (!_readonly_scores_heap) { + score_t score = _algo.get_upper_bound(); + _localScores.push_back(score); + if (_localScores.size() == _matchParams.scoresAdjustFrequency) { + _matchParams.scores.adjust(&_localScores[0], &_localScores[0] + _localScores.size()); + _localScores.clear(); + } } ref_t *end = _heaps.present_end(); for (ref_t *ref = _heaps.present_begin(); ref != end; ++ref) { @@ -105,48 +120,52 @@ WeakAndSearch::visitMembers(vespalib::ObjectVisitor &visitor) const template<typename Scorer> SearchIterator::UP -WeakAndSearch::createArrayWand(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict) +WeakAndSearch::createArrayWand(const Terms &terms, const MatchParams & params, + const Scorer & scorer, uint32_t n, bool strict, + bool readonly_scores_heap) { if (strict) { - return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>>(terms, scorer, n); + return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>>(terms, params, scorer, n, readonly_scores_heap); } else { - return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>>(terms, scorer, n); + return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>>(terms, params, scorer, n, readonly_scores_heap); } } template<typename Scorer> SearchIterator::UP -WeakAndSearch::createHeapWand(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict) +WeakAndSearch::createHeapWand(const Terms &terms, const MatchParams & params, + const Scorer & scorer, uint32_t n, bool strict, + bool readonly_scores_heap) { if (strict) { - return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, true>>(terms, scorer, n); + return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, true>>(terms, params, scorer, n, readonly_scores_heap); } else { - return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, false>>(terms, scorer, n); + return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, false>>(terms, params, scorer, n, readonly_scores_heap); } } template<typename Scorer> SearchIterator::UP -WeakAndSearch::create(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict) +WeakAndSearch::create(const Terms &terms, const MatchParams & params, const Scorer & scorer, uint32_t n, bool strict, bool readonly_scores_heap) { if (terms.size() < 128) { - return createArrayWand(terms, scorer, n, strict); + return createArrayWand(terms, params, scorer, n, strict, readonly_scores_heap); } else { - return createHeapWand(terms, scorer, n, strict); + return createHeapWand(terms, params, scorer, n, strict, readonly_scores_heap); } } SearchIterator::UP -WeakAndSearch::create(const Terms &terms, uint32_t n, bool strict) +WeakAndSearch::create(const Terms &terms, const MatchParams & params, uint32_t n, bool strict, bool readonly_scores_heap) { - return create(terms, wand::TermFrequencyScorer(), n, strict); + return create(terms, params, wand::TermFrequencyScorer(), n, strict, readonly_scores_heap); } //----------------------------------------------------------------------------- -template SearchIterator::UP WeakAndSearch::create<wand::TermFrequencyScorer>(const Terms &terms, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict); -template SearchIterator::UP WeakAndSearch::create<wand::Bm25TermFrequencyScorer>(const Terms &terms, const wand::Bm25TermFrequencyScorer & scorer, uint32_t n, bool strict); -template SearchIterator::UP WeakAndSearch::createArrayWand<wand::TermFrequencyScorer>(const Terms &terms, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict); -template SearchIterator::UP WeakAndSearch::createHeapWand<wand::TermFrequencyScorer>(const Terms &terms, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict); +template SearchIterator::UP WeakAndSearch::create<wand::TermFrequencyScorer>(const Terms &terms, const MatchParams & params, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict, bool readonly_scores_heap); +template SearchIterator::UP WeakAndSearch::create<wand::Bm25TermFrequencyScorer>(const Terms &terms, const MatchParams & params, const wand::Bm25TermFrequencyScorer & scorer, uint32_t n, bool strict, bool readonly_scores_heap); +template SearchIterator::UP WeakAndSearch::createArrayWand<wand::TermFrequencyScorer>(const Terms &terms, const MatchParams & params, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict, bool readonly_scores_heap); +template SearchIterator::UP WeakAndSearch::createHeapWand<wand::TermFrequencyScorer>(const Terms &terms, const MatchParams & params, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict, bool readonly_scores_heap); } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h index a91b2860a63..4032a7b1f5e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h @@ -9,19 +9,25 @@ namespace search::queryeval { struct WeakAndSearch : SearchIterator { using Terms = wand::Terms; + using MatchParams = wand::MatchParams; virtual size_t get_num_terms() const = 0; virtual int32_t get_term_weight(size_t idx) const = 0; virtual wand::score_t get_max_score(size_t idx) const = 0; virtual const Terms &getTerms() const = 0; virtual uint32_t getN() const = 0; + WeakAndSearch *as_weak_and() noexcept override { return this; } void visitMembers(vespalib::ObjectVisitor &visitor) const override; template<typename Scorer> - static SearchIterator::UP createArrayWand(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict); + static SearchIterator::UP createArrayWand(const Terms &terms, const MatchParams & matchParams, + const Scorer & scorer, uint32_t n, bool strict, bool readonly_scores_heap); template<typename Scorer> - static SearchIterator::UP createHeapWand(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict); + static SearchIterator::UP createHeapWand(const Terms &terms, const MatchParams & matchParams, + const Scorer & scorer, uint32_t n, bool strict, bool readonly_scores_heap); template<typename Scorer> - static SearchIterator::UP create(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict); - static SearchIterator::UP create(const Terms &terms, uint32_t n, bool strict); + static SearchIterator::UP create(const Terms &terms, const MatchParams & matchParams, + const Scorer & scorer, uint32_t n, bool strict, bool readonly_scores_heap); + static SearchIterator::UP create(const Terms &terms, const MatchParams & matchParams, + uint32_t n, bool strict, bool readonly_scores_heap); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp index d1ebb1f4e4e..3bdfd3c0e7b 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp @@ -2,7 +2,7 @@ #include "angular_distance.h" #include "temporary_vector_store.h" -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <numbers> #include <cmath> @@ -13,16 +13,17 @@ using vespalib::eval::Int8Float; namespace search::tensor { -template<typename FloatType> +template <typename VectorStoreType> class BoundAngularDistance final : public BoundDistanceFunction { private: - const vespalib::hwaccelrated::IAccelrated & _computer; - mutable TemporaryVectorStore<FloatType> _tmpSpace; + using FloatType = VectorStoreType::FloatType; + const vespalib::hwaccelerated::IAccelerated & _computer; + mutable VectorStoreType _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs; double _lhs_norm_sq; public: explicit BoundAngularDistance(TypedCells lhs) - : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), + : _computer(vespalib::hwaccelerated::IAccelerated::getAccelerator()), _tmpSpace(lhs.size), _lhs(_tmpSpace.storeLhs(lhs)) { @@ -66,21 +67,30 @@ public: } }; -template class BoundAngularDistance<float>; -template class BoundAngularDistance<double>; +template class BoundAngularDistance<TemporaryVectorStore<float>>; +template class BoundAngularDistance<TemporaryVectorStore<double>>; +template class BoundAngularDistance<TemporaryVectorStore<Int8Float>>; +template class BoundAngularDistance<ReferenceVectorStore<float>>; +template class BoundAngularDistance<ReferenceVectorStore<double>>; +template class BoundAngularDistance<ReferenceVectorStore<Int8Float>>; template <typename FloatType> BoundDistanceFunction::UP AngularDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) const { - using DFT = BoundAngularDistance<FloatType>; + using DFT = BoundAngularDistance<TemporaryVectorStore<FloatType>>; return std::make_unique<DFT>(lhs); } template <typename FloatType> BoundDistanceFunction::UP AngularDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) const { - using DFT = BoundAngularDistance<FloatType>; - return std::make_unique<DFT>(lhs); + if (_reference_insertion_vector) { + using DFT = BoundAngularDistance<ReferenceVectorStore<FloatType>>; + return std::make_unique<DFT>(lhs); + } else { + using DFT = BoundAngularDistance<TemporaryVectorStore<FloatType>>; + return std::make_unique<DFT>(lhs); + } } template class AngularDistanceFunctionFactory<float>; diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.h b/searchlib/src/vespa/searchlib/tensor/angular_distance.h index aa51f58b3cd..7dcc6e80484 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.h @@ -10,11 +10,19 @@ namespace search::tensor { * Calculates angular distance between vectors * Will use instruction optimal for the cpu it is running on * after converting both vectors to an optimal cell type. + * + * When reference_insertion_vector == true: + * - Vectors passed to for_insertion_vector() and BoundDistanceFunction::calc() are assumed to have the same type as FloatType. + * - The TypedCells memory is just referenced and used directly in calculations, + * and thus no transformation via a temporary memory buffer occurs. */ template <typename FloatType> class AngularDistanceFunctionFactory : public DistanceFunctionFactory { +private: + bool _reference_insertion_vector; public: - AngularDistanceFunctionFactory() = default; + AngularDistanceFunctionFactory() noexcept : AngularDistanceFunctionFactory(false) {} + AngularDistanceFunctionFactory(bool reference_insertion_vector) noexcept : _reference_insertion_vector(reference_insertion_vector) {} BoundDistanceFunction::UP for_query_vector(TypedCells lhs) const override; BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index f39994dfdcf..b8918e23ce7 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -16,40 +16,44 @@ make_distance_function_factory(DistanceMetric variant, CellType cell_type) switch (variant) { case DistanceMetric::Angular: switch (cell_type) { - case CellType::DOUBLE: return std::make_unique<AngularDistanceFunctionFactory<double>>(); - case CellType::INT8: return std::make_unique<AngularDistanceFunctionFactory<Int8Float>>(); + case CellType::DOUBLE: return std::make_unique<AngularDistanceFunctionFactory<double>>(true); + case CellType::INT8: return std::make_unique<AngularDistanceFunctionFactory<Int8Float>>(true); + case CellType::FLOAT: return std::make_unique<AngularDistanceFunctionFactory<float>>(true); default: return std::make_unique<AngularDistanceFunctionFactory<float>>(); } case DistanceMetric::Euclidean: switch (cell_type) { - case CellType::DOUBLE: return std::make_unique<EuclideanDistanceFunctionFactory<double>>(); - case CellType::INT8: return std::make_unique<EuclideanDistanceFunctionFactory<Int8Float>>(); - case CellType::BFLOAT16: return std::make_unique<EuclideanDistanceFunctionFactory<vespalib::BFloat16>>(); + case CellType::DOUBLE: return std::make_unique<EuclideanDistanceFunctionFactory<double>>(true); + case CellType::INT8: return std::make_unique<EuclideanDistanceFunctionFactory<Int8Float>>(true); + case CellType::FLOAT: return std::make_unique<EuclideanDistanceFunctionFactory<float>>(true); default: return std::make_unique<EuclideanDistanceFunctionFactory<float>>(); } case DistanceMetric::InnerProduct: case DistanceMetric::PrenormalizedAngular: switch (cell_type) { - case CellType::DOUBLE: return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<double>>(); - case CellType::INT8: return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<Int8Float>>(); - default: return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<float>>(); + case CellType::DOUBLE: return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<double>>(true); + case CellType::INT8: return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<Int8Float>>(true); + case CellType::FLOAT: return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<float>>(true); + default: return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<float>>(); } case DistanceMetric::Dotproduct: switch (cell_type) { - case CellType::DOUBLE: return std::make_unique<MipsDistanceFunctionFactory<double>>(); - case CellType::INT8: return std::make_unique<MipsDistanceFunctionFactory<Int8Float>>(); + case CellType::DOUBLE: return std::make_unique<MipsDistanceFunctionFactory<double>>(true); + case CellType::INT8: return std::make_unique<MipsDistanceFunctionFactory<Int8Float>>(true); + case CellType::FLOAT: return std::make_unique<MipsDistanceFunctionFactory<float>>(true); default: return std::make_unique<MipsDistanceFunctionFactory<float>>(); } case DistanceMetric::GeoDegrees: return std::make_unique<GeoDistanceFunctionFactory>(); case DistanceMetric::Hamming: switch (cell_type) { - case CellType::DOUBLE: return std::make_unique<HammingDistanceFunctionFactory<double>>(); - case CellType::INT8: return std::make_unique<HammingDistanceFunctionFactory<Int8Float>>(); + case CellType::DOUBLE: return std::make_unique<HammingDistanceFunctionFactory<double>>(true); + case CellType::INT8: return std::make_unique<HammingDistanceFunctionFactory<Int8Float>>(true); + case CellType::FLOAT: return std::make_unique<HammingDistanceFunctionFactory<float>>(true); default: return std::make_unique<HammingDistanceFunctionFactory<float>>(); } } - // not reached: + // Not reached: return {}; } diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp index 62b92b43ad9..6c4be1e12aa 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp @@ -2,7 +2,7 @@ #include "euclidean_distance.h" #include "temporary_vector_store.h" -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <cmath> using vespalib::typify_invoke; @@ -12,18 +12,17 @@ using vespalib::eval::TypedCells; namespace search::tensor { using vespalib::eval::Int8Float; -using vespalib::BFloat16; -template<typename AttributeCellType> +template <typename VectorStoreType> class BoundEuclideanDistance final : public BoundDistanceFunction { - using FloatType = std::conditional_t<std::is_same<AttributeCellType, BFloat16>::value, float, AttributeCellType>; private: - const vespalib::hwaccelrated::IAccelrated & _computer; - mutable TemporaryVectorStore<FloatType> _tmpSpace; + using FloatType = VectorStoreType::FloatType; + const vespalib::hwaccelerated::IAccelerated & _computer; + mutable VectorStoreType _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs_vector; public: explicit BoundEuclideanDistance(TypedCells lhs) - : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), + : _computer(vespalib::hwaccelerated::IAccelerated::getAccelerator()), _tmpSpace(lhs.size), _lhs_vector(_tmpSpace.storeLhs(lhs)) {} @@ -46,27 +45,33 @@ public: } }; -template class BoundEuclideanDistance<Int8Float>; -template class BoundEuclideanDistance<BFloat16>; -template class BoundEuclideanDistance<float>; -template class BoundEuclideanDistance<double>; +template class BoundEuclideanDistance<TemporaryVectorStore<Int8Float>>; +template class BoundEuclideanDistance<TemporaryVectorStore<float>>; +template class BoundEuclideanDistance<TemporaryVectorStore<double>>; +template class BoundEuclideanDistance<ReferenceVectorStore<Int8Float>>; +template class BoundEuclideanDistance<ReferenceVectorStore<float>>; +template class BoundEuclideanDistance<ReferenceVectorStore<double>>; template <typename FloatType> BoundDistanceFunction::UP EuclideanDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) const { - using DFT = BoundEuclideanDistance<FloatType>; + using DFT = BoundEuclideanDistance<TemporaryVectorStore<FloatType>>; return std::make_unique<DFT>(lhs); } template <typename FloatType> BoundDistanceFunction::UP EuclideanDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) const { - using DFT = BoundEuclideanDistance<FloatType>; - return std::make_unique<DFT>(lhs); + if (_reference_insertion_vector) { + using DFT = BoundEuclideanDistance<ReferenceVectorStore<FloatType>>; + return std::make_unique<DFT>(lhs); + } else { + using DFT = BoundEuclideanDistance<TemporaryVectorStore<FloatType>>; + return std::make_unique<DFT>(lhs); + } } template class EuclideanDistanceFunctionFactory<Int8Float>; -template class EuclideanDistanceFunctionFactory<BFloat16>; template class EuclideanDistanceFunctionFactory<float>; template class EuclideanDistanceFunctionFactory<double>; diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h index 78460c93307..bd82e48fb0b 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h @@ -10,11 +10,19 @@ namespace search::tensor { * Calculates the square of the standard Euclidean distance. * Will use instruction optimal for the cpu it is running on * after converting both vectors to an optimal cell type. + * + * When reference_insertion_vector == true: + * - Vectors passed to for_insertion_vector() and BoundDistanceFunction::calc() are assumed to have the same type as FloatType. + * - The TypedCells memory is just referenced and used directly in calculations, + * and thus no transformation via a temporary memory buffer occurs. */ template <typename FloatType> class EuclideanDistanceFunctionFactory : public DistanceFunctionFactory { +private: + bool _reference_insertion_vector; public: - EuclideanDistanceFunctionFactory() noexcept = default; + EuclideanDistanceFunctionFactory() noexcept : EuclideanDistanceFunctionFactory(false) {} + EuclideanDistanceFunctionFactory(bool reference_insertion_vector) noexcept : _reference_insertion_vector(reference_insertion_vector) {} BoundDistanceFunction::UP for_query_vector(TypedCells lhs) const override; BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp index 7ea2e440a51..281a20cef87 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp @@ -5,17 +5,18 @@ #include <vespa/vespalib/util/binary_hamming_distance.h> using vespalib::typify_invoke; -using vespalib::eval::TypifyCellType; using vespalib::eval::TypedCells; +using vespalib::eval::TypifyCellType; namespace search::tensor { using vespalib::eval::Int8Float; -template<typename FloatType> +template <typename VectorStoreType> class BoundHammingDistance final : public BoundDistanceFunction { private: - mutable TemporaryVectorStore<FloatType> _tmpSpace; + using FloatType = VectorStoreType::FloatType; + mutable VectorStoreType _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs_vector; public: explicit BoundHammingDistance(TypedCells lhs) @@ -47,18 +48,30 @@ public: } }; +template class BoundHammingDistance<TemporaryVectorStore<Int8Float>>; +template class BoundHammingDistance<TemporaryVectorStore<float>>; +template class BoundHammingDistance<TemporaryVectorStore<double>>; +template class BoundHammingDistance<ReferenceVectorStore<Int8Float>>; +template class BoundHammingDistance<ReferenceVectorStore<float>>; +template class BoundHammingDistance<ReferenceVectorStore<double>>; + template <typename FloatType> BoundDistanceFunction::UP HammingDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) const { - using DFT = BoundHammingDistance<FloatType>; + using DFT = BoundHammingDistance<TemporaryVectorStore<FloatType>>; return std::make_unique<DFT>(lhs); } template <typename FloatType> BoundDistanceFunction::UP HammingDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) const { - using DFT = BoundHammingDistance<FloatType>; - return std::make_unique<DFT>(lhs); + if (_reference_insertion_vector) { + using DFT = BoundHammingDistance<ReferenceVectorStore<FloatType>>; + return std::make_unique<DFT>(lhs); + } else { + using DFT = BoundHammingDistance<TemporaryVectorStore<FloatType>>; + return std::make_unique<DFT>(lhs); + } } template class HammingDistanceFunctionFactory<Int8Float>; diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.h b/searchlib/src/vespa/searchlib/tensor/hamming_distance.h index 2e3b75cc61f..768665653c0 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.h @@ -7,15 +7,22 @@ namespace search::tensor { /** - * Calculates the Hamming distance defined as - * "number of cells where the values are different" - * or (for int8 cells, aka binary data only) - * "number of bits that are different" + * Calculates the Hamming distance defined as "number of cells where the values are different" + * or (for int8 cells, aka binary data only) "number of bits that are different". + * + * When reference_insertion_vector == true: + * - Vectors passed to for_insertion_vector() and BoundDistanceFunction::calc() are assumed to have the same type as FloatType. + * - The TypedCells memory is referenced and used directly in calculations, + * and thus no transformation via a temporary memory buffer occurs. */ template <typename FloatType> class HammingDistanceFunctionFactory : public DistanceFunctionFactory { +private: + bool _reference_insertion_vector; public: - HammingDistanceFunctionFactory() = default; + HammingDistanceFunctionFactory() noexcept : HammingDistanceFunctionFactory(false) {} + HammingDistanceFunctionFactory(bool reference_insertion_vector) noexcept : _reference_insertion_vector(reference_insertion_vector) {} + BoundDistanceFunction::UP for_query_vector(TypedCells lhs) const override; BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h index a1a9e9632be..88b7681f23c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -82,7 +82,7 @@ struct HnswGraph { if (levels_ref.valid()) { return levels_store.get(levels_ref); } - return LevelArrayRef(); + return {}; } LevelArrayRef get_level_array(uint32_t nodeid) const { @@ -102,7 +102,7 @@ struct HnswGraph { return links_store.get(links_ref); } } - return LinkArrayRef(); + return {}; } LinkArrayRef get_link_array(uint32_t nodeid, uint32_t level) const { @@ -126,12 +126,12 @@ struct HnswGraph { uint32_t nodeid; LevelsRef levels_ref; int32_t level; - EntryNode() + EntryNode() noexcept : nodeid(0), // Note that nodeid 0 is reserved and never used levels_ref(), level(-1) {} - EntryNode(uint32_t nodeid_in, LevelsRef levels_ref_in, int32_t level_in) + EntryNode(uint32_t nodeid_in, LevelsRef levels_ref_in, int32_t level_in) noexcept : nodeid(nodeid_in), levels_ref(levels_ref_in), level(level_in) diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index b542c422f50..c815d5e7ed3 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -42,7 +42,7 @@ constexpr float alloc_grow_factor = 0.3; // TODO: Adjust these numbers to what we accept as max in config. constexpr size_t max_level_array_size = 16; constexpr size_t max_link_array_size = 193; -constexpr vespalib::duration MAX_COUNT_DURATION(100ms); +constexpr vespalib::duration MAX_COUNT_DURATION(1000ms); const vespalib::string hnsw_max_squared_norm = "hnsw.max_squared_norm"; @@ -155,8 +155,8 @@ HnswIndex<type>::make_default_level_array_store_config() vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, ArrayStoreConfig::default_max_buffer_size, - min_num_arrays_for_new_buffer, - alloc_grow_factor).enable_free_lists(true); + min_num_arrays_for_new_buffer, + alloc_grow_factor).enable_free_lists(true); } template <HnswIndexType type> @@ -373,9 +373,7 @@ HnswIndex<type>::estimate_visited_nodes(uint32_t level, uint32_t nodeid_limit, u template <HnswIndexType type> HnswCandidate -HnswIndex<type>::find_nearest_in_layer( - const BoundDistanceFunction &df, - const HnswCandidate& entry_point, uint32_t level) const +HnswIndex<type>::find_nearest_in_layer(const BoundDistanceFunction &df, const HnswCandidate& entry_point, uint32_t level) const { HnswCandidate nearest = entry_point; bool keep_searching = true; @@ -401,12 +399,10 @@ HnswIndex<type>::find_nearest_in_layer( template <HnswIndexType type> template <class VisitedTracker, class BestNeighbors> void -HnswIndex<type>::search_layer_helper( - const BoundDistanceFunction &df, - uint32_t neighbors_to_find, - BestNeighbors& best_neighbors, uint32_t level, const GlobalFilter *filter, - uint32_t nodeid_limit, const vespalib::Doom* const doom, - uint32_t estimated_visited_nodes) const +HnswIndex<type>::search_layer_helper(const BoundDistanceFunction &df, uint32_t neighbors_to_find, + BestNeighbors& best_neighbors, uint32_t level, const GlobalFilter *filter, + uint32_t nodeid_limit, const vespalib::Doom* const doom, + uint32_t estimated_visited_nodes) const { NearestPriQ candidates; GlobalFilterWrapper<type> filter_wrapper(filter); @@ -471,11 +467,8 @@ HnswIndex<type>::search_layer_helper( template <HnswIndexType type> template <class BestNeighbors> void -HnswIndex<type>::search_layer( - const BoundDistanceFunction &df, - uint32_t neighbors_to_find, - BestNeighbors& best_neighbors, uint32_t level, - const vespalib::Doom* const doom, const GlobalFilter *filter) const +HnswIndex<type>::search_layer(const BoundDistanceFunction &df, uint32_t neighbors_to_find, BestNeighbors& best_neighbors, + uint32_t level, const vespalib::Doom* const doom, const GlobalFilter *filter) const { uint32_t nodeid_limit = _graph.nodes_size.load(std::memory_order_acquire); uint32_t estimated_visited_nodes = estimate_visited_nodes(level, nodeid_limit, neighbors_to_find, filter); @@ -488,7 +481,7 @@ HnswIndex<type>::search_layer( template <HnswIndexType type> HnswIndex<type>::HnswIndex(const DocVectorAccess& vectors, DistanceFunctionFactory::UP distance_ff, - RandomLevelGenerator::UP level_generator, const HnswIndexConfig& cfg) + RandomLevelGenerator::UP level_generator, const HnswIndexConfig& cfg) : _graph(), _vectors(vectors), _distance_ff(std::move(distance_ff)), @@ -633,9 +626,7 @@ HnswIndex<type>::internal_complete_add_node(uint32_t nodeid, uint32_t docid, uin template <HnswIndexType type> std::unique_ptr<PrepareResult> -HnswIndex<type>::prepare_add_document(uint32_t docid, - VectorBundle vectors, - vespalib::GenerationHandler::Guard read_guard) const +HnswIndex<type>::prepare_add_document(uint32_t docid, VectorBundle vectors, vespalib::GenerationHandler::Guard read_guard) const { uint32_t active_nodes = _graph.get_active_nodes(); if (active_nodes < _cfg.min_size_before_two_phase()) { @@ -930,12 +921,8 @@ struct NeighborsByDocId { template <HnswIndexType type> std::vector<NearestNeighborIndex::Neighbor> -HnswIndex<type>::top_k_by_docid( - uint32_t k, - const BoundDistanceFunction &df, - const GlobalFilter *filter, uint32_t explore_k, - const vespalib::Doom& doom, - double distance_threshold) const +HnswIndex<type>::top_k_by_docid(uint32_t k, const BoundDistanceFunction &df, const GlobalFilter *filter, + uint32_t explore_k, const vespalib::Doom& doom, double distance_threshold) const { SearchBestNeighbors candidates = top_k_candidates(df, std::max(k, explore_k), filter, doom); auto result = candidates.get_neighbors(k, distance_threshold); @@ -945,34 +932,23 @@ HnswIndex<type>::top_k_by_docid( template <HnswIndexType type> std::vector<NearestNeighborIndex::Neighbor> -HnswIndex<type>::find_top_k( - uint32_t k, - const BoundDistanceFunction &df, - uint32_t explore_k, - const vespalib::Doom& doom, - double distance_threshold) const +HnswIndex<type>::find_top_k(uint32_t k, const BoundDistanceFunction &df, uint32_t explore_k, + const vespalib::Doom& doom, double distance_threshold) const { return top_k_by_docid(k, df, nullptr, explore_k, doom, distance_threshold); } template <HnswIndexType type> std::vector<NearestNeighborIndex::Neighbor> -HnswIndex<type>::find_top_k_with_filter( - uint32_t k, - const BoundDistanceFunction &df, - const GlobalFilter &filter, uint32_t explore_k, - const vespalib::Doom& doom, - double distance_threshold) const +HnswIndex<type>::find_top_k_with_filter(uint32_t k, const BoundDistanceFunction &df, const GlobalFilter &filter, + uint32_t explore_k, const vespalib::Doom& doom, double distance_threshold) const { return top_k_by_docid(k, df, &filter, explore_k, doom, distance_threshold); } template <HnswIndexType type> typename HnswIndex<type>::SearchBestNeighbors -HnswIndex<type>::top_k_candidates( - const BoundDistanceFunction &df, - uint32_t k, const GlobalFilter *filter, - const vespalib::Doom& doom) const +HnswIndex<type>::top_k_candidates(const BoundDistanceFunction &df, uint32_t k, const GlobalFilter *filter, const vespalib::Doom& doom) const { SearchBestNeighbors best_neighbors; auto entry = _graph.get_entry_node(); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 4d4440c1bcb..8c74f1e5264 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -171,24 +171,19 @@ protected: /** * Performs a greedy search in the given layer to find the candidate that is nearest the input vector. */ - HnswCandidate find_nearest_in_layer(const BoundDistanceFunction &df, const HnswCandidate& entry_point, uint32_t level) const; + HnswCandidate find_nearest_in_layer(const BoundDistanceFunction &df, const HnswCandidate& entry_point, uint32_t level) const __attribute__((noinline)); template <class VisitedTracker, class BestNeighbors> void search_layer_helper(const BoundDistanceFunction &df, uint32_t neighbors_to_find, BestNeighbors& best_neighbors, - uint32_t level, const GlobalFilter *filter, - uint32_t nodeid_limit, - const vespalib::Doom* const doom, - uint32_t estimated_visited_nodes) const; + uint32_t level, const GlobalFilter *filter, uint32_t nodeid_limit, + const vespalib::Doom* const doom, uint32_t estimated_visited_nodes) const __attribute__((noinline)); template <class BestNeighbors> void search_layer(const BoundDistanceFunction &df, uint32_t neighbors_to_find, BestNeighbors& best_neighbors, - uint32_t level, const vespalib::Doom* const doom, - const GlobalFilter *filter = nullptr) const; - std::vector<Neighbor> top_k_by_docid(uint32_t k, const BoundDistanceFunction &df, - const GlobalFilter *filter, uint32_t explore_k, - const vespalib::Doom& doom, - double distance_threshold) const; + uint32_t level, const vespalib::Doom* const doom, const GlobalFilter *filter = nullptr) const; + std::vector<Neighbor> top_k_by_docid(uint32_t k, const BoundDistanceFunction &df, const GlobalFilter *filter, + uint32_t explore_k, const vespalib::Doom& doom, double distance_threshold) const; internal::PreparedAddDoc internal_prepare_add(uint32_t docid, VectorBundle input_vectors, - vespalib::GenerationHandler::Guard read_guard) const; + vespalib::GenerationHandler::Guard read_guard) const; void internal_prepare_add_node(internal::PreparedAddDoc& op, TypedCells input_vector, const typename GraphType::EntryNode& entry) const; LinkArray filter_valid_nodeids(uint32_t level, const internal::PreparedAddNode::Links &neighbors, uint32_t self_nodeid); void internal_complete_add(uint32_t docid, internal::PreparedAddDoc &op); @@ -205,8 +200,7 @@ public: // Implements NearestNeighborIndex void add_document(uint32_t docid) override; - std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid, - VectorBundle vectors, + std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid, VectorBundle vectors, vespalib::GenerationHandler::Guard read_guard) const override; void complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> prepare_result) override; void remove_node(uint32_t nodeid); @@ -225,26 +219,16 @@ public: std::unique_ptr<NearestNeighborIndexSaver> make_saver(vespalib::GenericHeader& header) const override; std::unique_ptr<NearestNeighborIndexLoader> make_loader(FastOS_FileInterface& file, const vespalib::GenericHeader& header) override; - std::vector<Neighbor> find_top_k( - uint32_t k, - const BoundDistanceFunction &df, - uint32_t explore_k, - const vespalib::Doom& doom, - double distance_threshold) const override; + std::vector<Neighbor> find_top_k(uint32_t k, const BoundDistanceFunction &df, uint32_t explore_k, + const vespalib::Doom& doom, double distance_threshold) const override; - std::vector<Neighbor> find_top_k_with_filter( - uint32_t k, - const BoundDistanceFunction &df, - const GlobalFilter &filter, uint32_t explore_k, - const vespalib::Doom& doom, - double distance_threshold) const override; + std::vector<Neighbor> find_top_k_with_filter(uint32_t k, const BoundDistanceFunction &df, const GlobalFilter &filter, + uint32_t explore_k, const vespalib::Doom& doom, double distance_threshold) const override; DistanceFunctionFactory &distance_function_factory() const override { return *_distance_ff; } - SearchBestNeighbors top_k_candidates( - const BoundDistanceFunction &df, - uint32_t k, const GlobalFilter *filter, - const vespalib::Doom& doom) const; + SearchBestNeighbors top_k_candidates(const BoundDistanceFunction &df, uint32_t k, const GlobalFilter *filter, + const vespalib::Doom& doom) const; uint32_t get_entry_nodeid() const { return _graph.get_entry_node().nodeid; } int32_t get_entry_level() const { return _graph.get_entry_node().level; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h index b8189090079..33a9fa2503f 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h @@ -16,10 +16,7 @@ class HnswSimpleNode { AtomicEntryRef _levels_ref; public: - HnswSimpleNode() - : _levels_ref() - { - } + HnswSimpleNode() noexcept : _levels_ref() { } AtomicEntryRef& levels_ref() noexcept { return _levels_ref; } const AtomicEntryRef& levels_ref() const noexcept { return _levels_ref; } void store_docid(uint32_t docid) noexcept { (void) docid; } diff --git a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp index 5bc727ebd97..dcaa863c509 100644 --- a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp +++ b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp @@ -2,7 +2,7 @@ #include "mips_distance_transform.h" #include "temporary_vector_store.h" -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <cmath> #include <variant> @@ -10,11 +10,13 @@ using vespalib::eval::Int8Float; namespace search::tensor { -template<typename FloatType, bool extra_dim> +template <typename VectorStoreType, bool extra_dim> class BoundMipsDistanceFunction final : public BoundDistanceFunction { - mutable TemporaryVectorStore<FloatType> _tmpSpace; +private: + using FloatType = VectorStoreType::FloatType; + mutable VectorStoreType _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs_vector; - const vespalib::hwaccelrated::IAccelrated & _computer; + const vespalib::hwaccelerated::IAccelerated & _computer; double _max_sq_norm; using ExtraDimT = std::conditional_t<extra_dim,double,std::monostate>; [[no_unique_address]] ExtraDimT _lhs_extra_dim; @@ -24,7 +26,7 @@ public: : BoundDistanceFunction(), _tmpSpace(lhs.size), _lhs_vector(_tmpSpace.storeLhs(lhs)), - _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) + _computer(vespalib::hwaccelerated::IAccelerated::getAccelerator()) { const FloatType * a = _lhs_vector.data(); if constexpr (extra_dim) { @@ -47,7 +49,7 @@ public: double dp = _computer.dotProduct(cast(a), cast(b), rhs.size); if constexpr (extra_dim) { double rhs_sq_norm = _computer.dotProduct(cast(b), cast(b), rhs.size); - // avoid sqrt(negative) for robustness: + // avoid sqrt(negative) for robustness: double diff = std::max(0.0, _max_sq_norm - rhs_sq_norm); double rhs_extra_dim = std::sqrt(diff); dp += _lhs_extra_dim * rhs_extra_dim; @@ -74,13 +76,17 @@ public: template<typename FloatType> BoundDistanceFunction::UP MipsDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) const { - return std::make_unique<BoundMipsDistanceFunction<FloatType, false>>(lhs, *_sq_norm_store); + return std::make_unique<BoundMipsDistanceFunction<TemporaryVectorStore<FloatType>, false>>(lhs, *_sq_norm_store); } template<typename FloatType> BoundDistanceFunction::UP MipsDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) const { - return std::make_unique<BoundMipsDistanceFunction<FloatType, true>>(lhs, *_sq_norm_store); + if (_reference_insertion_vector) { + return std::make_unique<BoundMipsDistanceFunction<ReferenceVectorStore<FloatType>, true>>(lhs, *_sq_norm_store); + } else { + return std::make_unique<BoundMipsDistanceFunction<TemporaryVectorStore<FloatType>, true>>(lhs, *_sq_norm_store); + } }; template class MipsDistanceFunctionFactory<Int8Float>; diff --git a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.h b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.h index 336511ab78f..7b82661179f 100644 --- a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.h +++ b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.h @@ -55,11 +55,19 @@ public: * problem. When inserting vectors, an extra dimension is * added ensuring behavior "as if" all vectors had length equal * to the longest vector inserted so far, or at least length 1. + * + * When reference_insertion_vector == true: + * - Vectors passed to for_insertion_vector() and BoundDistanceFunction::calc() are assumed to have the same type as FloatType. + * - The TypedCells memory is just referenced and used directly in calculations, + * and thus no transformation via a temporary memory buffer occurs. */ -template<typename FloatType> +template <typename FloatType> class MipsDistanceFunctionFactory : public MipsDistanceFunctionFactoryBase { +private: + bool _reference_insertion_vector; public: - MipsDistanceFunctionFactory() noexcept = default; + MipsDistanceFunctionFactory() noexcept : MipsDistanceFunctionFactory(false) {} + MipsDistanceFunctionFactory(bool reference_insertion_vector) noexcept : _reference_insertion_vector(reference_insertion_vector) {} ~MipsDistanceFunctionFactory() override = default; BoundDistanceFunction::UP for_query_vector(TypedCells lhs) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp index 6f0966e7fb3..fdfbdb5a4a4 100644 --- a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp @@ -2,24 +2,25 @@ #include "prenormalized_angular_distance.h" #include "temporary_vector_store.h" -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> -using vespalib::typify_invoke; -using vespalib::eval::TypifyCellType; using vespalib::eval::Int8Float; +using vespalib::eval::TypifyCellType; +using vespalib::typify_invoke; namespace search::tensor { -template<typename FloatType> +template <typename VectorStoreType> class BoundPrenormalizedAngularDistance final : public BoundDistanceFunction { private: - const vespalib::hwaccelrated::IAccelrated & _computer; - mutable TemporaryVectorStore<FloatType> _tmpSpace; + using FloatType = VectorStoreType::FloatType; + const vespalib::hwaccelerated::IAccelerated & _computer; + mutable VectorStoreType _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs; double _lhs_norm_sq; public: explicit BoundPrenormalizedAngularDistance(TypedCells lhs) - : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), + : _computer(vespalib::hwaccelerated::IAccelerated::getAccelerator()), _tmpSpace(lhs.size), _lhs(_tmpSpace.storeLhs(lhs)) { @@ -46,7 +47,7 @@ public: double to_rawscore(double distance) const noexcept override { double dot_product = _lhs_norm_sq - distance; double cosine_similarity = dot_product / _lhs_norm_sq; - // should be in in range [-1,1] but roundoff may cause problems: + // should be in range [-1,1] but roundoff may cause problems: cosine_similarity = std::min(1.0, cosine_similarity); cosine_similarity = std::max(-1.0, cosine_similarity); double cosine_distance = 1.0 - cosine_similarity; // in range [0,2] @@ -58,21 +59,30 @@ public: } }; -template class BoundPrenormalizedAngularDistance<float>; -template class BoundPrenormalizedAngularDistance<double>; +template class BoundPrenormalizedAngularDistance<TemporaryVectorStore<float>>; +template class BoundPrenormalizedAngularDistance<TemporaryVectorStore<double>>; +template class BoundPrenormalizedAngularDistance<TemporaryVectorStore<Int8Float>>; +template class BoundPrenormalizedAngularDistance<ReferenceVectorStore<float>>; +template class BoundPrenormalizedAngularDistance<ReferenceVectorStore<double>>; +template class BoundPrenormalizedAngularDistance<ReferenceVectorStore<Int8Float>>; template <typename FloatType> BoundDistanceFunction::UP PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) const { - using DFT = BoundPrenormalizedAngularDistance<FloatType>; + using DFT = BoundPrenormalizedAngularDistance<TemporaryVectorStore<FloatType>>; return std::make_unique<DFT>(lhs); } template <typename FloatType> BoundDistanceFunction::UP PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) const { - using DFT = BoundPrenormalizedAngularDistance<FloatType>; - return std::make_unique<DFT>(lhs); + if (_reference_insertion_vector) { + using DFT = BoundPrenormalizedAngularDistance<ReferenceVectorStore<FloatType>>; + return std::make_unique<DFT>(lhs); + } else { + using DFT = BoundPrenormalizedAngularDistance<TemporaryVectorStore<FloatType>>; + return std::make_unique<DFT>(lhs); + } } template class PrenormalizedAngularDistanceFunctionFactory<float>; diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h index 6a791e0b6ec..639138df574 100644 --- a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h @@ -9,11 +9,19 @@ namespace search::tensor { /** * Calculates inner-product "distance" between vectors assuming a common norm. * Should give same ordering as Angular distance, but is less expensive. + * + * When reference_insertion_vector == true: + * - Vectors passed to for_insertion_vector() and BoundDistanceFunction::calc() are assumed to have the same type as FloatType. + * - The TypedCells memory is just referenced and used directly in calculations, + * and thus no transformation via a temporary memory buffer occurs. */ template <typename FloatType> class PrenormalizedAngularDistanceFunctionFactory : public DistanceFunctionFactory { +private: + bool _reference_insertion_vector; public: - PrenormalizedAngularDistanceFunctionFactory() = default; + PrenormalizedAngularDistanceFunctionFactory() noexcept : PrenormalizedAngularDistanceFunctionFactory(false) {} + PrenormalizedAngularDistanceFunctionFactory(bool reference_insertion_vector) noexcept : _reference_insertion_vector(reference_insertion_vector) {} BoundDistanceFunction::UP for_query_vector(TypedCells lhs) const override; BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp index 097ea67cc9e..d57ff575b43 100644 --- a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp @@ -1,13 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "temporary_vector_store.h" -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> using vespalib::ConstArrayRef; using vespalib::ArrayRef; using vespalib::eval::CellType; using vespalib::eval::TypedCells; -using vespalib::hwaccelrated::IAccelrated; +using vespalib::hwaccelerated::IAccelerated; namespace search::tensor { @@ -33,7 +33,7 @@ template<> ConstArrayRef<float> convert_cells<vespalib::BFloat16, float>(ArrayRef<float> space, TypedCells cells) noexcept { - static const IAccelrated & accelerator = IAccelrated::getAccelerator(); + static const IAccelerated & accelerator = IAccelerated::getAccelerator(); accelerator.convert_bfloat16_to_float(reinterpret_cast<const uint16_t *>(cells.data), space.data(), space.size()); return space; } diff --git a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h index 3dc237c85a4..d6702d8278a 100644 --- a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h +++ b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h @@ -6,9 +6,13 @@ namespace search::tensor { -/** helper class - temporary storage of possibly-converted vector cells */ -template <typename FloatType> +/** + * Helper class containing temporary memory storage for possibly converted vector cells. + */ +template <typename FloatTypeT> class TemporaryVectorStore { +public: + using FloatType = FloatTypeT; private: using TypedCells = vespalib::eval::TypedCells; std::vector<FloatType> _tmpSpace; @@ -27,4 +31,24 @@ public: } }; +/** + * Helper class used when TypedCells vector memory is just referenced, + * and used directly in calculations without any transforms. + */ +template <typename FloatTypeT> +class ReferenceVectorStore { +public: + using FloatType = FloatTypeT; +private: + using TypedCells = vespalib::eval::TypedCells; +public: + explicit ReferenceVectorStore(size_t vector_size) noexcept { (void) vector_size; } + vespalib::ConstArrayRef<FloatType> storeLhs(TypedCells cells) noexcept { + return cells.unsafe_typify<FloatType>(); + } + vespalib::ConstArrayRef<FloatType> convertRhs(TypedCells cells) noexcept { + return cells.unsafe_typify<FloatType>(); + } +}; + } diff --git a/searchlib/src/vespa/searchlib/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/CMakeLists.txt index 83e185dbfb6..05188e7b4f9 100644 --- a/searchlib/src/vespa/searchlib/test/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/CMakeLists.txt @@ -14,11 +14,12 @@ vespa_add_library(searchlib_test schema_builder.cpp string_field_builder.cpp vector_buffer_writer.cpp + weightedchildrenverifiers.cpp $<TARGET_OBJECTS:searchlib_test_fakedata> $<TARGET_OBJECTS:searchlib_searchlib_test_diskindex> $<TARGET_OBJECTS:searchlib_test_gtest_migration> DEPENDS - searchlib + vespa_searchlib searchlib_searchlib_test_features searchlib_searchlib_test_memoryindex GTest::GTest diff --git a/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt index e13ec5fca43..9976f8c7d9b 100644 --- a/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(searchlib_searchlib_test_features SOURCES distance_closeness_fixture.cpp DEPENDS - searchlib + vespa_searchlib GTest::GTest ) diff --git a/searchlib/src/vespa/searchlib/test/mock_attribute_context.cpp b/searchlib/src/vespa/searchlib/test/mock_attribute_context.cpp index 75e85d9d828..1003160249d 100644 --- a/searchlib/src/vespa/searchlib/test/mock_attribute_context.cpp +++ b/searchlib/src/vespa/searchlib/test/mock_attribute_context.cpp @@ -28,7 +28,7 @@ MockAttributeContext::getAttributeList(std::vector<const IAttributeVector *> & l MockAttributeContext::~MockAttributeContext() = default; void -MockAttributeContext::add(std::shared_ptr<IAttributeVector> attr) { +MockAttributeContext::add(std::shared_ptr<const IAttributeVector> attr) { _vectors[attr->getName()] = attr; } diff --git a/searchlib/src/vespa/searchlib/test/mock_attribute_context.h b/searchlib/src/vespa/searchlib/test/mock_attribute_context.h index 5b522e907b1..165a37994b9 100644 --- a/searchlib/src/vespa/searchlib/test/mock_attribute_context.h +++ b/searchlib/src/vespa/searchlib/test/mock_attribute_context.h @@ -11,12 +11,12 @@ namespace search::attribute::test { class MockAttributeContext : public IAttributeContext { private: - using Map = std::map<string, std::shared_ptr<IAttributeVector>>; + using Map = std::map<string, std::shared_ptr<const IAttributeVector>>; Map _vectors; public: ~MockAttributeContext() override; - void add(std::shared_ptr<IAttributeVector> attr); + void add(std::shared_ptr<const IAttributeVector> attr); const IAttributeVector *get(const string &name) const; const IAttributeVector * getAttribute(const string &name) const override; diff --git a/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.cpp b/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.cpp new file mode 100644 index 00000000000..b22dd1a3aa9 --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.cpp @@ -0,0 +1,71 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "weightedchildrenverifiers.h" + +using search::queryeval::SearchIterator; + +namespace search::test { + +WeightedChildrenVerifier::WeightedChildrenVerifier() + : _weights(_num_children, 1) +{ } +WeightedChildrenVerifier::~WeightedChildrenVerifier() = default; + + +IteratorChildrenVerifier::IteratorChildrenVerifier() + : WeightedChildrenVerifier(), + _split_lists(_num_children) +{ + auto full_list = getExpectedDocIds(); + for (size_t i = 0; i < full_list.size(); ++i) { + _split_lists[i % _num_children].push_back(full_list[i]); + } +} +IteratorChildrenVerifier::~IteratorChildrenVerifier() = default; + +SearchIterator::UP +IteratorChildrenVerifier::create(bool strict) const { + (void) strict; + std::vector<SearchIterator*> children; + for (size_t i = 0; i < _num_children; ++i) { + children.push_back(createIterator(_split_lists[i], true).release()); + } + return create(children); +} + +SearchIterator::UP +IteratorChildrenVerifier::create(const std::vector<SearchIterator*> &children) const { + (void) children; + return {}; +} + + +DwwIteratorChildrenVerifier::DwwIteratorChildrenVerifier() + : WeightedChildrenVerifier(), + _helper() +{ + _helper.add_docs(getDocIdLimit()); + auto full_list = getExpectedDocIds(); + for (size_t i = 0; i < full_list.size(); ++i) { + _helper.set_doc(full_list[i], i % _num_children, 1); + } +} +DwwIteratorChildrenVerifier::~DwwIteratorChildrenVerifier() = default; + +SearchIterator::UP +DwwIteratorChildrenVerifier::create(bool strict) const { + (void) strict; + std::vector<DocidWithWeightIterator> children; + for (size_t i = 0; i < _num_children; ++i) { + auto dict_entry = _helper.dww().lookup(vespalib::make_string("%zu", i).c_str(), _helper.dww().get_dictionary_snapshot()); + _helper.dww().create(dict_entry.posting_idx, children); + } + return create(std::move(children)); +} +SearchIterator::UP +DwwIteratorChildrenVerifier::create(std::vector<DocidWithWeightIterator> &&) const { + return {}; +} + + +} diff --git a/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.h b/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.h index 86d2fb9aa67..037d1086950 100644 --- a/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.h +++ b/searchlib/src/vespa/searchlib/test/weightedchildrenverifiers.h @@ -8,11 +8,8 @@ namespace search::test { class WeightedChildrenVerifier : public SearchIteratorVerifier { public: - WeightedChildrenVerifier() - : _weights(_num_children, 1) - { } - ~WeightedChildrenVerifier() override {} - + WeightedChildrenVerifier(); + ~WeightedChildrenVerifier() override; protected: static constexpr size_t _num_children = 7; mutable fef::TermFieldMatchData _tfmd; @@ -21,58 +18,21 @@ protected: class IteratorChildrenVerifier : public WeightedChildrenVerifier { public: - IteratorChildrenVerifier() - : WeightedChildrenVerifier(), - _split_lists(_num_children) - { - auto full_list = getExpectedDocIds(); - for (size_t i = 0; i < full_list.size(); ++i) { - _split_lists[i % _num_children].push_back(full_list[i]); - } - } - ~IteratorChildrenVerifier() override { } - SearchIterator::UP create(bool strict) const override { - (void) strict; - std::vector<SearchIterator*> children; - for (size_t i = 0; i < _num_children; ++i) { - children.push_back(createIterator(_split_lists[i], true).release()); - } - return create(children); - } + IteratorChildrenVerifier(); + ~IteratorChildrenVerifier() override; + SearchIterator::UP create(bool strict) const override; protected: - virtual SearchIterator::UP create(const std::vector<SearchIterator*> &children) const { - (void) children; - return SearchIterator::UP(); - } + virtual SearchIterator::UP create(const std::vector<SearchIterator*> &children) const; std::vector<DocIds> _split_lists; }; class DwwIteratorChildrenVerifier : public WeightedChildrenVerifier { public: - DwwIteratorChildrenVerifier() : - WeightedChildrenVerifier(), - _helper() - { - _helper.add_docs(getDocIdLimit()); - auto full_list = getExpectedDocIds(); - for (size_t i = 0; i < full_list.size(); ++i) { - _helper.set_doc(full_list[i], i % _num_children, 1); - } - } - ~DwwIteratorChildrenVerifier() override {} - SearchIterator::UP create(bool strict) const override { - (void) strict; - std::vector<DocidWithWeightIterator> children; - for (size_t i = 0; i < _num_children; ++i) { - auto dict_entry = _helper.dww().lookup(vespalib::make_string("%zu", i).c_str(), _helper.dww().get_dictionary_snapshot()); - _helper.dww().create(dict_entry.posting_idx, children); - } - return create(std::move(children)); - } + DwwIteratorChildrenVerifier(); + ~DwwIteratorChildrenVerifier() override; + SearchIterator::UP create(bool strict) const override; protected: - virtual SearchIterator::UP create(std::vector<DocidWithWeightIterator> &&) const { - return {}; - } + virtual SearchIterator::UP create(std::vector<DocidWithWeightIterator> &&) const; DocumentWeightAttributeHelper _helper; }; diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index a5d65721aeb..07f9c3213bb 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -3,9 +3,9 @@ vespa_define_module( DEPENDS vespalog vespalib - configdefinitions - document - searchlib + vespa_configdefinitions + vespa_document + vespa_searchlib LIBS src/vespa/juniper diff --git a/searchsummary/src/tests/docsummary/CMakeLists.txt b/searchsummary/src/tests/docsummary/CMakeLists.txt index 9f5d767c094..0f32e51a6bb 100644 --- a/searchsummary/src/tests/docsummary/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_positionsdfw_test_app TEST SOURCES positionsdfw_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::GTest ) vespa_add_test(NAME searchsummary_positionsdfw_test_app COMMAND searchsummary_positionsdfw_test_app) diff --git a/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt index cfd65e28bae..d4796823314 100644 --- a/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_annotation_converter_test_app TEST SOURCES annotation_converter_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::GTest ) vespa_add_test(NAME searchsummary_annotation_converter_test_app COMMAND searchsummary_annotation_converter_test_app) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt index 36f9cd5535d..c25587dbd28 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_attribute_combiner_test_app TEST SOURCES attribute_combiner_test.cpp DEPENDS - searchsummary + vespa_searchsummary searchsummary_test GTest::GTest ) diff --git a/searchsummary/src/tests/docsummary/attribute_tokens_dfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_tokens_dfw/CMakeLists.txt index adcb18585d0..b13f44b1d53 100644 --- a/searchsummary/src/tests/docsummary/attribute_tokens_dfw/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/attribute_tokens_dfw/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_attribute_tokens_dfw_test_app TEST SOURCES attribute_tokens_dfw_test.cpp DEPENDS - searchsummary + vespa_searchsummary searchsummary_test GTest::GTest ) diff --git a/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt index c5b220474ba..4949bf40c09 100644 --- a/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_attributedfw_test_app TEST SOURCES attributedfw_test.cpp DEPENDS - searchsummary + vespa_searchsummary searchsummary_test GTest::GTest ) diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt index 6dc3168c78a..513051bf448 100644 --- a/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_document_id_dfw_test_app TEST SOURCES document_id_dfw_test.cpp DEPENDS - searchsummary + vespa_searchsummary searchsummary_test GTest::GTest ) diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt b/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt index b4d6eabe44a..7a09892a813 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_matched_elements_filter_test_app TEST SOURCES matched_elements_filter_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::GTest ) vespa_add_test(NAME searchsummary_matched_elements_filter_test_app COMMAND searchsummary_matched_elements_filter_test_app) diff --git a/searchsummary/src/tests/docsummary/query_term_filter_factory/CMakeLists.txt b/searchsummary/src/tests/docsummary/query_term_filter_factory/CMakeLists.txt index 842722cd05e..e991e991a10 100644 --- a/searchsummary/src/tests/docsummary/query_term_filter_factory/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/query_term_filter_factory/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_query_term_filter_factory_test_app TEST SOURCES query_term_filter_factory_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::GTest ) vespa_add_test(NAME searchsummary_query_term_filter_factory_test_app COMMAND searchsummary_query_term_filter_factory_test_app) diff --git a/searchsummary/src/tests/docsummary/result_class/CMakeLists.txt b/searchsummary/src/tests/docsummary/result_class/CMakeLists.txt index 6882408f669..5becde7530e 100644 --- a/searchsummary/src/tests/docsummary/result_class/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/result_class/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_result_class_test_app TEST SOURCES result_class_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::GTest ) vespa_add_test(NAME searchsummary_result_class_test_app COMMAND searchsummary_result_class_test_app) diff --git a/searchsummary/src/tests/docsummary/slime_filler/CMakeLists.txt b/searchsummary/src/tests/docsummary/slime_filler/CMakeLists.txt index a435430bca2..0322235f700 100644 --- a/searchsummary/src/tests/docsummary/slime_filler/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/slime_filler/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_slime_filler_test_app TEST SOURCES slime_filler_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::GTest ) vespa_add_test(NAME searchsummary_slime_filler_test_app COMMAND searchsummary_slime_filler_test_app) diff --git a/searchsummary/src/tests/docsummary/slime_filler_filter/CMakeLists.txt b/searchsummary/src/tests/docsummary/slime_filler_filter/CMakeLists.txt index 47ab5eafd21..f52adb2f825 100644 --- a/searchsummary/src/tests/docsummary/slime_filler_filter/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/slime_filler_filter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_slime_filler_filter_test_app TEST SOURCES slime_filler_filter_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::GTest ) vespa_add_test(NAME searchsummary_slime_filler_filter_test_app COMMAND searchsummary_slime_filler_filter_test_app) diff --git a/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt b/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt index 943e65a59bc..93da14140f6 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_slime_summary_test_app TEST SOURCES slime_summary_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::GTest ) vespa_add_test(NAME searchsummary_slime_summary_test_app COMMAND searchsummary_slime_summary_test_app) diff --git a/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt index 68885a74b1b..691ea889c46 100644 --- a/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchsummary_tokens_converter_test_app TEST SOURCES tokens_converter_test.cpp DEPENDS - searchsummary + vespa_searchsummary GTest::gtest ) diff --git a/searchsummary/src/tests/juniper/CMakeLists.txt b/searchsummary/src/tests/juniper/CMakeLists.txt index e25eb01e85c..c0beff95fd2 100644 --- a/searchsummary/src/tests/juniper/CMakeLists.txt +++ b/searchsummary/src/tests/juniper/CMakeLists.txt @@ -11,7 +11,7 @@ vespa_add_executable(juniper_mcandTest_app TEST mcandTestApp.cpp testenv.cpp DEPENDS - searchsummary + vespa_searchsummary juniper_testsuite ) vespa_add_test(NAME juniper_mcandTest_app COMMAND juniper_mcandTest_app) @@ -22,7 +22,7 @@ vespa_add_executable(juniper_queryparserTest_app TEST fakerewriter.cpp testenv.cpp DEPENDS - searchsummary + vespa_searchsummary juniper_testsuite ) vespa_add_test(NAME juniper_queryparserTest_app COMMAND juniper_queryparserTest_app) @@ -33,7 +33,7 @@ vespa_add_executable(juniper_matchobjectTest_app TEST testenv.cpp fakerewriter.cpp DEPENDS - searchsummary + vespa_searchsummary juniper_testsuite ) vespa_add_test(NAME juniper_matchobjectTest_app COMMAND juniper_matchobjectTest_app) @@ -41,14 +41,14 @@ vespa_add_executable(juniper_appender_test_app TEST SOURCES appender_test.cpp DEPENDS - searchsummary + vespa_searchsummary ) vespa_add_test(NAME juniper_appender_test_app COMMAND juniper_appender_test_app) vespa_add_executable(juniper_queryvisitor_test_app TEST SOURCES queryvisitor_test.cpp DEPENDS - searchsummary + vespa_searchsummary ) vespa_add_test(NAME juniper_queryvisitor_test_app COMMAND juniper_queryvisitor_test_app) vespa_add_executable(juniper_auxTest_app TEST @@ -57,7 +57,7 @@ vespa_add_executable(juniper_auxTest_app TEST auxTestApp.cpp testenv.cpp DEPENDS - searchsummary + vespa_searchsummary juniper_testsuite ) vespa_add_test(NAME juniper_auxTest_app COMMAND juniper_auxTest_app) @@ -71,7 +71,7 @@ vespa_add_executable(juniper_SrcTestSuite_app TEST auxTest.cpp testenv.cpp DEPENDS - searchsummary + vespa_searchsummary juniper_testsuite ) vespa_add_test(NAME juniper_SrcTestSuite_app COMMAND juniper_SrcTestSuite_app) diff --git a/searchsummary/src/tests/juniper/SrcTestSuite.cpp b/searchsummary/src/tests/juniper/SrcTestSuite.cpp index 955ed01fd85..0a45d6ffc95 100644 --- a/searchsummary/src/tests/juniper/SrcTestSuite.cpp +++ b/searchsummary/src/tests/juniper/SrcTestSuite.cpp @@ -6,7 +6,7 @@ #include "queryparserTest.h" #include "matchobjectTest.h" #include "auxTest.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> /** * The SrcTestSuite class runs all the unit tests for the src module. * diff --git a/searchsummary/src/tests/juniper/appender_test.cpp b/searchsummary/src/tests/juniper/appender_test.cpp index 97d07998e3f..17cd4a77472 100644 --- a/searchsummary/src/tests/juniper/appender_test.cpp +++ b/searchsummary/src/tests/juniper/appender_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #define _NEED_SUMMARY_CONFIG_IMPL #include <vespa/juniper/SummaryConfig.h> diff --git a/searchsummary/src/tests/juniper/auxTestApp.cpp b/searchsummary/src/tests/juniper/auxTestApp.cpp index 62b2d3f934c..1344188e249 100644 --- a/searchsummary/src/tests/juniper/auxTestApp.cpp +++ b/searchsummary/src/tests/juniper/auxTestApp.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "auxTest.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> void Usage(char* s) { diff --git a/searchsummary/src/tests/juniper/matchobjectTestApp.cpp b/searchsummary/src/tests/juniper/matchobjectTestApp.cpp index bd199f9d1d6..abcc496e717 100644 --- a/searchsummary/src/tests/juniper/matchobjectTestApp.cpp +++ b/searchsummary/src/tests/juniper/matchobjectTestApp.cpp @@ -2,7 +2,7 @@ #include "matchobjectTest.h" #include "testenv.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/juniper/wildcard_match.h> #include <iostream> diff --git a/searchsummary/src/tests/juniper/mcandTestApp.cpp b/searchsummary/src/tests/juniper/mcandTestApp.cpp index 38a3cdee367..e92bf8b89d3 100644 --- a/searchsummary/src/tests/juniper/mcandTestApp.cpp +++ b/searchsummary/src/tests/juniper/mcandTestApp.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mcandTest.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> int main(int argc, char **argv) { juniper::TestEnv te(argc, argv, TEST_PATH("./testclient.rc").c_str()); diff --git a/searchsummary/src/tests/juniper/queryparserTestApp.cpp b/searchsummary/src/tests/juniper/queryparserTestApp.cpp index 95fd198cffe..c818afc5533 100644 --- a/searchsummary/src/tests/juniper/queryparserTestApp.cpp +++ b/searchsummary/src/tests/juniper/queryparserTestApp.cpp @@ -2,7 +2,7 @@ #include "queryparserTest.h" #include "testenv.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> int main(int argc, char **argv) { diff --git a/searchsummary/src/tests/juniper/queryvisitor_test.cpp b/searchsummary/src/tests/juniper/queryvisitor_test.cpp index a5862c0aec3..f4501ef1909 100644 --- a/searchsummary/src/tests/juniper/queryvisitor_test.cpp +++ b/searchsummary/src/tests/juniper/queryvisitor_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <memory> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/juniper/queryhandle.h> #include <vespa/juniper/queryvisitor.h> diff --git a/searchsummary/src/vespa/searchsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/CMakeLists.txt index 56a62b14767..b82060a51be 100644 --- a/searchsummary/src/vespa/searchsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchsummary +vespa_add_library(vespa_searchsummary SOURCES $<TARGET_OBJECTS:searchsummary_config> $<TARGET_OBJECTS:searchsummary_docsummary> diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 102dec0d511..341cbd310bd 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -152,7 +152,6 @@ public class SlobrokMonitorManagerImpl extends AbstractComponent implements Slob case "logserver": case "metricsproxy": case "slobrok": - case "transactionlogserver": return Optional.empty(); case "qrserver": diff --git a/slobrok/CMakeLists.txt b/slobrok/CMakeLists.txt index 84f63e86292..c72c9b27adc 100644 --- a/slobrok/CMakeLists.txt +++ b/slobrok/CMakeLists.txt @@ -2,8 +2,8 @@ vespa_define_module( DEPENDS vespalib - fnet - configdefinitions + vespa_fnet + vespa_configdefinitions LIBS src/vespa/slobrok diff --git a/slobrok/src/apps/slobrok/CMakeLists.txt b/slobrok/src/apps/slobrok/CMakeLists.txt index 9492ff54b30..33b90ddc466 100644 --- a/slobrok/src/apps/slobrok/CMakeLists.txt +++ b/slobrok/src/apps/slobrok/CMakeLists.txt @@ -5,5 +5,5 @@ vespa_add_executable(slobrok_app OUTPUT_NAME vespa-slobrok INSTALL sbin DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver ) diff --git a/slobrok/src/tests/backoff/CMakeLists.txt b/slobrok/src/tests/backoff/CMakeLists.txt index ac0bf9cac78..6015bb28c4a 100644 --- a/slobrok/src/tests/backoff/CMakeLists.txt +++ b/slobrok/src/tests/backoff/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(slobrok_backoff_test_app TEST SOURCES testbackoff.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver ) vespa_add_test(NAME slobrok_backoff_test_app COMMAND slobrok_backoff_test_app) diff --git a/slobrok/src/tests/backoff/testbackoff.cpp b/slobrok/src/tests/backoff/testbackoff.cpp index 23193cc2d98..9f9072c513e 100644 --- a/slobrok/src/tests/backoff/testbackoff.cpp +++ b/slobrok/src/tests/backoff/testbackoff.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/slobrok/backoff.h> #include <vespa/log/log.h> @@ -7,14 +7,9 @@ LOG_SETUP("backoff_test"); using slobrok::api::BackOff; -TEST_SETUP(Test); - //----------------------------------------------------------------------------- -int -Test::Main() -{ - TEST_INIT("backoff_test"); +TEST("backoff_test") { BackOff one; EXPECT_FALSE(one.shouldWarn()); @@ -64,5 +59,6 @@ Test::Main() EXPECT_FALSE(two.shouldWarn()); } } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/slobrok/src/tests/configure/CMakeLists.txt b/slobrok/src/tests/configure/CMakeLists.txt index 169971095f5..bc08abc0021 100644 --- a/slobrok/src/tests/configure/CMakeLists.txt +++ b/slobrok/src/tests/configure/CMakeLists.txt @@ -8,7 +8,7 @@ vespa_add_executable(slobrok_configure_test_app TEST SOURCES configure.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver ) vespa_add_test( NAME slobrok_configure_test_app diff --git a/slobrok/src/tests/configure/configure.cpp b/slobrok/src/tests/configure/configure.cpp index d16c241b905..e6f17d23f01 100644 --- a/slobrok/src/tests/configure/configure.cpp +++ b/slobrok/src/tests/configure/configure.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/slobrok/sbregister.h> #include <vespa/slobrok/server/slobrokserver.h> diff --git a/slobrok/src/tests/local_rpc_monitor_map/CMakeLists.txt b/slobrok/src/tests/local_rpc_monitor_map/CMakeLists.txt index fe194f9db9f..dc91bf211ab 100644 --- a/slobrok/src/tests/local_rpc_monitor_map/CMakeLists.txt +++ b/slobrok/src/tests/local_rpc_monitor_map/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(slobrok_local_rpc_monitor_map_test_app TEST SOURCES local_rpc_monitor_map_test.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver GTest::GTest ) vespa_add_test(NAME slobrok_local_rpc_monitor_map_test_app COMMAND slobrok_local_rpc_monitor_map_test_app) diff --git a/slobrok/src/tests/mirrorapi/CMakeLists.txt b/slobrok/src/tests/mirrorapi/CMakeLists.txt index 6180544886a..fe35e47e238 100644 --- a/slobrok/src/tests/mirrorapi/CMakeLists.txt +++ b/slobrok/src/tests/mirrorapi/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(slobrok_mirrorapi_test_app TEST SOURCES mirrorapi.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver ) vespa_add_test(NAME slobrok_mirrorapi_test_app COMMAND slobrok_mirrorapi_test_app) vespa_add_executable(slobrok_mirror_match_test_app TEST SOURCES match_test.cpp DEPENDS - slobrok + vespa_slobrok ) vespa_add_test(NAME slobrok_mirror_match_test_app COMMAND slobrok_mirror_match_test_app) diff --git a/slobrok/src/tests/mirrorapi/mirrorapi.cpp b/slobrok/src/tests/mirrorapi/mirrorapi.cpp index 5e340a86a33..664ce9dc5b5 100644 --- a/slobrok/src/tests/mirrorapi/mirrorapi.cpp +++ b/slobrok/src/tests/mirrorapi/mirrorapi.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/config-slobroks.h> #include <algorithm> @@ -16,8 +16,6 @@ LOG_SETUP("mirrorapi_test"); using slobrok::api::MirrorAPI; using slobrok::SlobrokServer; -TEST_SETUP(Test); - //----------------------------------------------------------------------------- class Server : public FRT_Invokable @@ -119,10 +117,7 @@ compare(MirrorAPI &api, const char *pattern, SpecList expect) } -int -Test::Main() -{ - TEST_INIT("mirrorapi_test"); +TEST("mirrorapi_test") { SlobrokServer mock(18501); std::this_thread::sleep_for(300ms); @@ -224,5 +219,6 @@ Test::Main() mock.stop(); transport.ShutDown(true); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/slobrok/src/tests/registerapi/CMakeLists.txt b/slobrok/src/tests/registerapi/CMakeLists.txt index 3b26447b05c..adc28f4d499 100644 --- a/slobrok/src/tests/registerapi/CMakeLists.txt +++ b/slobrok/src/tests/registerapi/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(slobrok_registerapi_test_app TEST SOURCES registerapi.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver ) vespa_add_test(NAME slobrok_registerapi_test_app COMMAND slobrok_registerapi_test_app) diff --git a/slobrok/src/tests/registerapi/registerapi.cpp b/slobrok/src/tests/registerapi/registerapi.cpp index 0499f054314..bad7ef819a7 100644 --- a/slobrok/src/tests/registerapi/registerapi.cpp +++ b/slobrok/src/tests/registerapi/registerapi.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/slobrok/sbregister.h> @@ -19,9 +19,6 @@ using slobrok::api::RegisterAPI; using slobrok::SlobrokServer; -TEST_SETUP(Test); - - std::string createSpec(FRT_Supervisor &orb) { @@ -71,10 +68,7 @@ compare(MirrorAPI &api, const char *pattern, SpecList expect) return false; } -int -Test::Main() -{ - TEST_INIT("registerapi_test"); +TEST("registerapi_test") { SlobrokServer mock(18548); std::this_thread::sleep_for(300ms); @@ -219,5 +213,6 @@ Test::Main() mock.stop(); server.shutdown(); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/slobrok/src/tests/rpc_mapping_monitor/CMakeLists.txt b/slobrok/src/tests/rpc_mapping_monitor/CMakeLists.txt index 787d629c38b..f4dfa019169 100644 --- a/slobrok/src/tests/rpc_mapping_monitor/CMakeLists.txt +++ b/slobrok/src/tests/rpc_mapping_monitor/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(slobrok_rpc_mapping_monitor_test_app TEST SOURCES rpc_mapping_monitor_test.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver GTest::GTest ) vespa_add_test(NAME slobrok_rpc_mapping_monitor_test_app COMMAND slobrok_rpc_mapping_monitor_test_app) diff --git a/slobrok/src/tests/service_map_history/CMakeLists.txt b/slobrok/src/tests/service_map_history/CMakeLists.txt index e8ea2bcb6b4..227ce238b1c 100644 --- a/slobrok/src/tests/service_map_history/CMakeLists.txt +++ b/slobrok/src/tests/service_map_history/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(slobrok_service_map_history_test_app TEST SOURCES service_map_history_test.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver GTest::GTest ) vespa_add_test(NAME slobrok_service_map_history_test_app COMMAND slobrok_service_map_history_test_app) diff --git a/slobrok/src/tests/service_map_mirror/CMakeLists.txt b/slobrok/src/tests/service_map_mirror/CMakeLists.txt index 595ed112f77..2c26e6f3270 100644 --- a/slobrok/src/tests/service_map_mirror/CMakeLists.txt +++ b/slobrok/src/tests/service_map_mirror/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(slobrok_service_map_mirror_test_app TEST SOURCES service_map_mirror_test.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver GTest::GTest ) vespa_add_test(NAME slobrok_service_map_mirror_test_app COMMAND slobrok_service_map_mirror_test_app) diff --git a/slobrok/src/tests/standalone/CMakeLists.txt b/slobrok/src/tests/standalone/CMakeLists.txt index fbb2d235810..3ae4be7c9f1 100644 --- a/slobrok/src/tests/standalone/CMakeLists.txt +++ b/slobrok/src/tests/standalone/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(slobrok_standalone_test_app TEST SOURCES standalone.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver ) vespa_add_test(NAME slobrok_standalone_test_app COMMAND slobrok_standalone_test_app) diff --git a/slobrok/src/tests/union_service_map/CMakeLists.txt b/slobrok/src/tests/union_service_map/CMakeLists.txt index 6993869c59f..603f817e075 100644 --- a/slobrok/src/tests/union_service_map/CMakeLists.txt +++ b/slobrok/src/tests/union_service_map/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(slobrok_union_service_map_test_app TEST SOURCES union_service_map_test.cpp DEPENDS - slobrok_slobrokserver + vespa_slobrok_slobrokserver GTest::GTest ) vespa_add_test(NAME slobrok_union_service_map_test_app COMMAND slobrok_union_service_map_test_app) diff --git a/slobrok/src/vespa/slobrok/CMakeLists.txt b/slobrok/src/vespa/slobrok/CMakeLists.txt index bb3954ce6ff..593675c293b 100644 --- a/slobrok/src/vespa/slobrok/CMakeLists.txt +++ b/slobrok/src/vespa/slobrok/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(slobrok +vespa_add_library(vespa_slobrok SOURCES backoff.cpp sblist.cpp diff --git a/slobrok/src/vespa/slobrok/server/CMakeLists.txt b/slobrok/src/vespa/slobrok/server/CMakeLists.txt index 564121aa200..9aad0cfec41 100644 --- a/slobrok/src/vespa/slobrok/server/CMakeLists.txt +++ b/slobrok/src/vespa/slobrok/server/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(slobrok_slobrokserver +vespa_add_library(vespa_slobrok_slobrokserver SOURCES configshim.cpp exchange_manager.cpp @@ -32,5 +32,5 @@ vespa_add_library(slobrok_slobrokserver union_service_map.cpp INSTALL lib64 DEPENDS - slobrok + vespa_slobrok ) diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index c59d095fb4c..87a031b9e33 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -2,16 +2,16 @@ vespa_define_module( DEPENDS vespadefaults - metrics - config_cloudconfig - configdefinitions + vespa_metrics + vespa_config + vespa_configdefinitions vespalog - messagebus - documentapi - document + vespa_messagebus + vespa_documentapi + vespa_document vespalib - vdslib - persistence + vespa_vdslib + vespa_persistence EXTERNAL_DEPENDS ${VESPA_GLIBC_RT_LIB} @@ -53,7 +53,7 @@ vespa_define_module( src/vespa/storageapi/messageapi TEST_DEPENDS - messagebus_messagebus-test + vespa_messagebus-test TEST_EXTERNAL_DEPENDS ${VESPA_ATOMIC_LIB} diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt index 4ebaa67d74e..a6337ff142e 100644 --- a/storage/src/tests/bucketdb/CMakeLists.txt +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(storage_bucketdb_gtest_runner_app TEST gtest_runner.cpp lockablemaptest.cpp DEPENDS - storage + vespa_storage storage_testcommon GTest::GTest ) diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index 437e8a14e3b..f41dae89eec 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -14,16 +14,18 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/metrics/updatehook.h> #include <vespa/storage/bucketdb/bucketmanager.h> -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/distribution/distribution.h> +#include <vespa/vdslib/distribution/global_bucket_space_distribution_converter.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/testkit/test_path.h> +#include <vespa/config-stor-distribution.h> #include <future> #include <vespa/log/log.h> @@ -129,7 +131,7 @@ void BucketManagerTest::setupTestEnvironment() { _config = StorageConfigSet::make_storage_node_config(); auto repo = std::make_shared<const DocumentTypeRepo>( - *ConfigGetter<DocumenttypesConfig>::getConfig("config-doctypes", FileSpec("../config-doctypes.cfg"))); + *ConfigGetter<DocumenttypesConfig>::getConfig("config-doctypes", FileSpec(TEST_PATH("../config-doctypes.cfg")))); _top = std::make_unique<DummyStorageLink>(); _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->config_uri()); _node->setTypeRepo(repo); @@ -667,7 +669,7 @@ public: static std::unique_ptr<lib::Distribution> default_grouped_distribution() { return std::make_unique<lib::Distribution>( - lib::Distribution::ConfigWrapper(GlobalBucketSpaceDistributionConverter::string_to_config(vespalib::string( + lib::Distribution::ConfigWrapper(lib::GlobalBucketSpaceDistributionConverter::string_to_config(vespalib::string( R"(redundancy 2 group[3] group[0].name "invalid" @@ -691,7 +693,7 @@ group[2].nodes[2].index 5 static std::shared_ptr<lib::Distribution> derived_global_grouped_distribution() { auto default_distr = default_grouped_distribution(); - return GlobalBucketSpaceDistributionConverter::convert_to_global(*default_distr); + return lib::GlobalBucketSpaceDistributionConverter::convert_to_global(*default_distr); } private: diff --git a/storage/src/tests/common/CMakeLists.txt b/storage/src/tests/common/CMakeLists.txt index 0e075e39194..dcbccd246fa 100644 --- a/storage/src/tests/common/CMakeLists.txt +++ b/storage/src/tests/common/CMakeLists.txt @@ -7,20 +7,19 @@ vespa_add_library(storage_testcommon TEST testnodestateupdater.cpp teststorageapp.cpp DEPENDS - storage + vespa_storage ) vespa_add_executable(storage_common_gtest_runner_app TEST SOURCES bucket_stripe_utils_test.cpp bucket_utils_test.cpp - global_bucket_space_distribution_converter_test.cpp gtest_runner.cpp metricstest.cpp storagelinktest.cpp DEPENDS storage_testcommon - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/common/hostreporter/CMakeLists.txt b/storage/src/tests/common/hostreporter/CMakeLists.txt index 99fdbb87a8d..a168d276eb8 100644 --- a/storage/src/tests/common/hostreporter/CMakeLists.txt +++ b/storage/src/tests/common/hostreporter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_library(storage_testhostreporter TEST SOURCES util.cpp DEPENDS - storage + vespa_storage ) vespa_add_executable(storage_hostreporter_gtest_runner_app TEST @@ -12,7 +12,7 @@ vespa_add_executable(storage_hostreporter_gtest_runner_app TEST hostinfotest.cpp versionreportertest.cpp DEPENDS - storage + vespa_storage storage_testhostreporter GTest::GTest ) diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index 250cb872223..33c48adf552 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -57,12 +57,18 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST DEPENDS storage_testcommon storage_testhostreporter - storage + vespa_storage GTest::gmock_main ) -vespa_add_test( - NAME storage_distributor_gtest_runner_app - COMMAND storage_distributor_gtest_runner_app - COST 350 -) +set(TOTAL_SHARDS 5) +math(EXPR MAX_SHARD_INDEX "${TOTAL_SHARDS} - 1") +foreach(SHARD_INDEX RANGE ${MAX_SHARD_INDEX}) + string(REGEX MATCH "...$" FMT_SHARD_INDEX "00" ${SHARD_INDEX}) + vespa_add_test( + NAME storage_distributor_gtest_runner_app_${FMT_SHARD_INDEX} + COMMAND storage_distributor_gtest_runner_app + ENVIRONMENT "GTEST_SHARD_INDEX=${SHARD_INDEX};GTEST_TOTAL_SHARDS=${TOTAL_SHARDS}" + COST 350 + ) +endforeach() diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index e9c1cea38f3..c56911a066e 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <chrono> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> namespace storage::distributor { @@ -182,7 +183,7 @@ TEST_F(DistributorHostInfoReporterTest, generate_example_json) { std::string jsonString = json.str(); - std::string path = "../../../../protocols/getnodestate/distributor.json"; + std::string path = TEST_PATH("../../../../protocols/getnodestate/distributor.json"); std::string goldenString = File::readAll(path); vespalib::Memory goldenMemory(goldenString); diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp index f0b3db3adf8..0628b62bdfe 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp @@ -77,7 +77,7 @@ DistributorStripeTestUtil::setup_stripe(int redundancy, int node_count, const li // trigger_distribution_change(). // This isn't pretty, folks, but it avoids breaking the world for now, // as many tests have implicit assumptions about this being the behavior. - auto new_configs = BucketSpaceDistributionConfigs::from_default_distribution(std::move(distribution)); + auto new_configs = lib::BucketSpaceDistributionConfigs::from_default_distribution(std::move(distribution)); _stripe->update_distribution_config(new_configs); } @@ -95,7 +95,7 @@ void DistributorStripeTestUtil::trigger_distribution_change(lib::Distribution::SP distr) { _node->getComponentRegister().setDistribution(distr); - auto new_config = BucketSpaceDistributionConfigs::from_default_distribution(std::move(distr)); + auto new_config = lib::BucketSpaceDistributionConfigs::from_default_distribution(std::move(distr)); _stripe->update_distribution_config(new_config); } diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp index 4450d21c651..cad8252772e 100644 --- a/storage/src/tests/distributor/getoperationtest.cpp +++ b/storage/src/tests/distributor/getoperationtest.cpp @@ -17,7 +17,8 @@ #include <vespa/storage/distributor/operations/external/getoperation.h> #include <vespa/storageapi/message/persistence.h> #include <iomanip> -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> #include <gmock/gmock.h> using config::ConfigGetter; @@ -43,7 +44,7 @@ struct GetOperationTest : Test, DistributorStripeTestUtil { _repo.reset( new document::DocumentTypeRepo(*ConfigGetter<DocumenttypesConfig>:: getConfig("config-doctypes", - FileSpec("../config-doctypes.cfg")))); + FileSpec(TEST_PATH("../config-doctypes.cfg"))))); createLinks(); docId = document::DocumentId("id:ns:text/html::uri"); diff --git a/storage/src/tests/distributor/mock_tickable_stripe.h b/storage/src/tests/distributor/mock_tickable_stripe.h index 2fb486bab28..3a9ead6ae10 100644 --- a/storage/src/tests/distributor/mock_tickable_stripe.h +++ b/storage/src/tests/distributor/mock_tickable_stripe.h @@ -10,7 +10,7 @@ struct MockTickableStripe : TickableStripe { bool tick() override { abort(); } void flush_and_close() override { abort(); } void update_total_distributor_config(std::shared_ptr<const DistributorConfiguration>) override { abort(); } - void update_distribution_config(const BucketSpaceDistributionConfigs&) override { abort(); } + void update_distribution_config(const lib::BucketSpaceDistributionConfigs&) override { abort(); } void set_pending_cluster_state_bundle(const lib::ClusterStateBundle&) override { abort(); } void clear_pending_cluster_state_bundle() override { abort(); } void enable_cluster_state_bundle(const lib::ClusterStateBundle&, bool) override { abort(); } diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index f0f8a4359fd..171dc5a42c0 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -27,8 +27,7 @@ struct OperationTargetResolverTest : Test, DistributorStripeTestUtil { const document::DocumentType* _html_type; std::unique_ptr<Operation> op; - BucketInstanceList getInstances(const BucketId& bid, - bool stripToRedundancy); + BucketInstanceList getInstances(const BucketId& bid, bool stripToRedundancy, bool symmetry_mode); void SetUp() override { _repo.reset(new document::DocumentTypeRepo( @@ -62,7 +61,7 @@ namespace { TestTargets::createTest(id, *this, *_asserters.back()) struct Asserter { - virtual ~Asserter() {} + virtual ~Asserter() = default; virtual void assertEqualMsg(std::string t1, OperationTargetList t2, OperationTargetList t3) = 0; @@ -73,21 +72,29 @@ struct TestTargets { OperationTargetList _expected; OperationTargetResolverTest& _test; Asserter& _asserter; + bool _symmetry_mode; TestTargets(const BucketId& id, OperationTargetResolverTest& test, Asserter& asserter) - : _id(id), _test(test), _asserter(asserter) {} + : _id(id), _test(test), _asserter(asserter), _symmetry_mode(true) + { + } ~TestTargets() { - BucketInstanceList result(_test.getInstances(_id, true)); - BucketInstanceList all(_test.getInstances(_id, false)); + BucketInstanceList result(_test.getInstances(_id, true, _symmetry_mode)); + BucketInstanceList all(_test.getInstances(_id, false, _symmetry_mode)); _asserter.assertEqualMsg( all.toString(), _expected, result.createTargets(makeBucketSpace())); delete _asserters.back(); _asserters.pop_back(); } + TestTargets& with_symmetric_replica_selection(bool symmetry) noexcept { + _symmetry_mode = symmetry; + return *this; + } + TestTargets& sendsTo(const BucketId& id, uint16_t node) { _expected.push_back(OperationTarget( makeDocumentBucket(id), lib::Node(lib::NodeType::STORAGE, node), false)); @@ -110,7 +117,7 @@ struct TestTargets { } // anonymous BucketInstanceList -OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedundancy) +OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedundancy, bool symmetry_mode) { auto &bucketSpaceRepo(operation_context().bucket_space_repo()); auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); @@ -118,6 +125,7 @@ OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedund distributorBucketSpace, distributorBucketSpace.getBucketDatabase(), 16, distributorBucketSpace.getDistribution().getRedundancy(), makeBucketSpace()); + resolver.use_symmetric_replica_selection(symmetry_mode); if (stripToRedundancy) { return resolver.getInstances(OperationTargetResolver::PUT, id); } else { @@ -143,14 +151,48 @@ TEST_F(OperationTargetResolverTest, choose_ideal_state_when_many_copies) { .sendsTo(BucketId(16, 0), 3); } -TEST_F(OperationTargetResolverTest, trusted_over_ideal_state) { +TEST_F(OperationTargetResolverTest, legacy_prefers_trusted_over_ideal_state) { setup_stripe(2, 4, "storage:4 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "0=0/0/0/t,1=0,2=0/0/0/t,3=0"); // ideal nodes: 1, 3 + MY_ASSERT_THAT(BucketId(32, 0)).with_symmetric_replica_selection(false) + .sendsTo(BucketId(16, 0), 0) + .sendsTo(BucketId(16, 0), 2); +} + +TEST_F(OperationTargetResolverTest, prefer_ready_over_ideal_state_order) { + setup_stripe(2, 4, "storage:4 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=1/2/3/u/i/r,1=1/2/3,2=1/2/3/u/i/r,3=1/2/3"); + // ideal nodes: 1, 3. 0 and 2 are ready. MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 0) .sendsTo(BucketId(16, 0), 2); } +TEST_F(OperationTargetResolverTest, prefer_ready_over_ideal_state_order_also_when_retired) { + setup_stripe(2, 4, "storage:4 .0.s:r distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=1/2/3/u/i/r,1=1/2/3,2=1/2/3/u/i/r,3=1/2/3"); + // ideal nodes: 1, 3. 0 and 2 are ready. + MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 0) + .sendsTo(BucketId(16, 0), 2); +} + +TEST_F(OperationTargetResolverTest, prefer_replicas_with_more_docs_over_replicas_with_fewer_docs) { + setup_stripe(2, 4, "storage:4 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=2/3/4,1=1/2/3,2=3/4/5,3=1/2/3"); + // ideal nodes: 1, 3. 0 and 2 have more docs. + MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 2) + .sendsTo(BucketId(16, 0), 0); +} + +TEST_F(OperationTargetResolverTest, fall_back_to_active_state_and_db_index_if_all_other_fields_equal) { + // All replica nodes tagged as retired, which means none are part of the ideal state order + setup_stripe(2, 4, "storage:4 .0.s:r .2.s:r .3.s:r distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=2/3/4/u/a,3=2/3/4,2=2/3/4"); + // ideal nodes: 1, 3. 0 is active and 3 is the remaining replica with the lowest DB order. + MY_ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 0) + .sendsTo(BucketId(16, 0), 3); +} + TEST_F(OperationTargetResolverTest, choose_highest_split_bucket) { setup_stripe(2, 2, "storage:2 distributor:1"); // 0, 1 are both in ideal state for both buckets. diff --git a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp index 51c0a75e45d..4fe8d88fb8d 100644 --- a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp +++ b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp @@ -45,7 +45,7 @@ public: std::vector<document::BucketSpace> _bucket_spaces; - size_t message_count(size_t messagesPerBucketSpace) const { + size_t message_count(size_t messagesPerBucketSpace) const noexcept { return messagesPerBucketSpace * _bucket_spaces.size(); } @@ -361,7 +361,7 @@ public: { auto cluster_info = owner.create_cluster_info(old_cluster_state); state = PendingClusterState::createForDistributionChange( - clock, cluster_info, sender, owner.bucket_space_states(), api::Timestamp(1)); + clock, cluster_info, sender, owner.bucket_space_states(), api::Timestamp(1), false); } }; @@ -475,6 +475,15 @@ public: return total; } + void expect_and_answer_bucket_info_requests(uint32_t n_msgs, const lib::ClusterState& expected_state) { + ASSERT_EQ(_sender.commands().size(), message_count(n_msgs)); + constexpr uint32_t n_buckets = 10; + ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(expected_state, message_count(n_msgs), n_buckets)); + _sender.clear(); + } + + void verify_state_bundle_propagated_to_stripes(const lib::ClusterStateBundle& expected_bundle); + }; TopLevelBucketDBUpdaterTest::TopLevelBucketDBUpdaterTest() @@ -616,8 +625,28 @@ TopLevelBucketDBUpdaterTest::trigger_completed_but_not_yet_activated_transition( _sender.clear(); } +void +TopLevelBucketDBUpdaterTest::verify_state_bundle_propagated_to_stripes(const lib::ClusterStateBundle& expected_bundle) +{ + ASSERT_TRUE(expected_bundle.has_distribution_config()); + const auto space = document::FixedBucketSpaces::default_space(); + for (auto* s : distributor_stripes()) { + // We only sample the default space, but assume this generalizes to other spaces. + auto& repo = s->getBucketSpaceRepo().get(space); + // TODO refactor so that only state bundle (with config) is used internally rather than + // separate (but intertwined) explicit pairs of state + config. This is not a small task... + EXPECT_FALSE(repo.has_pending_cluster_state()); // should have converged + EXPECT_EQ(repo.getClusterState(), *expected_bundle.getDerivedClusterState(space)); + EXPECT_EQ(repo.getDistribution(), expected_bundle.distribution_config_bundle()->default_distribution()); + } +} + +// TODO all tests that change distribution config should be rewritten to do this via +// a cluster state bundle, but this cannot happen until the legacy behavior is no +// longer supported (>= Vespa 9). + TEST_F(TopLevelBucketDBUpdaterTest, normal_usage) { - set_cluster_state(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); // FIXME init mode why? + set_cluster_state(lib::ClusterState("distributor:2 storage:3")); ASSERT_EQ(message_count(3), _sender.commands().size()); @@ -625,21 +654,21 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage) { ASSERT_EQ(_component->getDistribution()->getNodeGraph().getDistributionConfigHash(), dynamic_cast<const RequestBucketInfoCommand&>(*_sender.command(0)).getDistributionHash()); - ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why? + ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:2 storage:3"), *_sender.command(0), 10)); _sender.clear(); // Optimization for not refetching unneeded data after cluster state // change is only implemented after completion of previous cluster state - set_cluster_state("distributor:2 .0.s:i storage:3"); // FIXME init mode why? + set_cluster_state("distributor:2 storage:3"); ASSERT_EQ(message_count(3), _sender.commands().size()); // Expect reply of first set SystemState request. ASSERT_EQ(size_t(1), _sender.replies().size()); ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why? + lib::ClusterState("distributor:2 storage:3"), message_count(3), 10)); ASSERT_NO_FATAL_FAILURE(assert_correct_buckets(10, "distributor:2 storage:3")); } @@ -648,9 +677,9 @@ TEST_F(TopLevelBucketDBUpdaterTest, distributor_change) { int num_buckets = 100; // First sends request - set_cluster_state("distributor:2 .0.s:i .1.s:i storage:3"); // FIXME init mode why? + set_cluster_state("distributor:2 storage:3"); ASSERT_EQ(message_count(3), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why? + ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:2 storage:3"), message_count(3), num_buckets)); _sender.clear(); @@ -705,14 +734,14 @@ TEST_F(TopLevelBucketDBUpdaterTest, distributor_change_with_grouping) { } TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) { - set_cluster_state("distributor:1 .0.s:i storage:1 .0.s:i"); // FIXME init mode why? + set_cluster_state("distributor:1 storage:1 .0.s:i"); ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size()); // Not yet passing on system state. ASSERT_EQ(size_t(0), _sender_down.commands().size()); - ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), // FIXME init mode why? + ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 storage:1"), _bucket_spaces.size(), 10, 10)); ASSERT_NO_FATAL_FAILURE(assert_correct_buckets(10, "distributor:1 storage:1")); @@ -727,12 +756,12 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) { _sender.clear(); _sender_down.clear(); - set_cluster_state("distributor:1 .0.s:i storage:1"); // FIXME init mode why? + set_cluster_state("distributor:1 storage:1"); // Send a new request bucket info up. ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), // FIXME init mode why? + ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 storage:1"), _bucket_spaces.size(), 20)); // Pass on cluster state and recheck buckets now. @@ -742,7 +771,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) { } TEST_F(TopLevelBucketDBUpdaterTest, failed_request_bucket_info) { - set_cluster_state("distributor:1 .0.s:i storage:1"); // FIXME init mode why? + set_cluster_state("distributor:1 storage:1"); // 2 messages sent up: 1 to the nodes, and one reply to the setsystemstate. ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size()); @@ -1427,7 +1456,7 @@ TopLevelBucketDBUpdaterTest::get_sent_nodes_distribution_changed(const std::stri auto cluster_info = create_cluster_info(old_cluster_state); std::unique_ptr<PendingClusterState> state( PendingClusterState::createForDistributionChange( - clock, cluster_info, sender, bucket_space_states(), api::Timestamp(1))); + clock, cluster_info, sender, bucket_space_states(), api::Timestamp(1), false)); sort_sent_messages_by_index(sender); @@ -1722,7 +1751,7 @@ TopLevelBucketDBUpdaterTest::merge_bucket_lists(const std::string& existing_data } TEST_F(TopLevelBucketDBUpdaterTest, pending_cluster_state_merge) { - // Result is on the form: [bucket w/o count bits]:[node indexes]|.. + // Result is on the form: [bucket w/o count bits]:[node indexes]|... // Input is on the form: [node]:[bucket w/o count bits]|... // Simple initializing case - ask all nodes for info @@ -2551,6 +2580,101 @@ TEST_F(TopLevelBucketDBUpdaterTest, outdated_bucket_info_reply_is_ignored) { EXPECT_TRUE(handled); // Should be returned as handled even though it's technically ignored. } +TEST_F(TopLevelBucketDBUpdaterTest, state_bundle_with_distributon_config_immediately_switches_config) { + auto baseline = std::make_shared<lib::ClusterState>("version:10 distributor:1 storage:3"); + auto dist = std::make_shared<lib::Distribution>(dist_config_6_nodes_across_2_groups()); + + lib::ClusterStateBundle initial_bundle(baseline, {}, {}, lib::DistributionConfigBundle::of(dist), false); + set_cluster_state_bundle(initial_bundle); + + // Distribution config should have been immediately applied internally + EXPECT_EQ(distributor_bucket_space(BucketId(16, 1)).getDistribution(), *dist); + + // We should now have a single set of bucket info requests that have both the correct distribution + // config hash and the most recent cluster state version. + ASSERT_EQ(_sender.commands().size(), message_count(3)); + // We sample one command and make the assumption that it holds for the rest...! + auto cmd = std::dynamic_pointer_cast<RequestBucketInfoCommand>(_sender.command(0)); + EXPECT_EQ(cmd->getSystemState().getVersion(), 10); + EXPECT_EQ(cmd->getDistributionHash(), dist->getNodeGraph().getDistributionConfigHash()); +} + +TEST_F(TopLevelBucketDBUpdaterTest, state_bundle_with_changed_config_but_unchanged_state_sends_to_full_node_set) { + auto initial_state = std::make_shared<lib::ClusterState>("version:10 distributor:1 storage:4"); + auto initial_dist = std::make_shared<lib::Distribution>(dist_config_6_nodes_across_2_groups()); + + lib::ClusterStateBundle initial_bundle(initial_state, {}, {}, lib::DistributionConfigBundle::of(initial_dist), false); + set_cluster_state_bundle(initial_bundle); + ASSERT_NO_FATAL_FAILURE(expect_and_answer_bucket_info_requests(4, *initial_state)); + + ASSERT_NO_FATAL_FAILURE(verify_state_bundle_propagated_to_stripes(initial_bundle)); + + // New distribution config with same number of nodes, but different topology + auto new_dist = std::make_shared<lib::Distribution>(dist_config_6_nodes_across_4_groups()); + // But the state remains the same aside from the bumped version + auto new_state = std::make_shared<lib::ClusterState>("version:11 distributor:1 storage:4"); + + lib::ClusterStateBundle new_bundle(new_state, {}, {}, lib::DistributionConfigBundle::of(new_dist), false); + set_cluster_state_bundle(new_bundle); + + // We should now consider all nodes as outdated due to the changed distribution config, even + // though the state itself is effectively unchanged. + ASSERT_EQ(_sender.commands().size(), message_count(4)); + auto cmd = std::dynamic_pointer_cast<RequestBucketInfoCommand>(_sender.command(0)); + EXPECT_EQ(cmd->getSystemState().getVersion(), 11); + EXPECT_EQ(cmd->getDistributionHash(), new_dist->getNodeGraph().getDistributionConfigHash()); + + // Distribution config should have been immediately applied internally + EXPECT_EQ(distributor_bucket_space(BucketId(16, 1)).getDistribution(), *new_dist); +} + +TEST_F(TopLevelBucketDBUpdaterTest, state_bundle_with_unchanged_config_and_state_is_immediately_applied) { + auto initial_state = std::make_shared<lib::ClusterState>("version:10 distributor:1 storage:4"); + auto initial_dist = std::make_shared<lib::Distribution>(dist_config_6_nodes_across_2_groups()); + + lib::ClusterStateBundle initial_bundle(initial_state, {}, {}, lib::DistributionConfigBundle::of(initial_dist), false); + set_cluster_state_bundle(initial_bundle); + ASSERT_NO_FATAL_FAILURE(expect_and_answer_bucket_info_requests(4, *initial_state)); + + auto new_state = std::make_shared<lib::ClusterState>("version:11 distributor:1 storage:4"); + lib::ClusterStateBundle new_bundle(new_state, {}, {}, lib::DistributionConfigBundle::of(initial_dist), false); + set_cluster_state_bundle(new_bundle); + + EXPECT_EQ(_sender.commands().size(), 0); + ASSERT_NO_FATAL_FAILURE(verify_state_bundle_propagated_to_stripes(new_bundle)); +} + +TEST_F(TopLevelBucketDBUpdaterTest, internal_distribution_config_usage_is_toggled_by_presence_of_cc_distribution) { + auto initial_state = std::make_shared<lib::ClusterState>("version:10 distributor:1 storage:4"); + auto initial_dist = std::make_shared<lib::Distribution>(dist_config_6_nodes_across_2_groups()); + + // State bundle _with_ distribution + lib::ClusterStateBundle initial_bundle(initial_state, {}, {}, lib::DistributionConfigBundle::of(initial_dist), false); + set_cluster_state_bundle(initial_bundle); + ASSERT_NO_FATAL_FAILURE(expect_and_answer_bucket_info_requests(4, *initial_state)); + + // We should now _ignore_ node-internal config changes that do not originate via bundles + lib::Distribution new_dist(dist_config_6_nodes_across_4_groups()); + set_distribution(dist_config_6_nodes_across_4_groups()); + EXPECT_EQ(distributor_bucket_space(BucketId(16, 1)).getDistribution(), *initial_dist); // _not_ new_dist + EXPECT_EQ(_sender.commands().size(), 0); + + // State _without_ distribution. + // This shall revert to picking up node distribution config (which we updated just above). + auto new_state = std::make_shared<lib::ClusterState>("version:11 distributor:1 storage:4"); + lib::ClusterStateBundle new_bundle(new_state, {}, {}, {}, false); + set_cluster_state_bundle(new_bundle); + // Manually simulate next tick where new distribution config would normally be picked up. + enable_next_distribution_if_changed(); + ASSERT_EQ(_sender.commands().size(), message_count(4)); + + EXPECT_EQ(distributor_bucket_space(BucketId(16, 1)).getDistribution(), new_dist); + + auto cmd = std::dynamic_pointer_cast<RequestBucketInfoCommand>(_sender.command(0)); + EXPECT_EQ(cmd->getSystemState().getVersion(), 11); + // RequestBucketInfo distribution reflects latest node config. + EXPECT_EQ(cmd->getDistributionHash(), new_dist.getNodeGraph().getDistributionConfigHash()); +} struct BucketDBUpdaterSnapshotTest : TopLevelBucketDBUpdaterTest { lib::ClusterState empty_state; diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp index e1b2bc93f62..4816c1dce22 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp @@ -444,6 +444,12 @@ TopLevelDistributorTestUtil::trigger_distribution_change(std::shared_ptr<lib::Di { _node->getComponentRegister().setDistribution(std::move(distr)); _distributor->storageDistributionChanged(); + enable_next_distribution_if_changed(); +} + +void +TopLevelDistributorTestUtil::enable_next_distribution_if_changed() +{ _distributor->enable_next_distribution_if_changed(); } diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.h b/storage/src/tests/distributor/top_level_distributor_test_util.h index 51f0739e3e6..5ae6646502f 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.h +++ b/storage/src/tests/distributor/top_level_distributor_test_util.h @@ -135,6 +135,7 @@ public: bool handle_top_level_message(const std::shared_ptr<api::StorageMessage>& msg); void trigger_distribution_change(std::shared_ptr<lib::Distribution> distr); + void enable_next_distribution_if_changed(); const lib::ClusterStateBundle& current_cluster_state_bundle() const; diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp index e00ce249298..d8a1affde58 100644 --- a/storage/src/tests/distributor/updateoperationtest.cpp +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -11,7 +11,8 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/state.h> -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> using config::ConfigGetter; using config::FileSpec; @@ -31,7 +32,7 @@ struct UpdateOperationTest : Test, DistributorStripeTestUtil { UpdateOperationTest() : _repo(std::make_shared<DocumentTypeRepo>(*ConfigGetter<DocumenttypesConfig>:: - getConfig("config-doctypes", FileSpec("../config-doctypes.cfg")))), + getConfig("config-doctypes", FileSpec(TEST_PATH("../config-doctypes.cfg"))))), _html_type(_repo->getDocumentType("text/html")) { } diff --git a/storage/src/tests/frameworkimpl/status/CMakeLists.txt b/storage/src/tests/frameworkimpl/status/CMakeLists.txt index 319b193a94b..c3d427ac702 100644 --- a/storage/src/tests/frameworkimpl/status/CMakeLists.txt +++ b/storage/src/tests/frameworkimpl/status/CMakeLists.txt @@ -6,7 +6,7 @@ vespa_add_executable(storage_status_gtest_runner_app TEST htmltabletest.cpp statustest.cpp DEPENDS - storage + vespa_storage storage_testcommon GTest::GTest ) diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt index 4c5bc8f324b..02cffc5b960 100644 --- a/storage/src/tests/persistence/CMakeLists.txt +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -16,7 +16,7 @@ vespa_add_executable(storage_persistence_gtest_runner_app TEST testandsettest.cpp gtest_runner.cpp DEPENDS - storage + vespa_storage storage_testpersistence_common GTest::GTest ) diff --git a/storage/src/tests/persistence/common/CMakeLists.txt b/storage/src/tests/persistence/common/CMakeLists.txt index 2083294422b..db0c3975350 100644 --- a/storage/src/tests/persistence/common/CMakeLists.txt +++ b/storage/src/tests/persistence/common/CMakeLists.txt @@ -5,6 +5,6 @@ vespa_add_library(storage_testpersistence_common TEST persistenceproviderwrapper.cpp DEPENDS GTest::GTest - persistence + vespa_persistence storage_testcommon ) diff --git a/storage/src/tests/persistence/filestorage/CMakeLists.txt b/storage/src/tests/persistence/filestorage/CMakeLists.txt index aa7c9fe995c..e4dc652ac0a 100644 --- a/storage/src/tests/persistence/filestorage/CMakeLists.txt +++ b/storage/src/tests/persistence/filestorage/CMakeLists.txt @@ -15,7 +15,7 @@ vespa_add_executable(storage_filestorage_gtest_runner_app TEST singlebucketjointest.cpp gtest_runner.cpp DEPENDS - storage + vespa_storage storage_testhostreporter storage_testpersistence_common GTest::GTest diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index e865c87e15e..524f5bae392 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -1439,4 +1439,52 @@ TEST_F(MergeHandlerTest, partially_filled_apply_bucket_diff_reply) LOG(debug, "got mergebucket reply"); } +TEST_F(MergeHandlerTest, multiple_versions_in_apply_diff_only_writes_newest_version) { + setUpChain(BACK); + + document::TestDocMan doc_mgr; + document::Document::SP doc(doc_mgr.createRandomDocumentAtLocation(_location, 1)); + spi::Timestamp ts_old(10'000); + spi::Timestamp ts_new(20'000); + + PersistenceProviderWrapper provider_wrapper(getPersistenceProvider()); + MergeHandler handler = createHandler(provider_wrapper); + std::vector<api::ApplyBucketDiffCommand::Entry> apply_diff; + // Diff contains two entries for the same document; one old Remove and one newer Put that + // subsumes the Remove operation. We should only schedule the Put to the SPI. + { + api::ApplyBucketDiffCommand::Entry e; + e._entry._timestamp = ts_old; + e._entry._hasMask = 0x1; + e._docName = doc->getId().toString(); + e._entry._flags = MergeHandler::IN_USE | MergeHandler::DELETED; + apply_diff.push_back(e); + } + { + api::ApplyBucketDiffCommand::Entry e; + e._entry._timestamp = ts_new; + e._entry._hasMask = 0x1; + e._entry._flags = MergeHandler::IN_USE; + fill_entry(e, *doc, doc_mgr.getTypeRepo()); + apply_diff.push_back(e); + } + + auto apply_bucket_diff_cmd = std::make_shared<api::ApplyBucketDiffCommand>(_bucket, _nodes); + apply_bucket_diff_cmd->getDiff() = std::move(apply_diff); + + provider_wrapper.clearOperationLog(); + auto tracker = handler.handleApplyBucketDiff(*apply_bucket_diff_cmd, createTracker(apply_bucket_diff_cmd, _bucket)); + ASSERT_FALSE(tracker); + handler.drain_async_writes(); + + // There should be no remove at time=ts_old, only a put at time=ts_new. + // TODO ideally we shouldn't have to know about the other operations... + EXPECT_EQ(provider_wrapper.toString(), + "createIterator(Bucket(0x40000000000004d2), ALL_VERSIONS)\n" + "iterate(1, 18446744073709551615)\n" + "destroyIterator(1)\n" + "put(Bucket(0x40000000000004d2), 20000, id:mail:testdoctype1:n=1234:9380.html)\n" + "getBucketInfo(Bucket(0x40000000000004d2))\n"); +} + } // storage diff --git a/storage/src/tests/storageapi/CMakeLists.txt b/storage/src/tests/storageapi/CMakeLists.txt index 653dc5fbcbe..bf652075b8b 100644 --- a/storage/src/tests/storageapi/CMakeLists.txt +++ b/storage/src/tests/storageapi/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(storageapi_gtest_runner_app TEST storageapi_testbuckets storageapi_testmbusprot storageapi_testmessageapi - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/storageapi/buckets/CMakeLists.txt b/storage/src/tests/storageapi/buckets/CMakeLists.txt index daa3926c3ce..fc5f7a7befb 100644 --- a/storage/src/tests/storageapi/buckets/CMakeLists.txt +++ b/storage/src/tests/storageapi/buckets/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(storageapi_testbuckets SOURCES bucketinfotest.cpp DEPENDS - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/storageapi/mbusprot/CMakeLists.txt b/storage/src/tests/storageapi/mbusprot/CMakeLists.txt index b9fa7303872..746e10e463f 100644 --- a/storage/src/tests/storageapi/mbusprot/CMakeLists.txt +++ b/storage/src/tests/storageapi/mbusprot/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(storageapi_testmbusprot SOURCES storageprotocoltest.cpp DEPENDS - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/storageapi/messageapi/CMakeLists.txt b/storage/src/tests/storageapi/messageapi/CMakeLists.txt index ea182783178..23342651646 100644 --- a/storage/src/tests/storageapi/messageapi/CMakeLists.txt +++ b/storage/src/tests/storageapi/messageapi/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(storageapi_testmessageapi SOURCES storage_message_address_test.cpp DEPENDS - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/storageframework/clock/CMakeLists.txt b/storage/src/tests/storageframework/clock/CMakeLists.txt index 61953ea8bf1..a734cb82566 100644 --- a/storage/src/tests/storageframework/clock/CMakeLists.txt +++ b/storage/src/tests/storageframework/clock/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(storageframework_testclock SOURCES timetest.cpp DEPENDS - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/storageframework/thread/CMakeLists.txt b/storage/src/tests/storageframework/thread/CMakeLists.txt index b88043ff8f3..0959c86ebe7 100644 --- a/storage/src/tests/storageframework/thread/CMakeLists.txt +++ b/storage/src/tests/storageframework/thread/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_library(storageframework_testthread tickingthreadtest.cpp taskthreadtest.cpp DEPENDS - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt index 6c6331c2da5..ffd154508bb 100644 --- a/storage/src/tests/storageserver/CMakeLists.txt +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_library(storage_teststorageserver TEST testvisitormessagesession.cpp DEPENDS storage_testcommon - storage + vespa_storage ) vespa_add_executable(storage_storageserver_gtest_runner_app TEST @@ -23,7 +23,7 @@ vespa_add_executable(storage_storageserver_gtest_runner_app TEST DEPENDS storage_testcommon storage_teststorageserver - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 1eb6bf5dd9a..365b9efeff0 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -19,6 +19,7 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/documentapi/messagebus/messages/testandsetcondition.h> using document::Bucket; @@ -71,7 +72,7 @@ struct DocumentApiConverterTest : Test { DocumentApiConverterTest() : _bucketResolver(std::make_shared<MockBucketResolver>()), - _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig("../config-doctypes.cfg"))), + _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig(TEST_PATH("../config-doctypes.cfg")))), _html_type(*_repo->getDocumentType("text/html")) { } diff --git a/storage/src/tests/storageserver/rpc/CMakeLists.txt b/storage/src/tests/storageserver/rpc/CMakeLists.txt index 919a131bcdb..f7f73ff299a 100644 --- a/storage/src/tests/storageserver/rpc/CMakeLists.txt +++ b/storage/src/tests/storageserver/rpc/CMakeLists.txt @@ -9,7 +9,7 @@ vespa_add_executable(storage_storageserver_rpc_gtest_runner_app TEST DEPENDS storage_testcommon storage_teststorageserver - storage + vespa_storage GTest::GTest ) diff --git a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp index c3641b9bc56..e59f6d22080 100644 --- a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp @@ -121,7 +121,7 @@ struct SetStateFixture : FixtureBase { } static lib::ClusterStateBundle dummy_baseline_bundle_with_deferred_activation(bool deferred) { - return lib::ClusterStateBundle(lib::ClusterState("version:123 distributor:3 storage:3"), {}, deferred); + return {lib::ClusterState("version:123 distributor:3 storage:3"), {}, deferred}; } }; @@ -166,6 +166,16 @@ TEST_F(ClusterControllerApiRpcServiceTest, set_distribution_states_rpc_with_feed f.assert_request_received_and_propagated(bundle); } +TEST_F(ClusterControllerApiRpcServiceTest, can_receive_cluster_state_bundle_with_embedded_distribution_config) { + auto distr_cfg = lib::DistributionConfigBundle::of(lib::Distribution::getDefaultDistributionConfig(3, 14)); + SetStateFixture f; + lib::ClusterStateBundle bundle( + std::make_shared<const lib::ClusterState>("version:123 distributor:3 storage:3"), + {}, std::nullopt, std::move(distr_cfg), false); + + f.assert_request_received_and_propagated(bundle); +} + TEST_F(ClusterControllerApiRpcServiceTest, compressed_bundle_is_transparently_uncompressed) { SetStateFixture f; auto state_str = make_compressable_state_string(); diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index b785bc141b6..79246cb3ce1 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -32,7 +32,18 @@ struct StateManagerTest : Test, NodeStateReporter { void SetUp() override; void TearDown() override; - void force_current_cluster_state_version(uint32_t version); + static std::shared_ptr<api::SetSystemStateCommand> make_set_state_cmd(vespalib::stringref state_str, uint16_t cc_index) { + auto cmd = std::make_shared<api::SetSystemStateCommand>(lib::ClusterState(state_str)); + cmd->setSourceIndex(cc_index); + return cmd; + } + + void get_single_reply(std::shared_ptr<api::StorageReply>& reply_out); + void get_only_ok_reply(std::shared_ptr<api::StorageReply>& reply_out); + void force_current_cluster_state_version(uint32_t version, uint16_t cc_index); + void force_current_cluster_state_version(uint32_t version) { + force_current_cluster_state_version(version, 0); + } void mark_reported_node_state_up(); void send_down_get_node_state_request(uint16_t controller_index); void assert_ok_get_node_state_reply_sent_and_clear(); @@ -86,11 +97,30 @@ StateManagerTest::TearDown() { } void -StateManagerTest::force_current_cluster_state_version(uint32_t version) +StateManagerTest::get_single_reply(std::shared_ptr<api::StorageReply>& reply_out) +{ + ASSERT_EQ(_upper->getNumReplies(), 1); + ASSERT_TRUE(_upper->getReply(0)->getType().isReply()); + reply_out = std::dynamic_pointer_cast<api::StorageReply>(_upper->getReply(0)); + ASSERT_TRUE(reply_out.get() != nullptr); + _upper->reset(); +} + +void +StateManagerTest::get_only_ok_reply(std::shared_ptr<api::StorageReply>& reply_out) +{ + ASSERT_NO_FATAL_FAILURE(get_single_reply(reply_out)); + ASSERT_EQ(reply_out->getResult(), api::ReturnCode(api::ReturnCode::OK)); +} + +void +StateManagerTest::force_current_cluster_state_version(uint32_t version, uint16_t cc_index) { ClusterState state(*_manager->getClusterStateBundle()->getBaselineClusterState()); state.setVersion(version); - _manager->setClusterStateBundle(lib::ClusterStateBundle(state)); + const auto maybe_rejected_by_ver = _manager->try_set_cluster_state_bundle( + std::make_shared<const lib::ClusterStateBundle>(state), cc_index); + ASSERT_EQ(maybe_rejected_by_ver, std::nullopt); } void @@ -118,23 +148,12 @@ StateManagerTest::extract_cluster_state_version_from_host_info(uint32_t& version version_out = clusterStateVersionCursor.asLong(); } -#define GET_ONLY_OK_REPLY(varname) \ -{ \ - ASSERT_EQ(size_t(1), _upper->getNumReplies()); \ - ASSERT_TRUE(_upper->getReply(0)->getType().isReply()); \ - varname = std::dynamic_pointer_cast<api::StorageReply>( \ - _upper->getReply(0)); \ - ASSERT_TRUE(varname.get() != nullptr); \ - _upper->reset(); \ - ASSERT_EQ(api::ReturnCode(api::ReturnCode::OK), \ - varname->getResult()); \ -} - TEST_F(StateManagerTest, cluster_state) { std::shared_ptr<api::StorageReply> reply; // Verify initial state on startup auto currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); EXPECT_EQ("cluster:d", currentState->toString(false)); + EXPECT_EQ(currentState->getVersion(), 0); auto currentNodeState = _manager->getCurrentNodeState(); EXPECT_EQ("s:d", currentNodeState->toString(false)); @@ -142,7 +161,7 @@ TEST_F(StateManagerTest, cluster_state) { ClusterState sendState("storage:4 .2.s:m"); auto cmd = std::make_shared<api::SetSystemStateCommand>(sendState); _upper->sendDown(cmd); - GET_ONLY_OK_REPLY(reply); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); currentState = _manager->getClusterStateBundle()->getBaselineClusterState(); EXPECT_EQ(sendState, *currentState); @@ -151,13 +170,64 @@ TEST_F(StateManagerTest, cluster_state) { EXPECT_EQ("s:m", currentNodeState->toString(false)); } +TEST_F(StateManagerTest, accept_lower_state_versions_if_strict_requirement_disabled) { + _manager->set_require_strictly_increasing_cluster_state_versions(false); + + ASSERT_NO_FATAL_FAILURE(force_current_cluster_state_version(123, 1)); // CC 1 + ASSERT_EQ(_manager->getClusterStateBundle()->getVersion(), 123); + + _upper->sendDown(make_set_state_cmd("version:122 distributor:1 storage:1", 0)); // CC 0 + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + EXPECT_EQ(_manager->getClusterStateBundle()->getVersion(), 122); +} + +TEST_F(StateManagerTest, reject_lower_state_versions_if_strict_requirement_enabled) { + _manager->set_require_strictly_increasing_cluster_state_versions(true); + + ASSERT_NO_FATAL_FAILURE(force_current_cluster_state_version(123, 1)); // CC 1 + ASSERT_EQ(_manager->getClusterStateBundle()->getVersion(), 123); + + _upper->sendDown(make_set_state_cmd("version:122 distributor:1 storage:1", 0)); // CC 0 + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_single_reply(reply)); + api::ReturnCode expected_res(api::ReturnCode::REJECTED, "Cluster state version 122 rejected; node already has " + "a higher cluster state version (123)"); + EXPECT_EQ(reply->getResult(), expected_res); + EXPECT_EQ(_manager->getClusterStateBundle()->getVersion(), 123); +} + +// Observing a lower cluster state version from the same CC index directly implies that the ZooKeeper +// state has been lost, at which point we pragmatically (but begrudgingly) accept the state version +// to avoid stalling the entire cluster for an indeterminate amount of time. +TEST_F(StateManagerTest, accept_lower_state_versions_from_same_cc_index_even_if_strict_requirement_enabled) { + _manager->set_require_strictly_increasing_cluster_state_versions(true); + + ASSERT_NO_FATAL_FAILURE(force_current_cluster_state_version(123, 1)); // CC 1 + ASSERT_EQ(_manager->getClusterStateBundle()->getVersion(), 123); + + ASSERT_NO_FATAL_FAILURE(force_current_cluster_state_version(124, 2)); // CC 2 + ASSERT_EQ(_manager->getClusterStateBundle()->getVersion(), 124); + + // CC 1 restarts from scratch with previous ZK state up in smoke. + _upper->sendDown(make_set_state_cmd("version:3 distributor:1 storage:1", 1)); + std::shared_ptr<api::StorageReply> reply; + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + EXPECT_EQ(_manager->getClusterStateBundle()->getVersion(), 3); + + // CC 2 restarts and continues from where CC 1 left off. + _upper->sendDown(make_set_state_cmd("version:4 distributor:1 storage:1", 2)); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); + EXPECT_EQ(_manager->getClusterStateBundle()->getVersion(), 4); +} + namespace { struct MyStateListener : public StateListener { const NodeStateUpdater& updater; lib::NodeState current; std::ostringstream ost; - MyStateListener(const NodeStateUpdater& upd); + explicit MyStateListener(const NodeStateUpdater& upd); ~MyStateListener() override; void handleNewState() noexcept override { @@ -194,7 +264,7 @@ TEST_F(StateManagerTest, reported_node_state) { // And get node state command (no expected state) auto cmd = std::make_shared<api::GetNodeStateCommand>(lib::NodeState::UP()); _upper->sendDown(cmd); - GET_ONLY_OK_REPLY(reply); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); ASSERT_EQ(api::MessageType::GETNODESTATE_REPLY, reply->getType()); nodeState = std::make_shared<NodeState>( dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState()); @@ -203,7 +273,7 @@ TEST_F(StateManagerTest, reported_node_state) { cmd = std::make_shared<api::GetNodeStateCommand>( std::make_unique<NodeState>(NodeType::STORAGE, State::INITIALIZING)); _upper->sendDown(cmd); - GET_ONLY_OK_REPLY(reply); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); ASSERT_EQ(api::MessageType::GETNODESTATE_REPLY, reply->getType()); nodeState = std::make_unique<NodeState>( dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState()); @@ -222,7 +292,7 @@ TEST_F(StateManagerTest, reported_node_state) { _manager->setReportedNodeState(ns); } - GET_ONLY_OK_REPLY(reply); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); ASSERT_EQ(api::MessageType::GETNODESTATE_REPLY, reply->getType()); nodeState = std::make_unique<NodeState>( dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState()); @@ -244,7 +314,7 @@ TEST_F(StateManagerTest, reported_node_state) { } TEST_F(StateManagerTest, current_cluster_state_version_is_included_in_host_info_json) { - force_current_cluster_state_version(123); + ASSERT_NO_FATAL_FAILURE(force_current_cluster_state_version(123)); uint32_t version; ASSERT_NO_FATAL_FAILURE(extract_cluster_state_version_from_host_info(version)); EXPECT_EQ(version, 123); @@ -266,7 +336,7 @@ void StateManagerTest::send_down_get_node_state_request(uint16_t controller_inde void StateManagerTest::assert_ok_get_node_state_reply_sent_and_clear() { ASSERT_EQ(1, _upper->getNumReplies()); std::shared_ptr<api::StorageReply> reply; - GET_ONLY_OK_REPLY(reply); // Implicitly clears messages from _upper + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); // Implicitly clears messages from _upper ASSERT_EQ(api::MessageType::GETNODESTATE_REPLY, reply->getType()); } @@ -344,7 +414,7 @@ TEST_F(StateManagerTest, request_almost_immediate_replies_triggers_fast_reply) } TEST_F(StateManagerTest, activation_command_is_bounced_with_current_cluster_state_version) { - force_current_cluster_state_version(12345); + ASSERT_NO_FATAL_FAILURE(force_current_cluster_state_version(12345)); auto cmd = std::make_shared<api::ActivateClusterStateVersionCommand>(12340); cmd->setTimeout(10000000ms); @@ -353,7 +423,7 @@ TEST_F(StateManagerTest, activation_command_is_bounced_with_current_cluster_stat ASSERT_EQ(1, _upper->getNumReplies()); std::shared_ptr<api::StorageReply> reply; - GET_ONLY_OK_REPLY(reply); // Implicitly clears messages from _upper + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); // Implicitly clears messages from _upper ASSERT_EQ(api::MessageType::ACTIVATE_CLUSTER_STATE_VERSION_REPLY, reply->getType()); auto& activate_reply = dynamic_cast<api::ActivateClusterStateVersionReply&>(*reply); EXPECT_EQ(12340, activate_reply.activateVersion()); @@ -366,7 +436,7 @@ TEST_F(StateManagerTest, non_deferred_cluster_state_sets_reported_cluster_state_ cmd->setSourceIndex(0); _upper->sendDown(cmd); std::shared_ptr<api::StorageReply> reply; - GET_ONLY_OK_REPLY(reply); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); uint32_t version; ASSERT_NO_FATAL_FAILURE(extract_cluster_state_version_from_host_info(version)); @@ -374,7 +444,7 @@ TEST_F(StateManagerTest, non_deferred_cluster_state_sets_reported_cluster_state_ } TEST_F(StateManagerTest, deferred_cluster_state_does_not_update_state_until_activation_edge) { - force_current_cluster_state_version(100); + ASSERT_NO_FATAL_FAILURE(force_current_cluster_state_version(100)); lib::ClusterStateBundle deferred_bundle(lib::ClusterState("version:101 distributor:1 storage:1"), {}, true); auto state_cmd = std::make_shared<api::SetSystemStateCommand>(deferred_bundle); @@ -382,7 +452,7 @@ TEST_F(StateManagerTest, deferred_cluster_state_does_not_update_state_until_acti state_cmd->setSourceIndex(0); _upper->sendDown(state_cmd); std::shared_ptr<api::StorageReply> reply; - GET_ONLY_OK_REPLY(reply); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); uint32_t version; ASSERT_NO_FATAL_FAILURE(extract_cluster_state_version_from_host_info(version)); @@ -392,7 +462,7 @@ TEST_F(StateManagerTest, deferred_cluster_state_does_not_update_state_until_acti activation_cmd->setTimeout(1000s); activation_cmd->setSourceIndex(0); _upper->sendDown(activation_cmd); - GET_ONLY_OK_REPLY(reply); + ASSERT_NO_FATAL_FAILURE(get_only_ok_reply(reply)); ASSERT_NO_FATAL_FAILURE(extract_cluster_state_version_from_host_info(version)); EXPECT_EQ(version, 101); diff --git a/storage/src/tests/visiting/CMakeLists.txt b/storage/src/tests/visiting/CMakeLists.txt index 211dea912d8..037336c3edf 100644 --- a/storage/src/tests/visiting/CMakeLists.txt +++ b/storage/src/tests/visiting/CMakeLists.txt @@ -8,7 +8,7 @@ vespa_add_executable(storage_visiting_gtest_runner_app TEST visitortest.cpp gtest_runner.cpp DEPENDS - storage + vespa_storage storage_teststorageserver GTest::GTest ) diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt index 7e02124d170..701afd39b1c 100644 --- a/storage/src/vespa/storage/CMakeLists.txt +++ b/storage/src/vespa/storage/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage +vespa_add_library(vespa_storage SOURCES $<TARGET_OBJECTS:storage_bucketdb> $<TARGET_OBJECTS:storage_common> @@ -36,4 +36,4 @@ vespa_add_library(storage protobuf::libprotobuf ) -vespa_add_target_package_dependency(storage Protobuf) +vespa_add_target_package_dependency(vespa_storage Protobuf) diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 5337be6d79f..1f20a19ec51 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -2,26 +2,26 @@ #include "bucketmanager.h" #include "minimumusedbitstracker.h" -#include <iomanip> +#include <vespa/config/helper/configgetter.hpp> +#include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/metrics/jsonwriter.h> #include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/nodestateupdater.h> -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> -#include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/storageutil/distributorstatecache.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/stat.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageframework/generic/status/xmlstatusreporter.h> #include <vespa/storageframework/generic/thread/thread.h> -#include <vespa/storageframework/generic/clock/timer.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/state.h> -#include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storageapi/message/stat.h> -#include <vespa/metrics/jsonwriter.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vdslib/distribution/global_bucket_space_distribution_converter.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vespalib/util/stringfmt.h> +#include <iomanip> #include <ranges> -#include <vespa/config/helper/configgetter.hpp> #include <chrono> #include <thread> @@ -526,6 +526,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac using RBISP = std::shared_ptr<api::RequestBucketInfoCommand>; std::map<uint16_t, RBISP> requests; + // TODO fetch distribution from bundle as well auto distribution(_component.getBucketSpaceRepo().get(bucketSpace).getDistribution()); auto clusterStateBundle(_component.getStateUpdater().getClusterStateBundle()); assert(clusterStateBundle); diff --git a/storage/src/vespa/storage/common/CMakeLists.txt b/storage/src/vespa/storage/common/CMakeLists.txt index 708f3dd05b9..a385867ac98 100644 --- a/storage/src/vespa/storage/common/CMakeLists.txt +++ b/storage/src/vespa/storage/common/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(storage_common OBJECT content_bucket_space.cpp content_bucket_space_repo.cpp distributorcomponent.cpp - global_bucket_space_distribution_converter.cpp messagebucket.cpp message_guard.cpp messagesender.cpp diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp index 0cedb78cfe6..92b5257b991 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.cpp +++ b/storage/src/vespa/storage/common/content_bucket_space.cpp @@ -4,44 +4,76 @@ namespace storage { +ClusterStateAndDistribution::ClusterStateAndDistribution( + std::shared_ptr<const lib::ClusterState> cluster_state, + std::shared_ptr<const lib::Distribution> distribution) noexcept + : _cluster_state(std::move(cluster_state)), + _distribution(std::move(distribution)) +{ +} + +ClusterStateAndDistribution::~ClusterStateAndDistribution() = default; + +std::shared_ptr<const ClusterStateAndDistribution> +ClusterStateAndDistribution::with_new_state(std::shared_ptr<const lib::ClusterState> cluster_state) const { + return std::make_shared<const ClusterStateAndDistribution>(std::move(cluster_state), _distribution); +} + +std::shared_ptr<const ClusterStateAndDistribution> +ClusterStateAndDistribution::with_new_distribution(std::shared_ptr<const lib::Distribution> distribution) const { + return std::make_shared<const ClusterStateAndDistribution>(_cluster_state, std::move(distribution)); +} + ContentBucketSpace::ContentBucketSpace(document::BucketSpace bucketSpace, const ContentBucketDbOptions& db_opts) : _bucketSpace(bucketSpace), _bucketDatabase(db_opts), _lock(), - _clusterState(), - _distribution(), + _state_and_distribution(std::make_shared<ClusterStateAndDistribution>()), _nodeUpInLastNodeStateSeenByProvider(false), _nodeMaintenanceInLastNodeStateSeenByProvider(false) { } void +ContentBucketSpace::set_state_and_distribution(std::shared_ptr<const ClusterStateAndDistribution> state_and_distr) noexcept { + assert(state_and_distr); + std::lock_guard guard(_lock); + _state_and_distribution = std::move(state_and_distr); +} + +std::shared_ptr<const ClusterStateAndDistribution> +ContentBucketSpace::state_and_distribution() const noexcept { + std::lock_guard guard(_lock); + return _state_and_distribution; +} + +void ContentBucketSpace::setClusterState(std::shared_ptr<const lib::ClusterState> clusterState) { std::lock_guard guard(_lock); - _clusterState = std::move(clusterState); + _state_and_distribution = _state_and_distribution->with_new_state(std::move(clusterState)); } std::shared_ptr<const lib::ClusterState> ContentBucketSpace::getClusterState() const { std::lock_guard guard(_lock); - return _clusterState; + return _state_and_distribution->_cluster_state; } void ContentBucketSpace::setDistribution(std::shared_ptr<const lib::Distribution> distribution) { std::lock_guard guard(_lock); - _distribution = std::move(distribution); + _state_and_distribution = _state_and_distribution->with_new_distribution(std::move(distribution)); } std::shared_ptr<const lib::Distribution> ContentBucketSpace::getDistribution() const { std::lock_guard guard(_lock); - return _distribution; + return _state_and_distribution->_distribution; } bool diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h index 93b171bd48e..eb48640c97b 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.h +++ b/storage/src/vespa/storage/common/content_bucket_space.h @@ -12,6 +12,27 @@ class ClusterState; class Distribution; } +struct ClusterStateAndDistribution { + std::shared_ptr<const lib::ClusterState> _cluster_state; + std::shared_ptr<const lib::Distribution> _distribution; + + ClusterStateAndDistribution() = default; + ClusterStateAndDistribution(std::shared_ptr<const lib::ClusterState> cluster_state, + std::shared_ptr<const lib::Distribution> distribution) noexcept; + ~ClusterStateAndDistribution(); + + [[nodiscard]] bool valid() const noexcept { return _cluster_state && _distribution; } + + // Precondition: valid() == true + [[nodiscard]] const lib::ClusterState& cluster_state() const noexcept { return *_cluster_state; } + [[nodiscard]] const lib::Distribution& distribution() const noexcept { return *_distribution; } + + [[nodiscard]] std::shared_ptr<const ClusterStateAndDistribution> with_new_state( + std::shared_ptr<const lib::ClusterState> cluster_state) const; + [[nodiscard]] std::shared_ptr<const ClusterStateAndDistribution> with_new_distribution( + std::shared_ptr<const lib::Distribution> distribution) const; +}; + /** * Class representing a bucket space (with associated bucket database) on a content node. */ @@ -20,8 +41,7 @@ private: document::BucketSpace _bucketSpace; StorBucketDatabase _bucketDatabase; mutable std::mutex _lock; - std::shared_ptr<const lib::ClusterState> _clusterState; - std::shared_ptr<const lib::Distribution> _distribution; + std::shared_ptr<const ClusterStateAndDistribution> _state_and_distribution; bool _nodeUpInLastNodeStateSeenByProvider; bool _nodeMaintenanceInLastNodeStateSeenByProvider; @@ -31,10 +51,18 @@ public: document::BucketSpace bucketSpace() const noexcept { return _bucketSpace; } StorBucketDatabase &bucketDatabase() { return _bucketDatabase; } + + void set_state_and_distribution(std::shared_ptr<const ClusterStateAndDistribution> state_and_distr) noexcept; + [[nodiscard]] std::shared_ptr<const ClusterStateAndDistribution> state_and_distribution() const noexcept; + // TODO deprecate; only use atomic state+distribution setter void setClusterState(std::shared_ptr<const lib::ClusterState> clusterState); + // TODO deprecate; only use atomic state+distribution getter std::shared_ptr<const lib::ClusterState> getClusterState() const; + // TODO deprecate; only use atomic state+distribution setter void setDistribution(std::shared_ptr<const lib::Distribution> distribution); + // TODO deprecate; only use atomic state+distribution getter std::shared_ptr<const lib::Distribution> getDistribution() const; + bool getNodeUpInLastNodeStateSeenByProvider() const; void setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider); bool getNodeMaintenanceInLastNodeStateSeenByProvider() const; diff --git a/storage/src/vespa/storage/common/storagecomponent.h b/storage/src/vespa/storage/common/storagecomponent.h index e9ac691c0e8..677d5652fe3 100644 --- a/storage/src/vespa/storage/common/storagecomponent.h +++ b/storage/src/vespa/storage/common/storagecomponent.h @@ -91,6 +91,8 @@ public: std::shared_ptr<Repos> getTypeRepo() const; const document::BucketIdFactory& getBucketIdFactory() const { return _bucketIdFactory; } + // Must NOT be used by lower-level components, as it may be out of sync with distribution + // config propagated from the cluster controller. DistributionSP getDistribution() const; NodeStateUpdater& getStateUpdater() const; uint64_t getGeneration() const { return _generation.load(std::memory_order_relaxed); } diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 6957e541d6b..cfbc4caf82d 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -46,6 +46,7 @@ DistributorConfiguration::DistributorConfiguration(StorageComponent& component) _use_weak_internal_read_consistency_for_client_gets(false), _enable_metadata_only_fetch_phase_for_inconsistent_updates(true), _enable_operation_cancellation(false), + _symmetric_put_and_activate_replica_selection(false), _minimumReplicaCountingMode(ReplicaCountingMode::TRUSTED) { } @@ -150,6 +151,7 @@ DistributorConfiguration::configure(const DistributorManagerConfig & config) _max_activation_inhibited_out_of_sync_groups = config.maxActivationInhibitedOutOfSyncGroups; _enable_operation_cancellation = config.enableOperationCancellation; _minimumReplicaCountingMode = deriveReplicaCountingMode(config.minimumReplicaCountingMode); + _symmetric_put_and_activate_replica_selection = config.symmetricPutAndActivateReplicaSelection; if (config.maxClusterClockSkewSec >= 0) { _maxClusterClockSkew = std::chrono::seconds(config.maxClusterClockSkewSec); diff --git a/storage/src/vespa/storage/config/distributorconfiguration.h b/storage/src/vespa/storage/config/distributorconfiguration.h index 38fac13150c..2b73fdc0fa1 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.h +++ b/storage/src/vespa/storage/config/distributorconfiguration.h @@ -234,8 +234,11 @@ public: [[nodiscard]] bool enable_operation_cancellation() const noexcept { return _enable_operation_cancellation; } + [[nodiscard]] bool symmetric_put_and_activate_replica_selection() const noexcept { + return _symmetric_put_and_activate_replica_selection; + } - bool containsTimeStatement(const std::string& documentSelection) const; + [[nodiscard]] bool containsTimeStatement(const std::string& documentSelection) const; private: StorageComponent& _component; @@ -276,6 +279,7 @@ private: bool _use_weak_internal_read_consistency_for_client_gets; bool _enable_metadata_only_fetch_phase_for_inconsistent_updates; //TODO Rewrite tests and GC bool _enable_operation_cancellation; + bool _symmetric_put_and_activate_replica_selection; ReplicaCountingMode _minimumReplicaCountingMode; }; diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def index 3f6028d7fa1..a4d4461ba68 100644 --- a/storage/src/vespa/storage/config/stor-distributormanager.def +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -159,6 +159,13 @@ num_distributor_stripes int default=0 restart ## requests partially or fully "invalidated" by such a change. enable_operation_cancellation bool default=false +## Iff true there will be an 1-1 symmetry between the replicas chosen as feed targets +## for Put operations and the replica selection logic for bucket activation. In particular, +## the most preferred replica for feed will be the most preferred bucket for activation. +## This helps ensure that new versions of documents are routed to replicas that are most +## likely to reflect these changes as part of visible search results. +symmetric_put_and_activate_replica_selection bool default=false + ## TODO GC very soon, it has no effect. priority_merge_out_of_sync_copies int default=120 diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def index 44e4b14eafc..49ce0b678d0 100644 --- a/storage/src/vespa/storage/config/stor-server.def +++ b/storage/src/vespa/storage/config/stor-server.def @@ -118,3 +118,11 @@ content_node_bucket_db_stripe_bits int default=4 restart ## Iff set, a special `pidfile` file is written under the node's root directory upon ## startup containing the PID of the running process. write_pid_file_on_startup bool default=true + +## Iff true, received cluster state versions that are lower than the current active +## (or pending to be active) version on the node will be explicitly rejected. This +## prevents race conditions caused by multiple cluster controllers believing they +## are the leader during overlapping time intervals, as only the most recent leader +## is able to increment the current state version in ZooKeeper, but the old controller +## may still attempt to publish its old state. +require_strictly_increasing_cluster_state_versions bool default=false diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index 195410cbe03..212570a3033 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(storage_distributor OBJECT blockingoperationstarter.cpp bucket_db_prune_elision.cpp bucket_ownership_calculator.cpp - bucket_space_distribution_configs.cpp bucket_space_distribution_context.cpp bucket_space_state_map.cpp bucket_spaces_stats_provider.cpp diff --git a/storage/src/vespa/storage/distributor/activecopy.cpp b/storage/src/vespa/storage/distributor/activecopy.cpp index 35070bcee3b..b823978a0cc 100644 --- a/storage/src/vespa/storage/distributor/activecopy.cpp +++ b/storage/src/vespa/storage/distributor/activecopy.cpp @@ -108,6 +108,7 @@ buildNodeList(const BucketDatabase::Entry& e,vespalib::ConstArrayRef<uint16_t> n struct ActiveStateOrder { bool operator()(const ActiveCopy & e1, const ActiveCopy & e2) noexcept { + // Replica selection order should be kept in sync with OperationTargetResolverImpl's InstanceOrder. if (e1._ready != e2._ready) { return e1._ready; } @@ -120,7 +121,9 @@ struct ActiveStateOrder { if (e1._active != e2._active) { return e1._active; } - return e1.nodeIndex() < e2.nodeIndex(); + // Use _entry_ order instead of node index, as it is in ideal state order (even for retired + // nodes), which avoids unintentional affinities towards lower node indexes. + return e1.entryIndex() < e2.entryIndex(); } }; diff --git a/storage/src/vespa/storage/distributor/distributor_interface.h b/storage/src/vespa/storage/distributor/distributor_interface.h index 67a8b47c503..8ee1de6fc64 100644 --- a/storage/src/vespa/storage/distributor/distributor_interface.h +++ b/storage/src/vespa/storage/distributor/distributor_interface.h @@ -14,9 +14,27 @@ class DistributorMetricSet; */ class DistributorInterface : public DistributorMessageSender { public: - virtual ~DistributorInterface() = default; - virtual DistributorMetricSet& metrics() = 0; - virtual const DistributorConfiguration& config() const = 0; + ~DistributorInterface() override = default; + [[nodiscard]] virtual DistributorMetricSet& metrics() = 0; + [[nodiscard]] virtual const DistributorConfiguration& config() const = 0; + // Called from our own bucket DB updater when a cluster state bundle with embedded distribution + // config is received. Once at least one such embedded config has been received, config from + // the storage component should be _ignored_, as the cluster controller is the lone source of + // truth for distribution config. + // Returns true iff `distribution` differs from the existing config. + [[nodiscard]] virtual bool receive_distribution_from_cluster_controller(std::shared_ptr<const lib::Distribution> distribution) = 0; + + // Whether this distributor treats the CC as the source of truth for distribution config, and + // thus ignores node-internal distribution config changes. + [[nodiscard]] virtual bool cluster_controller_is_distribution_source_of_truth() const noexcept = 0; + + // Indicates that we are no longer receiving distribution config from the cluster controller, + // and that the process' own distribution config should be used. This is a safety valve in + // the case the cluster controller is rolled back or reconfigured to not send distribution + // config as part of state bundles. + // This may trigger a distribution change on the next tick if internal distribution differs + // from that previously received from the cluster controller. + virtual void revert_distribution_source_of_truth_to_node_internal_config() = 0; }; } diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.cpp b/storage/src/vespa/storage/distributor/distributor_stripe.cpp index c00ab7080da..f8abdf78c2b 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe.cpp @@ -12,7 +12,6 @@ #include "stripe_host_info_notifier.h" #include "throttlingoperationstarter.h" #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storage/common/node_identity.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> @@ -21,6 +20,7 @@ #include <vespa/storage/config/distributorconfiguration.h> #include <vespa/storageframework/generic/status/xmlstatusreporter.h> #include <vespa/vdslib/distribution/distribution.h> +#include <vespa/vdslib/distribution/global_bucket_space_distribution_converter.h> #include <vespa/vespalib/util/memoryusage.h> #include <algorithm> @@ -545,7 +545,7 @@ DistributorStripe::checkBucketForSplit(document::BucketSpace bucketSpace, const void DistributorStripe::propagateDefaultDistribution(std::shared_ptr<const lib::Distribution> distribution) { - auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); + auto global_distr = lib::GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); for (auto* repo : {_bucketSpaceRepo.get(), _readOnlyBucketSpaceRepo.get()}) { repo->get(document::FixedBucketSpaces::default_space()).setDistribution(distribution); repo->get(document::FixedBucketSpaces::global_space()).setDistribution(global_distr); @@ -554,7 +554,7 @@ DistributorStripe::propagateDefaultDistribution(std::shared_ptr<const lib::Distr // Only called when stripe is in rendezvous freeze void -DistributorStripe::update_distribution_config(const BucketSpaceDistributionConfigs& new_configs) { +DistributorStripe::update_distribution_config(const lib::BucketSpaceDistributionConfigs& new_configs) { auto default_distr = new_configs.get_or_nullptr(document::FixedBucketSpaces::default_space()); auto global_distr = new_configs.get_or_nullptr(document::FixedBucketSpaces::global_space()); assert(default_distr && global_distr); diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.h b/storage/src/vespa/storage/distributor/distributor_stripe.h index d782432ab35..f5793e4d39b 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe.h @@ -283,7 +283,7 @@ private: void propagate_config_snapshot_to_internal_components(); // Additional implementations of TickableStripe: - void update_distribution_config(const BucketSpaceDistributionConfigs& new_configs) override; + void update_distribution_config(const lib::BucketSpaceDistributionConfigs& new_configs) override; void update_total_distributor_config(std::shared_ptr<const DistributorConfiguration> config) override; void set_pending_cluster_state_bundle(const lib::ClusterStateBundle& pending_state) override; void clear_pending_cluster_state_bundle() override; diff --git a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp index f1cce40ee8b..991a73ec5c6 100644 --- a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp +++ b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp @@ -33,7 +33,7 @@ void MultiThreadedStripeAccessGuard::update_total_distributor_config(std::shared }); } -void MultiThreadedStripeAccessGuard::update_distribution_config(const BucketSpaceDistributionConfigs& new_configs) { +void MultiThreadedStripeAccessGuard::update_distribution_config(const lib::BucketSpaceDistributionConfigs& new_configs) { for_each_stripe([&](TickableStripe& stripe) { stripe.update_distribution_config(new_configs); }); diff --git a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.h b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.h index a4392416025..3ce22a3e1a7 100644 --- a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.h +++ b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.h @@ -31,7 +31,7 @@ public: void update_total_distributor_config(std::shared_ptr<const DistributorConfiguration> config) override; - void update_distribution_config(const BucketSpaceDistributionConfigs& new_configs) override; + void update_distribution_config(const lib::BucketSpaceDistributionConfigs& new_configs) override; void set_pending_cluster_state_bundle(const lib::ClusterStateBundle& pending_state) override; void clear_pending_cluster_state_bundle() override; void enable_cluster_state_bundle(const lib::ClusterStateBundle& new_state, diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index 54087850e1b..a92896279b0 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -180,6 +180,8 @@ void PutOperation::start_direct_put_dispatch(DistributorStripeMessageSender& sen _op_ctx.distributor_config().getMinimalBucketSplit(), _bucket_space.getDistribution().getRedundancy(), _msg->getBucket().getBucketSpace()); + targetResolver.use_symmetric_replica_selection( + _op_ctx.distributor_config().symmetric_put_and_activate_replica_selection()); OperationTargetList targets(targetResolver.getTargets(OperationTargetResolver::PUT, _doc_id_bucket_id)); for (const auto& target : targets) { diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp index 394c13c2bad..618cfb56359 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp @@ -10,9 +10,12 @@ namespace storage::distributor { BucketInstance::BucketInstance(const document::BucketId& id, const api::BucketInfo& info, lib::Node node, - uint16_t idealLocationPriority, bool trusted, bool exist) noexcept + uint16_t ideal_location_priority, uint16_t db_entry_order, + bool trusted, bool exist) noexcept : _bucket(id), _info(info), _node(node), - _idealLocationPriority(idealLocationPriority), _trusted(trusted), _exist(exist) + _ideal_location_priority(ideal_location_priority), + _db_entry_order(db_entry_order), + _trusted(trusted), _exists(exist) { } @@ -24,8 +27,8 @@ BucketInstance::print(vespalib::asciistream& out, const PrintProperties&) const std::ostringstream ost; ost << std::hex << _bucket.getId(); - out << "(" << ost.str() << ", " << infoString << ", node " << _node.getIndex() << ", ideal " << _idealLocationPriority - << (_trusted ? ", trusted" : "") << (_exist ? "" : ", new copy") << ")"; + out << "(" << ost.str() << ", " << infoString << ", node " << _node.getIndex() << ", ideal " << _ideal_location_priority + << (_trusted ? ", trusted" : "") << (_exists ? "" : ", new copy") << ")"; } bool @@ -42,7 +45,7 @@ BucketInstanceList::add(const BucketDatabase::Entry& e, const IdealServiceLayerN for (uint32_t i = 0; i < e.getBucketInfo().getNodeCount(); ++i) { const BucketCopy& copy(e.getBucketInfo().getNodeRef(i)); lib::Node node(lib::NodeType::STORAGE, copy.getNode()); - _instances.emplace_back(e.getBucketId(), copy.getBucketInfo(), node, idealState.lookup(copy.getNode()), copy.trusted(), true); + _instances.emplace_back(e.getBucketId(), copy.getBucketInfo(), node, idealState.lookup(copy.getNode()), i, copy.trusted(), true); } } @@ -106,7 +109,8 @@ BucketInstanceList::extendToEnoughCopies(const DistributorBucketSpace& distribut for (uint32_t i=0; i<idealNodes.size(); ++i) { lib::Node node(lib::NodeType::STORAGE, idealNodes[i]); if (!contains(node)) { - _instances.emplace_back(newTarget, api::BucketInfo(), node, i, false, false); + // We don't sort `_instances` after extending, so just reuse `i` as dummy DB entry order. + _instances.emplace_back(newTarget, api::BucketInfo(), node, i, i, false, false); } } } @@ -116,7 +120,7 @@ BucketInstanceList::createTargets(document::BucketSpace bucketSpace) { OperationTargetList result; for (const auto& bi : _instances) { - result.emplace_back(document::Bucket(bucketSpace, bi._bucket), bi._node, !bi._exist); + result.emplace_back(document::Bucket(bucketSpace, bi._bucket), bi._node, !bi._exists); } return result; } @@ -129,6 +133,49 @@ BucketInstanceList::print(vespalib::asciistream& out, const PrintProperties& p) namespace { /** + * To maintain a symmetry between which replicas receive Puts and which versions are + * preferred for activation, use an identical ordering predicate for both (for the case + * where replicas are for the same concrete bucket). + * + * Must only be used with BucketInstances that have a distinct _db_entry_order set per instance. + */ +struct ActiveReplicaSymmetricInstanceOrder { + bool operator()(const BucketInstance& a, const BucketInstance& b) noexcept { + if (a._bucket == b._bucket) { + if (a._info.isReady() != b._info.isReady()) { + return a._info.isReady(); + } + if (a._info.getDocumentCount() != b._info.getDocumentCount()) { + return a._info.getDocumentCount() > b._info.getDocumentCount(); + } + if (a._ideal_location_priority != b._ideal_location_priority) { + return a._ideal_location_priority < b._ideal_location_priority; + } + if (a._info.isActive() != b._info.isActive()) { + return a._info.isActive(); + } + // If all else is equal, this implies both A and B are on retired nodes, which is unlikely + // but possible. Fall back to the existing DB _entry order_, which is equal to an ideal + // state order where retired nodes are considered part of the ideal state (which is not the + // case for most ideal state operations). Since the DB entry order is in ideal state order, + // using this instead of node _index_ avoids affinities to lower indexes in such edge cases. + return a._db_entry_order < b._db_entry_order; + } else { + // TODO this inconsistent split case is equal to the legacy logic (aside from the tie-breaking), + // but is considered to be extremely unlikely in practice, so not worth optimizing for. + if ((a._info.getMetaCount() == 0) ^ (b._info.getMetaCount() == 0)) { + return (a._info.getMetaCount() == 0); + } + if (a._bucket.getUsedBits() != b._bucket.getUsedBits()) { + return (a._bucket.getUsedBits() > b._bucket.getUsedBits()); + } + return a._db_entry_order < b._db_entry_order; + } + return false; + } +}; + +/** * - Trusted copies should be preferred over non-trusted copies for the same bucket. * - Buckets in ideal locations should be preferred over non-ideal locations for the * same bucket across several nodes. @@ -137,14 +184,14 @@ namespace { * - Right after split/join, bucket is often not in ideal location, but should be * preferred instead of source anyhow. */ -struct InstanceOrder { - bool operator()(const BucketInstance& a, const BucketInstance& b) { +struct LegacyInstanceOrder { + bool operator()(const BucketInstance& a, const BucketInstance& b) noexcept { if (a._bucket == b._bucket) { - // Trusted only makes sense within same bucket - // Prefer trusted buckets over non-trusted ones. + // Trusted only makes sense within same bucket + // Prefer trusted buckets over non-trusted ones. if (a._trusted != b._trusted) return a._trusted; - if (a._idealLocationPriority != b._idealLocationPriority) { - return a._idealLocationPriority < b._idealLocationPriority; + if (a._ideal_location_priority != b._ideal_location_priority) { + return a._ideal_location_priority < b._ideal_location_priority; } } else { if ((a._info.getMetaCount() == 0) ^ (b._info.getMetaCount() == 0)) { @@ -164,7 +211,11 @@ OperationTargetResolverImpl::getAllInstances(OperationType type, const document: BucketInstanceList instances; if (type == PUT) { instances.populate(id, _distributor_bucket_space, _bucketDatabase); - instances.sort(InstanceOrder()); + if (_symmetric_replica_selection) { + instances.sort(ActiveReplicaSymmetricInstanceOrder()); + } else { + instances.sort(LegacyInstanceOrder()); + } instances.removeNodeDuplicates(); instances.extendToEnoughCopies(_distributor_bucket_space, _bucketDatabase, _bucketDatabase.getAppropriateBucket(_minUsedBucketBits, id), id); diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index 9f367a89cba..6ab38928200 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -15,15 +15,17 @@ struct BucketInstance : public vespalib::AsciiPrintable { document::BucketId _bucket; api::BucketInfo _info; lib::Node _node; - uint16_t _idealLocationPriority; - bool _trusted; - bool _exist; + uint16_t _ideal_location_priority; + uint16_t _db_entry_order; + bool _trusted; // TODO remove + bool _exists; BucketInstance() noexcept - : _idealLocationPriority(0xffff), _trusted(false), _exist(false) {} + : _ideal_location_priority(0xffff), _db_entry_order(0xffff), _trusted(false), _exists(false) + {} BucketInstance(const document::BucketId& id, const api::BucketInfo& info, - lib::Node node, uint16_t idealLocationPriority, bool trusted, - bool exist) noexcept; + lib::Node node, uint16_t ideal_location_priority, + uint16_t db_entry_order, bool trusted, bool exist) noexcept; void print(vespalib::asciistream& out, const PrintProperties&) const override; }; @@ -83,6 +85,7 @@ class OperationTargetResolverImpl : public OperationTargetResolver { uint32_t _minUsedBucketBits; uint16_t _redundancy; document::BucketSpace _bucketSpace; + bool _symmetric_replica_selection; public: OperationTargetResolverImpl(const DistributorBucketSpace& distributor_bucket_space, @@ -94,9 +97,14 @@ public: _bucketDatabase(bucketDatabase), _minUsedBucketBits(minUsedBucketBits), _redundancy(redundancy), - _bucketSpace(bucketSpace) + _bucketSpace(bucketSpace), + _symmetric_replica_selection(true) {} + void use_symmetric_replica_selection(bool symmetry) noexcept { + _symmetric_replica_selection = symmetry; + } + BucketInstanceList getAllInstances(OperationType type, const document::BucketId& id); BucketInstanceList getInstances(OperationType type, const document::BucketId& id) { BucketInstanceList result(getAllInstances(type, id)); diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index b756c2e421b..657dc0eec7b 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -5,7 +5,6 @@ #include "pendingclusterstate.h" #include "top_level_bucket_db_updater.h" #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/xmlstream.hpp> @@ -52,7 +51,7 @@ PendingClusterState::PendingClusterState( _node_features() { logConstructionInformation(); - initializeBucketSpaceTransitions(false, outdatedNodesMap); + initializeBucketSpaceTransitions(false, outdatedNodesMap, false); } PendingClusterState::PendingClusterState( @@ -60,7 +59,8 @@ PendingClusterState::PendingClusterState( const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, const BucketSpaceStateMap& bucket_space_states, - api::Timestamp creationTimestamp) + api::Timestamp creationTimestamp, + bool inhibit_request_sending) : _requestedNodes(clusterInfo->getStorageNodeCount()), _prevClusterStateBundle(clusterInfo->getClusterStateBundle()), _newClusterStateBundle(clusterInfo->getClusterStateBundle()), @@ -76,13 +76,15 @@ PendingClusterState::PendingClusterState( _node_features() { logConstructionInformation(); - initializeBucketSpaceTransitions(true, OutdatedNodesMap()); + initializeBucketSpaceTransitions(true, OutdatedNodesMap(), inhibit_request_sending); } PendingClusterState::~PendingClusterState() = default; void -PendingClusterState::initializeBucketSpaceTransitions(bool distributionChanged, const OutdatedNodesMap& outdatedNodesMap) +PendingClusterState::initializeBucketSpaceTransitions(bool distributionChanged, + const OutdatedNodesMap& outdatedNodesMap, + bool inhibit_request_sending) { OutdatedNodes emptyOutdatedNodes; for (const auto &elem : _bucket_space_states) { @@ -97,7 +99,7 @@ PendingClusterState::initializeBucketSpaceTransitions(bool distributionChanged, } _pendingTransitions.emplace(elem.first, std::move(pendingTransition)); } - if (shouldRequestBucketInfo()) { + if (!inhibit_request_sending && shouldRequestBucketInfo()) { requestNodes(); } } diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.h b/storage/src/vespa/storage/distributor/pendingclusterstate.h index f75ebdc5ba3..366693267c1 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.h +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.h @@ -66,11 +66,12 @@ public: const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, const BucketSpaceStateMap& bucket_space_states, - api::Timestamp creationTimestamp) + api::Timestamp creationTimestamp, + bool inhibit_request_sending) { // Naked new due to private constructor return std::unique_ptr<PendingClusterState>(new PendingClusterState( - clock, clusterInfo, sender, bucket_space_states, creationTimestamp)); + clock, clusterInfo, sender, bucket_space_states, creationTimestamp, inhibit_request_sending)); } PendingClusterState(const PendingClusterState &) = delete; @@ -188,7 +189,8 @@ private: const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, const BucketSpaceStateMap& bucket_space_states, - api::Timestamp creationTimestamp); + api::Timestamp creationTimestamp, + bool inhibit_request_sending); struct BucketSpaceAndNode { document::BucketSpace bucketSpace; @@ -200,7 +202,9 @@ private: } }; - void initializeBucketSpaceTransitions(bool distributionChanged, const OutdatedNodesMap& outdatedNodesMap); + void initializeBucketSpaceTransitions(bool distributionChanged, + const OutdatedNodesMap& outdatedNodesMap, + bool inhibit_request_sending); void logConstructionInformation() const; void requestNode(BucketSpaceAndNode bucketSpaceAndNode); void requestNodes(); diff --git a/storage/src/vespa/storage/distributor/stripe_access_guard.h b/storage/src/vespa/storage/distributor/stripe_access_guard.h index 1618bc9be9d..8a930ed3305 100644 --- a/storage/src/vespa/storage/distributor/stripe_access_guard.h +++ b/storage/src/vespa/storage/distributor/stripe_access_guard.h @@ -1,11 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "bucket_space_distribution_configs.h" #include "pending_bucket_space_db_transition_entry.h" #include "potential_data_loss_report.h" #include "outdated_nodes.h" #include <vespa/document/bucket/bucketspace.h> +#include <vespa/vdslib/distribution/bucket_space_distribution_configs.h> #include <vespa/storageapi/defs.h> #include <unordered_set> // TODO use hash_set instead @@ -38,7 +38,7 @@ public: virtual void update_total_distributor_config(std::shared_ptr<const DistributorConfiguration> config) = 0; - virtual void update_distribution_config(const BucketSpaceDistributionConfigs& new_configs) = 0; + virtual void update_distribution_config(const lib::BucketSpaceDistributionConfigs& new_configs) = 0; virtual void set_pending_cluster_state_bundle(const lib::ClusterStateBundle& pending_state) = 0; virtual void clear_pending_cluster_state_bundle() = 0; virtual void enable_cluster_state_bundle(const lib::ClusterStateBundle& new_state, diff --git a/storage/src/vespa/storage/distributor/tickable_stripe.h b/storage/src/vespa/storage/distributor/tickable_stripe.h index ab1cd570089..2605c24639e 100644 --- a/storage/src/vespa/storage/distributor/tickable_stripe.h +++ b/storage/src/vespa/storage/distributor/tickable_stripe.h @@ -39,7 +39,7 @@ public: virtual void update_total_distributor_config(std::shared_ptr<const DistributorConfiguration> config) = 0; - virtual void update_distribution_config(const BucketSpaceDistributionConfigs& new_configs) = 0; + virtual void update_distribution_config(const lib::BucketSpaceDistributionConfigs& new_configs) = 0; virtual void set_pending_cluster_state_bundle(const lib::ClusterStateBundle& pending_state) = 0; virtual void clear_pending_cluster_state_bundle() = 0; virtual void enable_cluster_state_bundle(const lib::ClusterStateBundle& new_state, diff --git a/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.cpp b/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.cpp index 1f7d11362e2..30306784c20 100644 --- a/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.cpp +++ b/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.cpp @@ -2,7 +2,6 @@ #include "top_level_bucket_db_updater.h" #include "bucket_db_prune_elision.h" -#include "bucket_space_distribution_configs.h" #include "bucket_space_distribution_context.h" #include "top_level_distributor.h" #include "distributor_bucket_space.h" @@ -11,11 +10,12 @@ #include "simpleclusterinformation.h" #include "stripe_access_guard.h" #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storage/config/distributorconfiguration.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> +#include <vespa/vdslib/distribution/bucket_space_distribution_configs.h> #include <vespa/vdslib/distribution/distribution.h> +#include <vespa/vdslib/distribution/global_bucket_space_distribution_converter.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/util/xmlstream.h> #include <thread> @@ -54,6 +54,7 @@ TopLevelBucketDBUpdater::TopLevelBucketDBUpdater(const DistributorNodeContext& n { // FIXME STRIPE top-level Distributor needs a proper way to track the current cluster state bundle! propagate_active_state_bundle_internally(true); // We're just starting up so assume ownership transfer. + // TODO bootstrap cluster state bundle instead? version:0 cluster:d bootstrap_distribution_config(std::move(bootstrap_distribution)); } @@ -71,7 +72,7 @@ TopLevelBucketDBUpdater::propagate_active_state_bundle_internally(bool has_bucke void TopLevelBucketDBUpdater::bootstrap_distribution_config(std::shared_ptr<const lib::Distribution> distribution) { - auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); + auto global_distr = lib::GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); _op_ctx.bucket_space_states().get(document::FixedBucketSpaces::default_space()).set_distribution(distribution); _op_ctx.bucket_space_states().get(document::FixedBucketSpaces::global_space()).set_distribution(global_distr); // TODO STRIPE do we need to bootstrap the stripes as well here? Or do they do this on their own volition? @@ -79,7 +80,7 @@ TopLevelBucketDBUpdater::bootstrap_distribution_config(std::shared_ptr<const lib } void -TopLevelBucketDBUpdater::propagate_distribution_config(const BucketSpaceDistributionConfigs& configs) { +TopLevelBucketDBUpdater::propagate_distribution_config(const lib::BucketSpaceDistributionConfigs& configs) { if (auto distr = configs.get_or_nullptr(document::FixedBucketSpaces::default_space())) { _op_ctx.bucket_space_states().get(document::FixedBucketSpaces::default_space()).set_distribution(distr); } @@ -183,29 +184,43 @@ TopLevelBucketDBUpdater::complete_transition_timer() } void -TopLevelBucketDBUpdater::storage_distribution_changed(const BucketSpaceDistributionConfigs& configs) +TopLevelBucketDBUpdater::storage_distribution_changed(const lib::BucketSpaceDistributionConfigs& configs) +{ + auto guard = _stripe_accessor.rendezvous_and_hold_all(); + storage_distribution_changed_impl(*guard, configs, false); +} + +void +TopLevelBucketDBUpdater::storage_distribution_changed_impl(StripeAccessGuard& guard, + const lib::BucketSpaceDistributionConfigs& configs, + bool inhibit_request_sending) { propagate_distribution_config(configs); ensure_transition_timer_started(); - auto guard = _stripe_accessor.rendezvous_and_hold_all(); - // FIXME STRIPE might this cause a mismatch with the component stuff's own distribution config..?! - guard->update_distribution_config(configs); - remove_superfluous_buckets(*guard, _active_state_bundle, true); + // TODO should be part of bundle only...!! + guard.update_distribution_config(configs); + remove_superfluous_buckets(guard, _active_state_bundle, true); auto clusterInfo = std::make_shared<const SimpleClusterInformation>( _node_ctx.node_index(), _active_state_bundle, storage_node_up_states()); + // If distribution has changed as part of a SetSystemState we do not send bucket info + // requests, but we do all the other work to enumerate the set of nodes that we _would_ + // have sent to (_outdated_nodes_map), which is then reused when we immediately after + // process the cluster state itself. The cluster state might not have any changes in and + // by itself, but the outdated nodes map ensures that we send necessary requests anyway. _pending_cluster_state = PendingClusterState::createForDistributionChange( _node_ctx.clock(), std::move(clusterInfo), _sender, _op_ctx.bucket_space_states(), - _op_ctx.generate_unique_timestamp()); + _op_ctx.generate_unique_timestamp(), + inhibit_request_sending); _outdated_nodes_map = _pending_cluster_state->getOutdatedNodesMap(); - guard->set_pending_cluster_state_bundle(_pending_cluster_state->getNewClusterStateBundle()); + guard.set_pending_cluster_state_bundle(_pending_cluster_state->getNewClusterStateBundle()); } void @@ -236,7 +251,7 @@ TopLevelBucketDBUpdater::onSetSystemState( const lib::ClusterStateBundle& state = cmd->getClusterStateBundle(); - if (state == _active_state_bundle) { + if (state == _active_state_bundle) { // Also considers distribution configs, if present return false; } ensure_transition_timer_started(); @@ -244,8 +259,22 @@ TopLevelBucketDBUpdater::onSetSystemState( framework::MilliSecTimer process_timer(_node_ctx.clock()); auto guard = _stripe_accessor.rendezvous_and_hold_all(); - guard->update_read_snapshot_before_db_pruning(); const auto& bundle = cmd->getClusterStateBundle(); + if (bundle.has_distribution_config()) { + const auto distr_bundle = bundle.distribution_config_bundle(); + if (_distributor_interface.receive_distribution_from_cluster_controller(distr_bundle->default_distribution_sp())) { + // This stages the distribution config change internally and does everything _except_ + // actually send bucket info requests to the content nodes, as they would be inherently + // doomed due to having a stale cluster state version. Instead, we remember the set of + // affected nodes and reuse this below when we send the properly versioned info requests. + storage_distribution_changed_impl(*guard, distr_bundle->bucket_space_distributions(), true); + } + } else if (_distributor_interface.cluster_controller_is_distribution_source_of_truth()) { + // Cluster controller has stopped sending config (possibly rolled back or reconfigured), so + // we have to follow suit and go back to listening to internal config changes instead. + _distributor_interface.revert_distribution_source_of_truth_to_node_internal_config(); + } + guard->update_read_snapshot_before_db_pruning(); remove_superfluous_buckets(*guard, bundle, false); guard->update_read_snapshot_after_db_pruning(bundle); reply_to_previous_pending_cluster_state_if_any(); @@ -261,7 +290,7 @@ TopLevelBucketDBUpdater::onSetSystemState( _op_ctx.bucket_space_states(), cmd, _outdated_nodes_map, - _op_ctx.generate_unique_timestamp()); // FIXME STRIPE must be atomic across all threads + _op_ctx.generate_unique_timestamp()); _outdated_nodes_map = _pending_cluster_state->getOutdatedNodesMap(); _distributor_interface.metrics().set_cluster_state_processing_time.addValue( diff --git a/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.h b/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.h index e76456329d4..ef3838f5785 100644 --- a/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.h +++ b/storage/src/vespa/storage/distributor/top_level_bucket_db_updater.h @@ -23,9 +23,12 @@ class XmlOutputStream; class XmlAttribute; } +namespace storage::lib { +struct BucketSpaceDistributionConfigs; +} + namespace storage::distributor { -struct BucketSpaceDistributionConfigs; class BucketSpaceDistributionContext; class ClusterStateBundleActivationListener; class DistributorInterface; @@ -57,9 +60,9 @@ public: bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; void resend_delayed_messages(); - void storage_distribution_changed(const BucketSpaceDistributionConfigs& configs); + void storage_distribution_changed(const lib::BucketSpaceDistributionConfigs& configs); void bootstrap_distribution_config(std::shared_ptr<const lib::Distribution>); - void propagate_distribution_config(const BucketSpaceDistributionConfigs& configs); + void propagate_distribution_config(const lib::BucketSpaceDistributionConfigs& configs); vespalib::string report_xml_status(vespalib::xml::XmlOutputStream& xos, const framework::HttpUrlPath&) const; @@ -91,6 +94,10 @@ private: void ensure_transition_timer_started(); void complete_transition_timer(); + void storage_distribution_changed_impl(StripeAccessGuard& guard, + const lib::BucketSpaceDistributionConfigs& configs, + bool inhibit_request_sending); + void remove_superfluous_buckets(StripeAccessGuard& guard, const lib::ClusterStateBundle& new_state, bool is_distribution_config_change); diff --git a/storage/src/vespa/storage/distributor/top_level_distributor.cpp b/storage/src/vespa/storage/distributor/top_level_distributor.cpp index 7348dbd6409..738dfb2c00e 100644 --- a/storage/src/vespa/storage/distributor/top_level_distributor.cpp +++ b/storage/src/vespa/storage/distributor/top_level_distributor.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // #include "blockingoperationstarter.h" -#include "bucket_space_distribution_configs.h" #include "top_level_bucket_db_updater.h" #include "top_level_distributor.h" #include "distributor_bucket_space.h" @@ -16,7 +15,6 @@ #include "throttlingoperationstarter.h" #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/storage/common/bucket_stripe_utils.h> -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storage/common/node_identity.h> #include <vespa/storage/common/nodestateupdater.h> @@ -25,7 +23,9 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/storageframework/generic/status/xmlstatusreporter.h> +#include <vespa/vdslib/distribution/bucket_space_distribution_configs.h> #include <vespa/vdslib/distribution/distribution.h> +#include <vespa/vdslib/distribution/global_bucket_space_distribution_converter.h> #include <vespa/vespalib/util/memoryusage.h> #include <algorithm> @@ -50,6 +50,7 @@ TopLevelDistributor::TopLevelDistributor(DistributorComponentRegister& compReg, _comp_reg(compReg), _done_init_handler(done_init_handler), _done_initializing(false), + _cc_is_distribution_source_of_truth(false), _total_metrics(std::make_shared<DistributorTotalMetrics>(num_distributor_stripes)), _ideal_state_total_metrics(std::make_shared<IdealStateTotalMetrics>(num_distributor_stripes)), _messageSender(messageSender), @@ -306,6 +307,12 @@ void TopLevelDistributor::storageDistributionChanged() { std::lock_guard guard(_distribution_mutex); + if (_cc_is_distribution_source_of_truth) { + LOG(debug, "Received changed distribution config %s, but ignoring it since we have received " + "at least one config injection from the cluster controller", + _component.getDistribution()->toString().c_str()); + return; + } if (!_distribution || (*_component.getDistribution() != *_distribution)) { LOG(debug, "Distribution changed to %s, must re-fetch bucket information", _component.getDistribution()->toString().c_str()); @@ -323,18 +330,53 @@ TopLevelDistributor::enable_next_distribution_if_changed() if (_next_distribution) { _distribution = _next_distribution; _next_distribution = std::shared_ptr<lib::Distribution>(); - auto new_configs = BucketSpaceDistributionConfigs::from_default_distribution(_distribution); + auto new_configs = lib::BucketSpaceDistributionConfigs::from_default_distribution(_distribution); _bucket_db_updater->storage_distribution_changed(new_configs); // Transitively updates all stripes' configs } } +bool +TopLevelDistributor::receive_distribution_from_cluster_controller(std::shared_ptr<const lib::Distribution> distribution) +{ + std::lock_guard guard(_distribution_mutex); + LOG(debug, "Received distribution config '%s' from the cluster controller. Any subsequent " + "distribution configs that do NOT originate from the cluster controller will be ignored.", + distribution->toString().c_str()); + // Signal that from now on we should explicitly ignore distribution config that is not received + // from the cluster controller. Otherwise, we'd introduce at least as many race conditions as + // we'd attempt to eliminate in the first place. + _cc_is_distribution_source_of_truth = true; + const bool changed = !_distribution || (*distribution != *_distribution); + _distribution = std::move(distribution); + return changed; +} + +bool +TopLevelDistributor::cluster_controller_is_distribution_source_of_truth() const noexcept +{ + std::lock_guard guard(_distribution_mutex); + return _cc_is_distribution_source_of_truth; +} + +void +TopLevelDistributor::revert_distribution_source_of_truth_to_node_internal_config() +{ + { + std::lock_guard guard(_distribution_mutex); + LOG(debug, "Reverting to use node-internal config as distribution config source of truth"); + _cc_is_distribution_source_of_truth = false; + } + // Re-sync internal distribution config with that received from the config server + storageDistributionChanged(); +} + void TopLevelDistributor::propagate_default_distribution_thread_unsafe( std::shared_ptr<const lib::Distribution> distribution) { // Should only be called at ctor time, at which point the pool is not yet running. assert(_stripe_pool.stripe_count() == 0); - auto new_configs = BucketSpaceDistributionConfigs::from_default_distribution(std::move(distribution)); + auto new_configs = lib::BucketSpaceDistributionConfigs::from_default_distribution(std::move(distribution)); for (auto& stripe : _stripes) { stripe->update_distribution_config(new_configs); } diff --git a/storage/src/vespa/storage/distributor/top_level_distributor.h b/storage/src/vespa/storage/distributor/top_level_distributor.h index d526c52ce8e..2c9d5057339 100644 --- a/storage/src/vespa/storage/distributor/top_level_distributor.h +++ b/storage/src/vespa/storage/distributor/top_level_distributor.h @@ -99,8 +99,13 @@ public: int getDistributorIndex() const override { return _component.node_index(); } const ClusterContext& cluster_context() const override { return _component.cluster_context(); } + // Called from top-level storage component when config is updated from config server void storageDistributionChanged() override; + bool receive_distribution_from_cluster_controller(std::shared_ptr<const lib::Distribution> distribution) override; + bool cluster_controller_is_distribution_source_of_truth() const noexcept override; + void revert_distribution_source_of_truth_to_node_internal_config() override; + // StatusReporter implementation vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; @@ -185,6 +190,7 @@ private: DistributorComponentRegister& _comp_reg; DoneInitializeHandler& _done_init_handler; bool _done_initializing; + bool _cc_is_distribution_source_of_truth; std::shared_ptr<DistributorTotalMetrics> _total_metrics; std::shared_ptr<IdealStateTotalMetrics> _ideal_state_total_metrics; ChainedMessageSender* _messageSender; @@ -218,8 +224,8 @@ private: DistributorHostInfoReporter _hostInfoReporter; mutable std::mutex _distribution_mutex; - std::shared_ptr<lib::Distribution> _distribution; - std::shared_ptr<lib::Distribution> _next_distribution; + std::shared_ptr<const lib::Distribution> _distribution; + std::shared_ptr<const lib::Distribution> _next_distribution; uint64_t _current_internal_config_generation; }; diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp index 94853ec18d1..ab1cbf0b4d7 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp @@ -3,7 +3,7 @@ #include "servicelayercomponentregisterimpl.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> +#include <vespa/vdslib/distribution/global_bucket_space_distribution_converter.h> namespace storage { @@ -26,7 +26,7 @@ void ServiceLayerComponentRegisterImpl::setDistribution(std::shared_ptr<lib::Distribution> distribution) { _bucketSpaceRepo.get(document::FixedBucketSpaces::default_space()).setDistribution(distribution); - auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); + auto global_distr = lib::GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); _bucketSpaceRepo.get(document::FixedBucketSpaces::global_space()).setDistribution(global_distr); StorageComponentRegisterImpl::setDistribution(distribution); } diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp index ee49b346d92..d5de11c7d6f 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp @@ -19,6 +19,7 @@ uint16_t BucketOwnershipNotifier::getOwnerDistributorForBucket(const document::Bucket &bucket) const { try { + // TODO use state updater bundle for everything? auto distribution(_component.getBucketSpaceRepo().get(bucket.getBucketSpace()).getDistribution()); const auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); const auto &clusterState = *clusterStateBundle->getDerivedClusterState(bucket.getBucketSpace()); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 23de39f7130..495497d507d 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -930,8 +930,9 @@ FileStorManager::updateState() for (const auto &elem : _component.getBucketSpaceRepo()) { BucketSpace bucketSpace(elem.first); ContentBucketSpace& contentBucketSpace = *elem.second; - auto derivedClusterState = contentBucketSpace.getClusterState(); - const bool node_up_in_space = derivedClusterState->getNodeState(node).getState().oneOf("uir"); + auto state_and_distr = contentBucketSpace.state_and_distribution(); + assert(state_and_distr->valid()); + const bool node_up_in_space = state_and_distr->cluster_state().getNodeState(node).getState().oneOf("uir"); if (should_deactivate_buckets(contentBucketSpace, node_up_in_space, in_maintenance)) { LOG(debug, "Received cluster state where this node is down; de-activating all buckets " "in database for bucket space %s", bucketSpace.toString().c_str()); @@ -941,9 +942,8 @@ FileStorManager::updateState() } contentBucketSpace.setNodeUpInLastNodeStateSeenByProvider(node_up_in_space); contentBucketSpace.setNodeMaintenanceInLastNodeStateSeenByProvider(in_maintenance); - spi::ClusterState spiState(*derivedClusterState, _component.getIndex(), - *contentBucketSpace.getDistribution(), - in_maintenance); + spi::ClusterState spiState(state_and_distr->cluster_state(), _component.getIndex(), + state_and_distr->distribution(), in_maintenance); _provider->setClusterState(bucketSpace, spiState); } } @@ -959,6 +959,7 @@ FileStorManager::propagateClusterStates() { auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); for (const auto &elem : _component.getBucketSpaceRepo()) { + // TODO also distribution! bundle and repo must be 1-1 elem.second->setClusterState(clusterStateBundle->getDerivedClusterState(elem.first)); } } diff --git a/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.cpp b/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.cpp index c3cb38bd7ac..582c69e943f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.cpp @@ -20,8 +20,8 @@ MergeHandlerMetrics::MergeHandlerMetrics(metrics::MetricSet* owner) "current node.", owner), mergeAverageDataReceivedNeeded("mergeavgdatareceivedneeded", {}, "Amount of data transferred from previous node " "in chain that we needed to apply locally.", owner), - put_latency("put_latency", {}, "Latency of individual puts that are part of merge operations", owner), - remove_latency("remove_latency", {}, "Latency of individual removes that are part of merge operations", owner) + merge_put_latency("merge_put_latency", {}, "Latency of individual puts that are part of merge operations", owner), + merge_remove_latency("merge_remove_latency", {}, "Latency of individual removes that are part of merge operations", owner) {} MergeHandlerMetrics::~MergeHandlerMetrics() = default; diff --git a/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.h b/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.h index 44b85570357..a2d68011695 100644 --- a/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.h @@ -21,8 +21,8 @@ struct MergeHandlerMetrics { metrics::DoubleAverageMetric mergeAverageDataReceivedNeeded; // Individual operation metrics. These capture both count and latency sum, so // no need for explicit count metric on the side. - metrics::DoubleAverageMetric put_latency; - metrics::DoubleAverageMetric remove_latency; + metrics::DoubleAverageMetric merge_put_latency; + metrics::DoubleAverageMetric merge_remove_latency; // Iteration over metadata and document payload data is already covered by // the merge[Meta]Data(Read|Write)Latency metrics, so not repeated here. Can be // explicitly added if deemed required. diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 7ee2d9f37bf..b3207428f5f 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -4,13 +4,14 @@ #include "persistenceutil.h" #include "apply_bucket_diff_entry_complete.h" #include "apply_bucket_diff_state.h" -#include <vespa/storage/persistence/filestorage/mergestatus.h> -#include <vespa/persistence/spi/persistenceprovider.h> -#include <vespa/persistence/spi/docentry.h> -#include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/persistence/spi/docentry.h> +#include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/storage/persistence/filestorage/mergestatus.h> +#include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <algorithm> @@ -506,8 +507,18 @@ void MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results, const spi::Bucket& bucket, const api::ApplyBucketDiffCommand::Entry& e, - const document::DocumentTypeRepo& repo) const + const document::DocumentTypeRepo& repo, + const NewestDocumentVersionMapping& newest_per_doc) const { + if (!e._docName.empty()) { + auto version_iter = newest_per_doc.find(e._docName); + assert(version_iter != newest_per_doc.end()); + if (e._entry._timestamp != version_iter->second) { + LOG(spam, "ApplyBucketDiff(%s): skipping diff entry %s since it is subsumed by a newer timestamp %" PRIu64, + bucket.toString().c_str(), e.toString().c_str(), version_iter->second); + return; + } + } auto throttle_token = _env._fileStorHandler.operation_throttler().blocking_acquire_one(); spi::Timestamp timestamp(e._entry._timestamp); if (!(e._entry._flags & (DELETED | DELETED_IN_PLACE))) { @@ -516,14 +527,14 @@ MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results document::DocumentId docId = doc->getId(); auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), std::move(docId), std::move(throttle_token), "put", - _clock, _env._metrics.merge_handler_metrics.put_latency); + _clock, _env._metrics.merge_handler_metrics.merge_put_latency); _spi.putAsync(bucket, timestamp, std::move(doc), std::move(complete)); } else { std::vector<spi::IdAndTimestamp> ids; ids.emplace_back(document::DocumentId(e._docName), timestamp); auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].id, std::move(throttle_token), "remove", - _clock, _env._metrics.merge_handler_metrics.remove_latency); + _clock, _env._metrics.merge_handler_metrics.merge_remove_latency); _spi.removeAsync(bucket, std::move(ids), std::move(complete)); } } @@ -548,6 +559,7 @@ MergeHandler::applyDiffLocally(const spi::Bucket& bucket, std::vector<api::Apply DocEntryList entries; populateMetaData(bucket, Timestamp::max(), entries, context); + const auto newest_versions = enumerate_newest_document_versions(diff); const document::DocumentTypeRepo & repo = _env.getDocumentTypeRepo(); uint32_t existingCount = entries.size(); @@ -580,7 +592,7 @@ MergeHandler::applyDiffLocally(const spi::Bucket& bucket, std::vector<api::Apply ++i; LOG(spam, "ApplyBucketDiff(%s): Adding slot %s", bucket.toString().c_str(), e.toString().c_str()); - applyDiffEntry(async_results, bucket, e, repo); + applyDiffEntry(async_results, bucket, e, repo, newest_versions); } else { assert(spi::Timestamp(e._entry._timestamp) == existing.getTimestamp()); // Diffing for existing timestamp; should either both be put @@ -591,7 +603,7 @@ MergeHandler::applyDiffLocally(const spi::Bucket& bucket, std::vector<api::Apply if ((e._entry._flags & DELETED) && !existing.isRemove()) { LOG(debug, "Slot in diff is remove for existing timestamp in %s. Diff slot: %s. Existing slot: %s", bucket.toString().c_str(), e.toString().c_str(), existing.toString().c_str()); - applyDiffEntry(async_results, bucket, e, repo); + applyDiffEntry(async_results, bucket, e, repo, newest_versions); } else { // Duplicate put, just ignore it. LOG(debug, "During diff apply, attempting to add slot whose timestamp already exists in %s, " @@ -619,7 +631,7 @@ MergeHandler::applyDiffLocally(const spi::Bucket& bucket, std::vector<api::Apply LOG(spam, "ApplyBucketDiff(%s): Adding slot %s", bucket.toString().c_str(), e.toString().c_str()); - applyDiffEntry(async_results, bucket, e, repo); + applyDiffEntry(async_results, bucket, e, repo, newest_versions); byteCount += e._headerBlob.size() + e._bodyBlob.size(); } if (byteCount + notNeededByteCount != 0) { @@ -631,6 +643,27 @@ MergeHandler::applyDiffLocally(const spi::Bucket& bucket, std::vector<api::Apply bucket.toString().c_str(), addedCount); } +MergeHandler::NewestDocumentVersionMapping +MergeHandler::enumerate_newest_document_versions(const std::vector<api::ApplyBucketDiffCommand::Entry>& diff) +{ + NewestDocumentVersionMapping newest_per_doc; + for (const auto& e : diff) { + // We expect the doc name to always be filled out, both for remove operations and for puts. + // But since the latter is technically redundant (ID is also found within the document), we + // guard on this to be forwards compatible in case this changes (e.g. to populate and use + // the GID field instead). Fallback to the legacy behavior if so. + if (e._docName.empty()) { + continue; + } + auto [existing_iter, inserted] = newest_per_doc.insert(std::make_pair(vespalib::stringref(e._docName), e._entry._timestamp)); + if (!inserted) { + assert(existing_iter != newest_per_doc.end()); + existing_iter->second = std::max(existing_iter->second, e._entry._timestamp); + } + } + return newest_per_doc; +} + void MergeHandler::sync_bucket_info(const spi::Bucket& bucket) const { diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index f3bef802229..2be45e7bc8b 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -18,6 +18,7 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/storage/common/cluster_context.h> #include <vespa/storage/common/messagesender.h> +#include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/storageframework/generic/clock/time.h> @@ -42,6 +43,8 @@ private: using MessageTrackerUP = std::unique_ptr<MessageTracker>; using Timestamp = framework::MicroSecTime; public: + using NewestDocumentVersionMapping = vespalib::hash_map<vespalib::stringref, api::Timestamp>; + enum StateFlag { IN_USE = 0x01, DELETED = 0x02, @@ -72,6 +75,17 @@ public: void handleApplyBucketDiffReply(api::ApplyBucketDiffReply&, MessageSender&, MessageTrackerUP) const; void drain_async_writes(); + /** + * Returns a mapping that, for each document ID, contains the newest version of that document that + * is present in the diff. + * + * The returned hash_map keys point directly into the `ApplyBucketDiffCommand::Entry::_docName` memory + * owned by `diff`, so this memory must remain unchanged and stable for the duration of the returned + * mapping's lifetime. + */ + static NewestDocumentVersionMapping enumerate_newest_document_versions( + const std::vector<api::ApplyBucketDiffCommand::Entry>& diff); + private: using DocEntryList = std::vector<std::unique_ptr<spi::DocEntry>>; const framework::Clock &_clock; @@ -90,9 +104,13 @@ private: /** * Invoke either put, remove or unrevertable remove on the SPI * depending on the flags in the diff entry. + * + * If `newest_doc_version` indicates that the entry is not the newest version present in the + * diff, the entry is silently ignored and is _not_ invoked on the SPI. */ void applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results, const spi::Bucket&, - const api::ApplyBucketDiffCommand::Entry&, const document::DocumentTypeRepo& repo) const; + const api::ApplyBucketDiffCommand::Entry&, const document::DocumentTypeRepo& repo, + const NewestDocumentVersionMapping& newest_per_doc) const; /** * Fill entries-vector with metadata for bucket up to maxTimestamp, diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index bbd4e87cb40..1b119a0e631 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -622,7 +622,14 @@ CommunicationManager::sendDirectRPCReply(RPCRequestWrapper& request, request.addReturnString(ns.str().c_str()); LOGBP(debug, "Sending getnodestate2 reply with no host info."); } else if (requestName == "setsystemstate2" || requestName == "setdistributionstates") { - // No data to return + // No data to return, but the request must be failed iff we rejected the state version + // due to a higher version having been previously received. + auto& state_reply = dynamic_cast<api::SetSystemStateReply&>(*reply); + if (state_reply.getResult().getResult() == api::ReturnCode::REJECTED) { + vespalib::string err_msg = state_reply.getResult().getMessage(); // ReturnCode message is stringref + request.returnError(FRTE_RPC_METHOD_FAILED, err_msg.c_str()); + return; + } } else if (requestName == "activate_cluster_state_version") { auto& activate_reply(dynamic_cast<api::ActivateClusterStateVersionReply&>(*reply)); request.addReturnInt(activate_reply.actualVersion()); diff --git a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp index 4d74eb1974b..3b53c0c9584 100644 --- a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp @@ -124,7 +124,8 @@ void ClusterControllerApiRpcService::RPC_setSystemState2(FRT_RPCRequest* req) { req->GetParams()->GetValue(0)._string._len); lib::ClusterState systemState(systemStateStr); - auto cmd = std::make_shared<api::SetSystemStateCommand>(lib::ClusterStateBundle(systemState)); + auto bundle = std::make_shared<const lib::ClusterStateBundle>(systemState); + auto cmd = std::make_shared<api::SetSystemStateCommand>(std::move(bundle)); cmd->setPriority(api::StorageMessage::VERYHIGH); detach_and_forward_to_enqueuer(std::move(cmd), req); @@ -167,8 +168,7 @@ void ClusterControllerApiRpcService::RPC_setDistributionStates(FRT_RPCRequest* r } LOG(debug, "Got state bundle %s", state_bundle->toString().c_str()); - // TODO add constructor taking in shared_ptr directly instead? - auto cmd = std::make_shared<api::SetSystemStateCommand>(*state_bundle); + auto cmd = std::make_shared<api::SetSystemStateCommand>(std::move(state_bundle)); cmd->setPriority(api::StorageMessage::VERYHIGH); detach_and_forward_to_enqueuer(std::move(cmd), req); diff --git a/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h b/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h index 81bd0897dbb..53b613ce2ab 100644 --- a/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h @@ -21,8 +21,8 @@ class ClusterStateBundleCodec { public: virtual ~ClusterStateBundleCodec() = default; - virtual EncodedClusterStateBundle encode(const lib::ClusterStateBundle&) const = 0; - virtual std::shared_ptr<const lib::ClusterStateBundle> decode(const EncodedClusterStateBundle&) const = 0; + [[nodiscard]] virtual EncodedClusterStateBundle encode(const lib::ClusterStateBundle&) const = 0; + [[nodiscard]] virtual std::shared_ptr<const lib::ClusterStateBundle> decode(const EncodedClusterStateBundle&) const = 0; }; } diff --git a/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp index 38d3f929549..f143066ac34 100644 --- a/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp @@ -1,9 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "slime_cluster_state_bundle_codec.h" +#include <vespa/config-stor-distribution.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/configgen/configpayload.h> +#include <vespa/config/print/configdatabuffer.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/vdslib/state/clusterstate.h> #include <vespa/vdslib/state/cluster_state_bundle.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/data/slime/array_traverser.h> +#include <vespa/vespalib/data/slime/object_traverser.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> @@ -17,6 +23,7 @@ using vespalib::compression::CompressionConfig; using vespalib::compression::decompress; using vespalib::compression::compress; using vespalib::Memory; +using DistributionConfigBuilder = storage::lib::Distribution::DistributionConfigBuilder; using namespace vespalib::slime; namespace storage::rpc { @@ -48,21 +55,94 @@ vespalib::string serialize_state(const lib::ClusterState& state) { return as.str(); } -const Memory StatesField("states"); const Memory BaselineField("baseline"); -const Memory SpacesField("spaces"); -const Memory DeferredActivationField("deferred-activation"); -const Memory FeedBlockField("feed-block"); const Memory BlockFeedInClusterField("block-feed-in-cluster"); +const Memory DeferredActivationField("deferred-activation"); const Memory DescriptionField("description"); +const Memory DistributionConfigField("distribution-config"); +const Memory FeedBlockField("feed-block"); +const Memory SpacesField("spaces"); +const Memory StatesField("states"); + +// Important: these conversion routines are NOT complete and NOT general! They are only to be used +// by code transitively used by unit tests that expect a particular type subset and "shape" of config. + +void convert_struct(const Inspector& in, Cursor& out); + +struct ConfigArrayConverter : ArrayTraverser { + Cursor& _out; + explicit ConfigArrayConverter(Cursor& out) noexcept: _out(out) {} + + void entry([[maybe_unused]] size_t idx, const Inspector& in) override { + assert(in.type().getId() == OBJECT::ID); + auto type = in["type"].asString(); + auto& value = in["value"]; + assert(value.valid()); + if (type == "int") { + _out.addLong(value.asLong()); + } else if (type == "bool") { + _out.addBool(value.asBool()); + } else if (type == "string") { + _out.addString(value.asString()); + } else if (type == "double") { + _out.addDouble(value.asDouble()); + } else if (type == "array") { + assert(value.type().getId() == ARRAY::ID); + ConfigArrayConverter arr_conv(_out.addArray()); + value.traverse(arr_conv); + } else if (type == "struct") { + convert_struct(value, _out.addObject()); + } else { + fprintf(stderr, "unknown array entry type '%s'\n", type.make_string().c_str()); + abort(); + } + } +}; + +struct ConfigObjectConverter : ObjectTraverser { + Cursor& _out; + explicit ConfigObjectConverter(Cursor& out) noexcept: _out(out) {} + + void field(const Memory& name, const Inspector& in) override { + assert(in.type().getId() == OBJECT::ID); + auto type = in["type"].asString(); + auto& value = in["value"]; + assert(value.valid()); + if (type == "int") { + _out.setLong(name, value.asLong()); + } else if (type == "bool") { + _out.setBool(name, value.asBool()); + } else if (type == "string") { + _out.setString(name, value.asString()); + } else if (type == "double") { + _out.setDouble(name, value.asDouble()); + } else if (type == "array") { + assert(value.type().getId() == ARRAY::ID); + ConfigArrayConverter arr_conv(_out.setArray(name)); + value.traverse(arr_conv); + } else if (type == "struct") { + convert_struct(value, _out.setObject(name)); + } else { + fprintf(stderr, "unknown struct entry type '%s'\n", type.make_string().c_str()); + abort(); + } + } +}; + +void convert_struct(const Inspector& in, Cursor& out) { + ConfigObjectConverter conv(out); + in.traverse(conv); +} +void convert_to_config_payload(const Inspector& in, Cursor& out) { + convert_struct(in["configPayload"], out); } +} // anon ns + // Only used from unit tests; the cluster controller encodes all bundles // we decode in practice. -EncodedClusterStateBundle SlimeClusterStateBundleCodec::encode( - const lib::ClusterStateBundle& bundle) const -{ +EncodedClusterStateBundle SlimeClusterStateBundleCodec::encode(const lib::ClusterStateBundle& bundle) const { vespalib::Slime slime; Cursor& root = slime.setObject(); if (bundle.deferredActivation()) { @@ -81,6 +161,15 @@ EncodedClusterStateBundle SlimeClusterStateBundleCodec::encode( feed_block.setString(DescriptionField, bundle.feed_block()->description()); } + if (bundle.has_distribution_config()) { + Cursor& distr_root = root.setObject(DistributionConfigField); + ::config::ConfigDataBuffer buf; + bundle.distribution_config_bundle()->config().serialize(buf); + // There is no way in C++ to directly serialize to the actual payload format we expect to + // deserialize, so we have to manually convert the type-annotated config snapshot :I + convert_to_config_payload(buf.slimeObject().get(), distr_root); + } + OutputBuf out_buf(4_Ki); BinaryFormat::encode(slime, out_buf); ConstBufferRef to_compress(out_buf.getBuf().getData(), out_buf.getBuf().getDataLen()); @@ -130,7 +219,7 @@ std::shared_ptr<const lib::ClusterStateBundle> SlimeClusterStateBundleCodec::dec BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), slime); Inspector& root = slime.get(); Inspector& states = root[StatesField]; - lib::ClusterState baseline(states[BaselineField].asString().make_string()); + auto baseline = std::make_shared<lib::ClusterState>(states[BaselineField].asString().make_string()); Inspector& spaces = states[SpacesField]; lib::ClusterStateBundle::BucketSpaceStateMapping space_states; @@ -138,16 +227,21 @@ std::shared_ptr<const lib::ClusterStateBundle> SlimeClusterStateBundleCodec::dec spaces.traverse(inserter); const bool deferred_activation = root[DeferredActivationField].asBool(); // Defaults to false if not set. + std::shared_ptr<const lib::DistributionConfigBundle> distribution_config; + std::optional<lib::ClusterStateBundle::FeedBlock> feed_block; Inspector& fb = root[FeedBlockField]; if (fb.valid()) { - lib::ClusterStateBundle::FeedBlock feed_block(fb[BlockFeedInClusterField].asBool(), - fb[DescriptionField].asString().make_string()); - return std::make_shared<lib::ClusterStateBundle>(baseline, std::move(space_states), feed_block, deferred_activation); + feed_block = lib::ClusterStateBundle::FeedBlock(fb[BlockFeedInClusterField].asBool(), + fb[DescriptionField].asString().make_string()); } - - // TODO add shared_ptr constructor for baseline? - return std::make_shared<lib::ClusterStateBundle>(baseline, std::move(space_states), deferred_activation); + Inspector& dc = root[DistributionConfigField]; + if (dc.valid()) { + auto raw_cfg = std::make_unique<DistributionConfigBuilder>(::config::ConfigPayload(dc)); + distribution_config = lib::DistributionConfigBundle::of(std::move(raw_cfg)); + } + return std::make_shared<lib::ClusterStateBundle>(std::move(baseline), std::move(space_states), std::move(feed_block), + std::move(distribution_config), deferred_activation); } } diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index d18935afe24..98796ee6440 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -125,24 +125,28 @@ ServiceLayerNode::initializeNodeSpecific() void ServiceLayerNode::handleLiveConfigUpdate(const InitialGuard & initGuard) { - if (_server_config.staging) { - bool updated = false; - vespa::config::content::core::StorServerConfigBuilder oldC(*_server_config.active); - StorServerConfig& newC(*_server_config.staging); - { - updated = false; - NodeStateUpdater::Lock::SP lock(_component->getStateUpdater().grabStateChangeLock()); - lib::NodeState ns(*_component->getStateUpdater().getReportedNodeState()); - if (DIFFER(nodeCapacity)) { - LOG(info, "Live config update: Updating node capacity from %f to %f.", - oldC.nodeCapacity, newC.nodeCapacity); - ASSIGN(nodeCapacity); - ns.setCapacity(newC.nodeCapacity); - } - if (updated) { - // FIXME this always gets overwritten by StorageNode::handleLiveConfigUpdate...! Intentional? - _server_config.active = std::make_unique<vespa::config::content::core::StorServerConfig>(oldC); - _component->getStateUpdater().setReportedNodeState(ns); + { + std::lock_guard config_lock(_configLock); + // Live server config patching happens both here and in StorageNode::handleLiveConfigUpdate, + // which we have to delegate to afterward (_without_ holding _configLock at the time). + if (_server_config.staging) { + bool updated = false; + vespa::config::content::core::StorServerConfigBuilder oldC(*_server_config.active); + StorServerConfig& newC(*_server_config.staging); + { + NodeStateUpdater::Lock::SP lock(_component->getStateUpdater().grabStateChangeLock()); + lib::NodeState ns(*_component->getStateUpdater().getReportedNodeState()); + if (DIFFER(nodeCapacity)) { + LOG(info, "Live config update: Updating node capacity from %f to %f.", + oldC.nodeCapacity, newC.nodeCapacity); + ASSIGN(nodeCapacity); + ns.setCapacity(newC.nodeCapacity); + } + if (updated) { + // FIXME the patching of old config vs new config is confusing and error-prone. Redesign! + _server_config.active = std::make_unique<vespa::config::content::core::StorServerConfig>(oldC); + _component->getStateUpdater().setReportedNodeState(ns); + } } } } diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index adebaa51c08..a2106dce8d2 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -21,7 +21,7 @@ #include <fstream> #include <ranges> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".state.manager"); using vespalib::make_string_short::fmt; @@ -74,10 +74,13 @@ StateManager::StateManager(StorageComponentRegister& compReg, _health_ping_time(), _health_ping_warn_interval(5min), _health_ping_warn_time(_start_time + _health_ping_warn_interval), + _last_accepted_cluster_state_time(), + _last_observed_version_from_cc(), _hostInfo(std::move(hostInfo)), _controllers_observed_explicit_node_state(), _noThreadTestMode(testMode), _grabbedExternalLock(false), + _require_strictly_increasing_cluster_state_versions(false), _notifyingListeners(false), _requested_almost_immediate_node_state_replies(false) { @@ -436,21 +439,67 @@ StateManager::mark_controller_as_having_observed_explicit_node_state(const std:: _controllers_observed_explicit_node_state.emplace(controller_index); } -void -StateManager::setClusterStateBundle(const ClusterStateBundle& c) +std::optional<uint32_t> +StateManager::try_set_cluster_state_bundle(std::shared_ptr<const ClusterStateBundle> c, + uint16_t origin_controller_index) { { std::lock_guard lock(_stateLock); - _nextSystemState = std::make_shared<const ClusterStateBundle>(c); + uint32_t effective_active_version = (_nextSystemState ? _nextSystemState->getVersion() + : _systemState->getVersion()); + const auto now = _component.getClock().getMonotonicTime(); + const uint32_t last_ver_from_cc = _last_observed_version_from_cc[origin_controller_index]; + _last_observed_version_from_cc[origin_controller_index] = c->getVersion(); + + if (_require_strictly_increasing_cluster_state_versions && (c->getVersion() < effective_active_version)) { + if (c->getVersion() >= last_ver_from_cc) { + constexpr auto reject_warn_threshold = 30s; + if (now - _last_accepted_cluster_state_time <= reject_warn_threshold) { + LOG(debug, "Rejecting cluster state with version %u from cluster controller %u, as " + "we've already accepted version %u. Recently accepted another cluster state, " + "so assuming transient CC leadership period overlap.", + c->getVersion(), origin_controller_index, effective_active_version); + } else { + // Rejections have happened for some time. Make a bit of noise. + LOGBP(warning, "Rejecting cluster state with version %u from cluster controller %u, as " + "we've already accepted version %u.", + c->getVersion(), origin_controller_index, effective_active_version); + } + return {effective_active_version}; + } else { + // SetSystemState RPCs are FIFO-ordered and a particular CC should enforce strictly increasing + // cluster state versions through its ZooKeeper quorum (but commands may be resent for a given + // version). This means that commands should contain _monotonically increasing_ versions from + // a given CC origin index. + // If this is _not_ the case, it indicates ZooKeeper state on the CCs has been lost or wiped, + // at which point we have no other realistic choice than to accept the version, or the system + // will stall until an operator manually intervenes by restarting the content cluster. + LOG(error, "Received cluster state version %u from cluster controller %u, which is lower than " + "the current state version (%u) and the last received version (%u) from the same controller. " + "This indicates loss of cluster controller ZooKeeper state; accepting lower version to " + "prevent content cluster operations from stalling for an indeterminate amount of time.", + c->getVersion(), origin_controller_index, effective_active_version, last_ver_from_cc); + // Fall through to state acceptance. + } + } + _last_accepted_cluster_state_time = now; + _nextSystemState = std::move(c); } notifyStateListeners(); + return std::nullopt; } bool StateManager::onSetSystemState(const std::shared_ptr<api::SetSystemStateCommand>& cmd) { - setClusterStateBundle(cmd->getClusterStateBundle()); - sendUp(std::make_shared<api::SetSystemStateReply>(*cmd)); + auto reply = std::make_shared<api::SetSystemStateReply>(*cmd); + const auto maybe_rejected_by_ver = try_set_cluster_state_bundle(cmd->cluster_state_bundle_ptr(), cmd->getSourceIndex()); + if (maybe_rejected_by_ver) { + reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, + fmt("Cluster state version %u rejected; node already has a higher cluster state version (%u)", + cmd->getClusterStateBundle().getVersion(), *maybe_rejected_by_ver))); + } + sendUp(reply); return true; } @@ -520,6 +569,13 @@ StateManager::tick() { warn_on_missing_health_ping(); } +void +StateManager::set_require_strictly_increasing_cluster_state_versions(bool req) noexcept +{ + std::lock_guard guard(_stateLock); + _require_strictly_increasing_cluster_state_versions = req; +} + bool StateManager::sendGetNodeStateReplies() { return sendGetNodeStateReplies(0xffff); diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index 72b89dc4d65..d116f968731 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -1,9 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * @class storage::StateManager - * @ingroup storageserver - * - * @brief Keeps and updates node and system states. + * Keeps and updates node and system states. * * This component implements the NodeStateUpdater interface to handle states * for all components. See that interface for documentation. @@ -22,11 +19,13 @@ #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/vespalib/objects/floatingpointtype.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <atomic> #include <deque> +#include <list> #include <map> +#include <optional> #include <unordered_set> -#include <list> -#include <atomic> namespace metrics { class MetricManager; @@ -69,6 +68,8 @@ class StateManager : public NodeStateUpdater, std::optional<vespalib::steady_time> _health_ping_time; vespalib::duration _health_ping_warn_interval; vespalib::steady_time _health_ping_warn_time; + vespalib::steady_time _last_accepted_cluster_state_time; + vespalib::hash_map<uint16_t, uint32_t> _last_observed_version_from_cc; std::unique_ptr<HostInfo> _hostInfo; std::unique_ptr<framework::Thread> _thread; // Controllers that have observed a GetNodeState response sent _after_ @@ -76,6 +77,7 @@ class StateManager : public NodeStateUpdater, std::unordered_set<uint16_t> _controllers_observed_explicit_node_state; bool _noThreadTestMode; bool _grabbedExternalLock; + bool _require_strictly_increasing_cluster_state_versions; std::atomic<bool> _notifyingListeners; std::atomic<bool> _requested_almost_immediate_node_state_replies; @@ -90,6 +92,9 @@ public: void tick(); void warn_on_missing_health_ping(); + // Precondition: internal state mutex must not be held + void set_require_strictly_increasing_cluster_state_versions(bool req) noexcept; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; @@ -102,7 +107,11 @@ public: Lock::SP grabStateChangeLock() override; void setReportedNodeState(const lib::NodeState& state) override; - void setClusterStateBundle(const ClusterStateBundle& c); + // Iff state was accepted, returns std::nullopt + // Otherwise (i.e. state was rejected due to a higher version already having been accepted) + // returns an optional containing the current, higher cluster state version. + [[nodiscard]] std::optional<uint32_t> try_set_cluster_state_bundle(std::shared_ptr<const ClusterStateBundle> c, + uint16_t origin_controller_index); HostInfo& getHostInfo() { return *_hostInfo; } void immediately_send_get_node_state_replies() override; diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 35b70dd853c..5b665f830d3 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -92,6 +92,7 @@ StorageNode::StorageNode( _statusMetrics(), _stateReporter(), _stateManager(), + _state_manager_ptr(nullptr), _chain(), _configLock(), _initial_config_mutex(), @@ -146,6 +147,8 @@ StorageNode::initialize(const NodeStateReporter & nodeStateReporter) std::move(_hostInfo), nodeStateReporter, _singleThreadedDebugMode); + _stateManager->set_require_strictly_increasing_cluster_state_versions(server_config().requireStrictlyIncreasingClusterStateVersions); + _state_manager_ptr = _stateManager.get(); _context.getComponentRegister().setNodeStateUpdater(*_stateManager); // Create storage root folder, in case it doesn't already exist. @@ -245,12 +248,21 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) DIFFERWARN(clusterName, "Cannot alter cluster name of node live"); DIFFERWARN(nodeIndex, "Cannot alter node index of node live"); DIFFERWARN(isDistributor, "Cannot alter role of node live"); - _server_config.active = std::make_unique<StorServerConfig>(oldC); // TODO this overwrites from ServiceLayerNode + [[maybe_unused]] bool updated = false; // magically touched by ASSIGN() macro. TODO rewrite this fun stuff. + if (DIFFER(requireStrictlyIncreasingClusterStateVersions)) { + LOG(info, "Live config update: require strictly increasing cluster state versions: %s -> %s", + (oldC.requireStrictlyIncreasingClusterStateVersions ? "true" : "false"), + (newC.requireStrictlyIncreasingClusterStateVersions ? "true" : "false")); + ASSIGN(requireStrictlyIncreasingClusterStateVersions); + } + _server_config.active = std::make_unique<StorServerConfig>(oldC); _server_config.staging.reset(); _deadLockDetector->enableWarning(server_config().enableDeadLockDetectorWarnings); _deadLockDetector->enableShutdown(server_config().enableDeadLockDetector); _deadLockDetector->setProcessSlack(vespalib::from_s(server_config().deadLockDetectorTimeoutSlack)); _deadLockDetector->setWaitSlack(vespalib::from_s(server_config().deadLockDetectorTimeoutSlack)); + assert(_state_manager_ptr); + _state_manager_ptr->set_require_strictly_increasing_cluster_state_versions(server_config().requireStrictlyIncreasingClusterStateVersions); } if (_distribution_config.staging) { StorDistributionConfigBuilder oldC(*_distribution_config.active); @@ -440,11 +452,17 @@ StorageNode::stage_config_change(ConfigWrapper<ConfigT>& cfg, std::unique_ptr<Co // else is doing configuration work, and then we write the new config // to a variable where we can find it later when processing config // updates + // TODO bail if we're shutting down to avoid racing with chain destruction? + // - only relevant for distribution config since it's not pushed by main thread + // - or have some way of injecting config changes from that level...? must be done atomically! + // - ideally want to expose cluster state _bundles_ to relevant components, not config alone! + bool live_update; { std::lock_guard config_lock_guard(_configLock); cfg.staging = std::move(new_cfg); + live_update = static_cast<bool>(cfg.active); } - if (cfg.active) { + if (live_update) { InitialGuard concurrent_config_guard(_initial_config_mutex); handleLiveConfigUpdate(concurrent_config_guard); } diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index a96f6b52a66..93265bece3c 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -135,6 +135,10 @@ private: // Depends on metric manager std::unique_ptr<StateReporter> _stateReporter; std::unique_ptr<StateManager> _stateManager; + // Node subclasses may take ownership of _stateManager in order to infuse it into + // their own storage link chain, but they MUST ensure its lifetime is maintained. + // We need to remember the original pointer in order to update its config. + StateManager* _state_manager_ptr; // The storage chain can depend on anything. std::unique_ptr<StorageLink> _chain; diff --git a/storage/src/vespa/storage/tools/CMakeLists.txt b/storage/src/vespa/storage/tools/CMakeLists.txt index 99f51a9109b..9551f658bbd 100644 --- a/storage/src/vespa/storage/tools/CMakeLists.txt +++ b/storage/src/vespa/storage/tools/CMakeLists.txt @@ -3,13 +3,13 @@ vespa_add_executable(storage_getidealstate_app SOURCES getidealstate.cpp DEPENDS - storage + vespa_storage ) vespa_add_executable(storage_generatedistributionbits_app SOURCES generatedistributionbits.cpp DEPENDS - storage + vespa_storage ) vespa_add_executable(storage_storage-cmd_app SOURCES diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 57198f6761a..9d03f2c597a 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -541,19 +541,19 @@ VisitorThread::onInternal(const std::shared_ptr<api::InternalCommand>& cmd) { auto& pcmd = dynamic_cast<PropagateVisitorConfig&>(*cmd); const vespa::config::content::core::StorVisitorConfig& config(pcmd.getConfig()); - LOG(config, "Updating visitor thread configuration in visitor " - "thread %u: " - "Current config(defaultParallelIterators %u," - " iteratorsPerBucket %u," - " visitorMemoryUsageLimit %u)" - "New config(defaultParallelIterators %u," - " visitorMemoryUsageLimit %u)", - _threadIndex, - _defaultParallelIterators, - _iteratorsPerBucket, - _visitorMemoryUsageLimit, - config.defaultparalleliterators, - config.visitorMemoryUsageLimit + LOG(debug, "Updating visitor thread configuration in visitor " + "thread %u: " + "Current config(defaultParallelIterators %u," + " iteratorsPerBucket %u," + " visitorMemoryUsageLimit %u)" + "New config(defaultParallelIterators %u," + " visitorMemoryUsageLimit %u)", + _threadIndex, + _defaultParallelIterators, + _iteratorsPerBucket, + _visitorMemoryUsageLimit, + config.defaultparalleliterators, + config.visitorMemoryUsageLimit ); _defaultParallelIterators = config.defaultparalleliterators; _visitorMemoryUsageLimit = config.visitorMemoryUsageLimit; diff --git a/storage/src/vespa/storageapi/app/CMakeLists.txt b/storage/src/vespa/storageapi/app/CMakeLists.txt index 19bd276867e..b1402bbcf23 100644 --- a/storage/src/vespa/storageapi/app/CMakeLists.txt +++ b/storage/src/vespa/storageapi/app/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_executable(storageapi_getbucketid_app SOURCES getbucketid.cpp DEPENDS - storage + vespa_storage ) diff --git a/storage/src/vespa/storageapi/message/state.cpp b/storage/src/vespa/storageapi/message/state.cpp index 5a50167f584..b4e8655d783 100644 --- a/storage/src/vespa/storageapi/message/state.cpp +++ b/storage/src/vespa/storageapi/message/state.cpp @@ -5,8 +5,7 @@ #include <vespa/vdslib/state/clusterstate.h> #include <ostream> -namespace storage { -namespace api { +namespace storage::api { IMPLEMENT_COMMAND(GetNodeStateCommand, GetNodeStateReply) IMPLEMENT_REPLY(GetNodeStateReply) @@ -45,7 +44,7 @@ GetNodeStateReply::GetNodeStateReply(const GetNodeStateCommand& cmd) GetNodeStateReply::GetNodeStateReply(const GetNodeStateCommand& cmd, const lib::NodeState& state) : StorageReply(cmd), - _state(new lib::NodeState(state)) + _state(std::make_unique<lib::NodeState>(state)) { } @@ -64,23 +63,31 @@ GetNodeStateReply::print(std::ostream& out, bool verbose, } } +SetSystemStateCommand::SetSystemStateCommand(std::shared_ptr<const lib::ClusterStateBundle> state) + : StorageCommand(MessageType::SETSYSTEMSTATE), + _state(std::move(state)) +{ +} + SetSystemStateCommand::SetSystemStateCommand(const lib::ClusterStateBundle& state) : StorageCommand(MessageType::SETSYSTEMSTATE), - _state(state) + _state(std::make_shared<const lib::ClusterStateBundle>(state)) { } SetSystemStateCommand::SetSystemStateCommand(const lib::ClusterState& state) : StorageCommand(MessageType::SETSYSTEMSTATE), - _state(state) + _state(std::make_shared<const lib::ClusterStateBundle>(state)) { } +SetSystemStateCommand::~SetSystemStateCommand() = default; + void SetSystemStateCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { - out << "SetSystemStateCommand(" << *_state.getBaselineClusterState() << ")"; + out << "SetSystemStateCommand(" << *_state->getBaselineClusterState() << ")"; if (verbose) { out << " : "; StorageCommand::print(out, verbose, indent); @@ -89,7 +96,7 @@ SetSystemStateCommand::print(std::ostream& out, bool verbose, SetSystemStateReply::SetSystemStateReply(const SetSystemStateCommand& cmd) : StorageReply(cmd), - _state(cmd.getClusterStateBundle()) + _state(cmd.cluster_state_bundle_ptr()) { } @@ -138,5 +145,4 @@ void ActivateClusterStateVersionReply::print(std::ostream& out, bool verbose, } } -} // api -} // storage +} // storage::api diff --git a/storage/src/vespa/storageapi/message/state.h b/storage/src/vespa/storageapi/message/state.h index 900355b12a2..afeb5ae9c11 100644 --- a/storage/src/vespa/storageapi/message/state.h +++ b/storage/src/vespa/storageapi/message/state.h @@ -9,12 +9,6 @@ namespace storage::api { -/** - * @class GetNodeStateCommand - * @ingroup message - * - * @brief Command for setting node state. No payload - */ class GetNodeStateCommand : public StorageCommand { lib::NodeState::UP _expectedState; @@ -27,12 +21,6 @@ public: DECLARE_STORAGECOMMAND(GetNodeStateCommand, onGetNodeState) }; -/** - * @class GetNodeStateReply - * @ingroup message - * - * @brief Reply to GetNodeStateCommand - */ class GetNodeStateReply : public StorageReply { lib::NodeState::UP _state; std::string _nodeInfo; @@ -53,41 +41,38 @@ public: }; /** - * @class SetSystemStateCommand - * @ingroup message - * - * @brief Command for telling a node about the system state - state of each node - * in the system and state of the system (all ok, no merging, block - * put/get/remove etx) + * Command for telling a node about the cluster state - state of each node + * in the cluster and state of the cluster itself (all ok, no merging, block + * put/get/remove etx) */ class SetSystemStateCommand : public StorageCommand { - lib::ClusterStateBundle _state; + std::shared_ptr<const lib::ClusterStateBundle> _state; public: + explicit SetSystemStateCommand(std::shared_ptr<const lib::ClusterStateBundle> state); explicit SetSystemStateCommand(const lib::ClusterStateBundle &state); explicit SetSystemStateCommand(const lib::ClusterState &state); - const lib::ClusterState& getSystemState() const { return *_state.getBaselineClusterState(); } - const lib::ClusterStateBundle& getClusterStateBundle() const { return _state; } + ~SetSystemStateCommand() override; + + [[nodiscard]] const lib::ClusterState& getSystemState() const { return *_state->getBaselineClusterState(); } + [[nodiscard]] const lib::ClusterStateBundle& getClusterStateBundle() const { return *_state; } + [[nodiscard]] std::shared_ptr<const lib::ClusterStateBundle> cluster_state_bundle_ptr() const noexcept { + return _state; + } void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_STORAGECOMMAND(SetSystemStateCommand, onSetSystemState) }; -/** - * @class SetSystemStateReply - * @ingroup message - * - * @brief Reply received after a SetSystemStateCommand. - */ class SetSystemStateReply : public StorageReply { - lib::ClusterStateBundle _state; + std::shared_ptr<const lib::ClusterStateBundle> _state; public: explicit SetSystemStateReply(const SetSystemStateCommand& cmd); // Not serialized. Available locally - const lib::ClusterState& getSystemState() const { return *_state.getBaselineClusterState(); } - const lib::ClusterStateBundle& getClusterStateBundle() const { return _state; } + const lib::ClusterState& getSystemState() const { return *_state->getBaselineClusterState(); } + const lib::ClusterStateBundle& getClusterStateBundle() const { return *_state; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_STORAGEREPLY(SetSystemStateReply, onSetSystemStateReply) diff --git a/storageserver/CMakeLists.txt b/storageserver/CMakeLists.txt index 617474d80b1..3016f0ae266 100644 --- a/storageserver/CMakeLists.txt +++ b/storageserver/CMakeLists.txt @@ -1,15 +1,15 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - storage - streamingvisitors + vespa_storage + vespa_streamingvisitors APPS src/apps/storaged src/vespa/storageserver/app TEST_DEPENDS - messagebus_messagebus-test + vespa_messagebus-test TESTS src/tests diff --git a/storageserver/src/vespa/storageserver/app/CMakeLists.txt b/storageserver/src/vespa/storageserver/app/CMakeLists.txt index 837571ae6ad..20da98f0662 100644 --- a/storageserver/src/vespa/storageserver/app/CMakeLists.txt +++ b/storageserver/src/vespa/storageserver/app/CMakeLists.txt @@ -6,5 +6,5 @@ vespa_add_library(storageserver_storageapp STATIC servicelayerprocess.cpp dummyservicelayerprocess.cpp DEPENDS - storage + vespa_storage ) diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index a990f6a4de0..7ecb91d387d 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -2,13 +2,13 @@ vespa_define_module( DEPENDS vespalog - storage - config_cloudconfig + vespa_storage + vespa_config vespalib - document - vdslib - searchlib - searchsummary + vespa_document + vespa_vdslib + vespa_searchlib + vespa_searchsummary LIBS src/vespa/searchvisitor diff --git a/streamingvisitors/src/tests/charbuffer/CMakeLists.txt b/streamingvisitors/src/tests/charbuffer/CMakeLists.txt index d4b3da21e20..93d59c81036 100644 --- a/streamingvisitors/src/tests/charbuffer/CMakeLists.txt +++ b/streamingvisitors/src/tests/charbuffer/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(vsm_charbuffer_test_app TEST SOURCES charbuffer_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors + GTest::gtest ) vespa_add_test(NAME vsm_charbuffer_test_app COMMAND vsm_charbuffer_test_app) diff --git a/streamingvisitors/src/tests/charbuffer/charbuffer_test.cpp b/streamingvisitors/src/tests/charbuffer/charbuffer_test.cpp index b4e954d1ce0..a42851fc00a 100644 --- a/streamingvisitors/src/tests/charbuffer/charbuffer_test.cpp +++ b/streamingvisitors/src/tests/charbuffer/charbuffer_test.cpp @@ -1,80 +1,65 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vsm/common/charbuffer.h> namespace vsm { -class CharBufferTest : public vespalib::TestApp +TEST(CharBufferTest, empty) { -private: - void test(); -public: - int Main() override; -}; + CharBuffer buf; + EXPECT_EQ(buf.getLength(), 0u); + EXPECT_EQ(buf.getPos(), 0u); + EXPECT_EQ(buf.getRemaining(), 0u); +} -void -CharBufferTest::test() +TEST(CharBufferTest, explicit_length) { - { // empty - CharBuffer buf; - EXPECT_EQUAL(buf.getLength(), 0u); - EXPECT_EQUAL(buf.getPos(), 0u); - EXPECT_EQUAL(buf.getRemaining(), 0u); - } - { // explicit length - CharBuffer buf(8); - EXPECT_EQUAL(buf.getLength(), 8u); - EXPECT_EQUAL(buf.getPos(), 0u); - EXPECT_EQUAL(buf.getRemaining(), 8u); - } - { // resize - CharBuffer buf(8); - EXPECT_EQUAL(buf.getLength(), 8u); - buf.resize(16); - EXPECT_EQUAL(buf.getLength(), 16u); - buf.resize(8); - EXPECT_EQUAL(buf.getLength(), 16u); - } - { // put with triggered resize - CharBuffer buf(8); - buf.put("123456", 6); - EXPECT_EQUAL(buf.getLength(), 8u); - EXPECT_EQUAL(buf.getPos(), 6u); - EXPECT_EQUAL(buf.getRemaining(), 2u); - EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456"); - buf.put("789", 3); - EXPECT_EQUAL(buf.getLength(), 12u); - EXPECT_EQUAL(buf.getPos(), 9u); - EXPECT_EQUAL(buf.getRemaining(), 3u); - EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456789"); - buf.put('a'); - EXPECT_EQUAL(buf.getLength(), 12u); - EXPECT_EQUAL(buf.getPos(), 10u); - EXPECT_EQUAL(buf.getRemaining(), 2u); - EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "123456789a"); - buf.reset(); - EXPECT_EQUAL(buf.getLength(), 12u); - EXPECT_EQUAL(buf.getPos(), 0u); - EXPECT_EQUAL(buf.getRemaining(), 12u); - buf.put("bcd", 3); - EXPECT_EQUAL(buf.getLength(), 12u); - EXPECT_EQUAL(buf.getPos(), 3u); - EXPECT_EQUAL(buf.getRemaining(), 9u); - EXPECT_EQUAL(std::string(buf.getBuffer(), buf.getPos()), "bcd"); - } + CharBuffer buf(8); + EXPECT_EQ(buf.getLength(), 8u); + EXPECT_EQ(buf.getPos(), 0u); + EXPECT_EQ(buf.getRemaining(), 8u); } -int -CharBufferTest::Main() +TEST(CharBufferTest, resize) { - TEST_INIT("charbuffer_test"); - - test(); + CharBuffer buf(8); + EXPECT_EQ(buf.getLength(), 8u); + buf.resize(16); + EXPECT_EQ(buf.getLength(), 16u); + buf.resize(8); + EXPECT_EQ(buf.getLength(), 16u); +} - TEST_DONE(); +TEST(CharBufferTest, put_with_triggered_resize) +{ + CharBuffer buf(8); + buf.put("123456", 6); + EXPECT_EQ(buf.getLength(), 8u); + EXPECT_EQ(buf.getPos(), 6u); + EXPECT_EQ(buf.getRemaining(), 2u); + EXPECT_EQ(std::string(buf.getBuffer(), buf.getPos()), "123456"); + buf.put("789", 3); + EXPECT_EQ(buf.getLength(), 12u); + EXPECT_EQ(buf.getPos(), 9u); + EXPECT_EQ(buf.getRemaining(), 3u); + EXPECT_EQ(std::string(buf.getBuffer(), buf.getPos()), "123456789"); + buf.put('a'); + EXPECT_EQ(buf.getLength(), 12u); + EXPECT_EQ(buf.getPos(), 10u); + EXPECT_EQ(buf.getRemaining(), 2u); + EXPECT_EQ(std::string(buf.getBuffer(), buf.getPos()), "123456789a"); + buf.reset(); + EXPECT_EQ(buf.getLength(), 12u); + EXPECT_EQ(buf.getPos(), 0u); + EXPECT_EQ(buf.getRemaining(), 12u); + buf.put("bcd", 3); + EXPECT_EQ(buf.getLength(), 12u); + EXPECT_EQ(buf.getPos(), 3u); + EXPECT_EQ(buf.getRemaining(), 9u); + EXPECT_EQ(std::string(buf.getBuffer(), buf.getPos()), "bcd"); } } -TEST_APPHOOK(vsm::CharBufferTest); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/streamingvisitors/src/tests/docsum/CMakeLists.txt b/streamingvisitors/src/tests/docsum/CMakeLists.txt index 15c1e26b08b..ac80824e21b 100644 --- a/streamingvisitors/src/tests/docsum/CMakeLists.txt +++ b/streamingvisitors/src/tests/docsum/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(vsm_docsum_test_app TEST SOURCES docsum_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors + GTest::gtest ) vespa_add_test(NAME vsm_docsum_test_app COMMAND vsm_docsum_test_app) diff --git a/streamingvisitors/src/tests/docsum/docsum_test.cpp b/streamingvisitors/src/tests/docsum/docsum_test.cpp index d6535046e87..112a4e7f679 100644 --- a/streamingvisitors/src/tests/docsum/docsum_test.cpp +++ b/streamingvisitors/src/tests/docsum/docsum_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> + #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/datatype/structdatatype.h> #include <vespa/document/datatype/weightedsetdatatype.h> @@ -9,6 +9,7 @@ #include <vespa/vsm/vsm/flattendocsumwriter.h> #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace document; @@ -48,24 +49,29 @@ public: }; -class DocsumTest : public vespalib::TestApp +class DocsumTest : public ::testing::Test { -private: +protected: ArrayFieldValue createFieldValue(const StringList & fv); WeightedSetFieldValue createFieldValue(const WeightedStringList & fv); - void assertFlattenDocsumWriter(const FieldValue & fv, const std::string & exp) { + void assertFlattenDocsumWriter(const FieldValue & fv, const std::string & exp, const std::string& label) { FlattenDocsumWriter fdw; - assertFlattenDocsumWriter(fdw, fv, exp); + assertFlattenDocsumWriter(fdw, fv, exp, label); } - void assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp); - void testFlattenDocsumWriter(); - void testDocSumCache(); + void assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp, const std::string& label); -public: - int Main() override; + DocsumTest(); + ~DocsumTest() override; }; +DocsumTest::DocsumTest() + : ::testing::Test() +{ +} + +DocsumTest::~DocsumTest() = default; + ArrayFieldValue DocsumTest::createFieldValue(const StringList & fv) { @@ -90,62 +96,54 @@ DocsumTest::createFieldValue(const WeightedStringList & fv) } void -DocsumTest::assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp) +DocsumTest::assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp, const std::string& label) { + SCOPED_TRACE(label); FieldPath empty; fv.iterateNested(empty.getFullRange(), fdw); std::string actual(fdw.getResult().getBuffer(), fdw.getResult().getPos()); - EXPECT_EQUAL(actual, exp); + EXPECT_EQ(exp, actual); } -void -DocsumTest::testFlattenDocsumWriter() +TEST_F(DocsumTest, flatten_docsum_writer_basic) { - { // basic tests - TEST_DO(assertFlattenDocsumWriter(StringFieldValue("foo bar"), "foo bar")); - TEST_DO(assertFlattenDocsumWriter(RawFieldValue("foo bar"), "foo bar")); - TEST_DO(assertFlattenDocsumWriter(BoolFieldValue(true), "true")); - TEST_DO(assertFlattenDocsumWriter(BoolFieldValue(false), "false")); - TEST_DO(assertFlattenDocsumWriter(LongFieldValue(123456789), "123456789")); - TEST_DO(assertFlattenDocsumWriter(createFieldValue(StringList().add("foo bar").add("baz").add(" qux ")), - "foo bar baz qux ")); - } - { // test mulitple invocations - FlattenDocsumWriter fdw("#"); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("foo"), "foo")); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("bar"), "foo#bar")); - fdw.clear(); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("baz"), "baz")); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("qux"), "baz qux")); - } - { // test resizing - FlattenDocsumWriter fdw("#"); - EXPECT_EQUAL(fdw.getResult().getPos(), 0u); - EXPECT_EQUAL(fdw.getResult().getLength(), 32u); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("aaaabbbbccccddddeeeeffffgggghhhh"), - "aaaabbbbccccddddeeeeffffgggghhhh")); - EXPECT_EQUAL(fdw.getResult().getPos(), 32u); - EXPECT_EQUAL(fdw.getResult().getLength(), 32u); - TEST_DO(assertFlattenDocsumWriter(fdw, StringFieldValue("aaaa"), "aaaabbbbccccddddeeeeffffgggghhhh#aaaa")); - EXPECT_EQUAL(fdw.getResult().getPos(), 37u); - EXPECT_TRUE(fdw.getResult().getLength() >= 37u); - fdw.clear(); - EXPECT_EQUAL(fdw.getResult().getPos(), 0u); - EXPECT_TRUE(fdw.getResult().getLength() >= 37u); - } + assertFlattenDocsumWriter(StringFieldValue("foo bar"), "foo bar", "string foo bar"); + assertFlattenDocsumWriter(RawFieldValue("foo bar"), "foo bar", "raw foo bar"); + assertFlattenDocsumWriter(BoolFieldValue(true), "true", "bool true"); + assertFlattenDocsumWriter(BoolFieldValue(false), "false", "bool false"); + assertFlattenDocsumWriter(LongFieldValue(123456789), "123456789", "long"); + assertFlattenDocsumWriter(createFieldValue(StringList().add("foo bar").add("baz").add(" qux ")), + "foo bar baz qux ", "wset"); } -int -DocsumTest::Main() +TEST_F(DocsumTest, flatten_docsum_writer_multiple_invocations) { - TEST_INIT("docsum_test"); - - TEST_DO(testFlattenDocsumWriter()); - - TEST_DONE(); + FlattenDocsumWriter fdw("#"); + assertFlattenDocsumWriter(fdw, StringFieldValue("foo"), "foo", "string foo"); + assertFlattenDocsumWriter(fdw, StringFieldValue("bar"), "foo#bar", "string bar"); + fdw.clear(); + assertFlattenDocsumWriter(fdw, StringFieldValue("baz"), "baz", "string baz"); + assertFlattenDocsumWriter(fdw, StringFieldValue("qux"), "baz qux", "string qux"); } +TEST_F(DocsumTest, flatten_docsum_writer_resizing) +{ + FlattenDocsumWriter fdw("#"); + EXPECT_EQ(fdw.getResult().getPos(), 0u); + EXPECT_EQ(fdw.getResult().getLength(), 32u); + assertFlattenDocsumWriter(fdw, StringFieldValue("aaaabbbbccccddddeeeeffffgggghhhh"), + "aaaabbbbccccddddeeeeffffgggghhhh", + "string long"); + EXPECT_EQ(fdw.getResult().getPos(), 32u); + EXPECT_EQ(fdw.getResult().getLength(), 32u); + assertFlattenDocsumWriter(fdw, StringFieldValue("aaaa"), "aaaabbbbccccddddeeeeffffgggghhhh#aaaa", "string second long"); + EXPECT_EQ(fdw.getResult().getPos(), 37u); + EXPECT_TRUE(fdw.getResult().getLength() >= 37u); + fdw.clear(); + EXPECT_EQ(fdw.getResult().getPos(), 0u); + EXPECT_TRUE(fdw.getResult().getLength() >= 37u); } -TEST_APPHOOK(vsm::DocsumTest); +} +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/streamingvisitors/src/tests/document/CMakeLists.txt b/streamingvisitors/src/tests/document/CMakeLists.txt index 717626a9492..b29171b5198 100644 --- a/streamingvisitors/src/tests/document/CMakeLists.txt +++ b/streamingvisitors/src/tests/document/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(vsm_document_test_app TEST SOURCES document_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors + GTest::gtest ) vespa_add_test(NAME vsm_document_test_app COMMAND vsm_document_test_app) diff --git a/streamingvisitors/src/tests/document/document_test.cpp b/streamingvisitors/src/tests/document/document_test.cpp index 9d35df80c73..8a2f8614b58 100644 --- a/streamingvisitors/src/tests/document/document_test.cpp +++ b/streamingvisitors/src/tests/document/document_test.cpp @@ -1,26 +1,16 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/vsm/common/storagedocument.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> using namespace document; namespace vsm { -class DocumentTest : public vespalib::TestApp -{ -private: - void testStorageDocument(); - void testStringFieldIdTMap(); -public: - int Main() override; -}; - -void -DocumentTest::testStorageDocument() +TEST(DocumentTest, storage_document) { DocumentType dt("testdoc", 0); @@ -46,84 +36,72 @@ DocumentTest::testStorageDocument() StorageDocument sdoc(std::move(doc), fpmap, 3); ASSERT_TRUE(sdoc.valid()); - EXPECT_EQUAL(std::string("foo"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString()); + EXPECT_EQ("foo", sdoc.getField(0)->getAsString()); + EXPECT_EQ("bar", sdoc.getField(1)->getAsString()); EXPECT_TRUE(sdoc.getField(2) == nullptr); // test caching - EXPECT_EQUAL(std::string("foo"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString()); + EXPECT_EQ("foo", sdoc.getField(0)->getAsString()); + EXPECT_EQ("bar", sdoc.getField(1)->getAsString()); EXPECT_TRUE(sdoc.getField(2) == nullptr); // set new values EXPECT_TRUE(sdoc.setField(0, FieldValue::UP(new StringFieldValue("baz")))); - EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("bar"), sdoc.getField(1)->getAsString()); + EXPECT_EQ("baz", sdoc.getField(0)->getAsString()); + EXPECT_EQ("bar", sdoc.getField(1)->getAsString()); EXPECT_TRUE(sdoc.getField(2) == nullptr); EXPECT_TRUE(sdoc.setField(1, FieldValue::UP(new StringFieldValue("qux")))); - EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("qux"), sdoc.getField(1)->getAsString()); + EXPECT_EQ("baz", sdoc.getField(0)->getAsString()); + EXPECT_EQ("qux", sdoc.getField(1)->getAsString()); EXPECT_TRUE(sdoc.getField(2) == nullptr); EXPECT_TRUE(sdoc.setField(2, FieldValue::UP(new StringFieldValue("quux")))); - EXPECT_EQUAL(std::string("baz"), sdoc.getField(0)->getAsString()); - EXPECT_EQUAL(std::string("qux"), sdoc.getField(1)->getAsString()); - EXPECT_EQUAL(std::string("quux"), sdoc.getField(2)->getAsString()); + EXPECT_EQ("baz", sdoc.getField(0)->getAsString()); + EXPECT_EQ("qux", sdoc.getField(1)->getAsString()); + EXPECT_EQ("quux", sdoc.getField(2)->getAsString()); EXPECT_TRUE(!sdoc.setField(3, FieldValue::UP(new StringFieldValue("thud")))); SharedFieldPathMap fim; StorageDocument s2(std::make_unique<document::Document>(), fim, 0); - EXPECT_EQUAL(IdString().toString(), s2.docDoc().getId().toString()); + EXPECT_EQ(IdString().toString(), s2.docDoc().getId().toString()); } -void DocumentTest::testStringFieldIdTMap() +TEST(DocumentTest, string_field_id_t_map) { StringFieldIdTMap m; - EXPECT_EQUAL(0u, m.highestFieldNo()); + EXPECT_EQ(0u, m.highestFieldNo()); EXPECT_TRUE(StringFieldIdTMap::npos == m.fieldNo("unknown")); m.add("f1"); - EXPECT_EQUAL(0u, m.fieldNo("f1")); - EXPECT_EQUAL(1u, m.highestFieldNo()); + EXPECT_EQ(0u, m.fieldNo("f1")); + EXPECT_EQ(1u, m.highestFieldNo()); m.add("f1"); - EXPECT_EQUAL(0u, m.fieldNo("f1")); - EXPECT_EQUAL(1u, m.highestFieldNo()); + EXPECT_EQ(0u, m.fieldNo("f1")); + EXPECT_EQ(1u, m.highestFieldNo()); m.add("f2"); - EXPECT_EQUAL(1u, m.fieldNo("f2")); - EXPECT_EQUAL(2u, m.highestFieldNo()); + EXPECT_EQ(1u, m.fieldNo("f2")); + EXPECT_EQ(2u, m.highestFieldNo()); m.add("f3", 7); - EXPECT_EQUAL(7u, m.fieldNo("f3")); - EXPECT_EQUAL(8u, m.highestFieldNo()); + EXPECT_EQ(7u, m.fieldNo("f3")); + EXPECT_EQ(8u, m.highestFieldNo()); m.add("f3"); - EXPECT_EQUAL(7u, m.fieldNo("f3")); - EXPECT_EQUAL(8u, m.highestFieldNo()); + EXPECT_EQ(7u, m.fieldNo("f3")); + EXPECT_EQ(8u, m.highestFieldNo()); m.add("f2", 13); - EXPECT_EQUAL(13u, m.fieldNo("f2")); - EXPECT_EQUAL(14u, m.highestFieldNo()); + EXPECT_EQ(13u, m.fieldNo("f2")); + EXPECT_EQ(14u, m.highestFieldNo()); m.add("f4"); - EXPECT_EQUAL(3u, m.fieldNo("f4")); - EXPECT_EQUAL(14u, m.highestFieldNo()); + EXPECT_EQ(3u, m.fieldNo("f4")); + EXPECT_EQ(14u, m.highestFieldNo()); { vespalib::asciistream os; StringFieldIdTMap t; t.add("b"); t.add("a"); os << t; - EXPECT_EQUAL(vespalib::string("a = 1\nb = 0\n"), os.str()); + EXPECT_EQ(vespalib::string("a = 1\nb = 0\n"), os.str()); } } -int -DocumentTest::Main() -{ - TEST_INIT("document_test"); - - testStorageDocument(); - testStringFieldIdTMap(); - - TEST_DONE(); -} - } -TEST_APPHOOK(vsm::DocumentTest); - +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt index 7af2bbb1355..8430c1fa679 100644 --- a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt +++ b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_hitcollector_test_app TEST SOURCES hitcollector_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors GTest::GTest ) vespa_add_test(NAME streamingvisitors_hitcollector_test_app COMMAND streamingvisitors_hitcollector_test_app) diff --git a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt index 69ae0084ee1..80509ebbad8 100644 --- a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt +++ b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(streamingvisitors_matching_elements_filler_test_app TEST matching_elements_filler_test.cpp DEPENDS searchlib_test - streamingvisitors + vespa_streamingvisitors GTest::GTest ) vespa_add_test(NAME streamingvisitors_matching_elements_filler_test_app COMMAND streamingvisitors_matching_elements_filler_test_app) diff --git a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/CMakeLists.txt b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/CMakeLists.txt index 3ded5a1dafb..f27d7801b17 100644 --- a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/CMakeLists.txt +++ b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/CMakeLists.txt @@ -3,9 +3,9 @@ vespa_add_executable(vsm_nearest_neighbor_field_searcher_test_app TEST SOURCES nearest_neighbor_field_searcher_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test - streamingvisitors + vespa_streamingvisitors GTest::GTest ) diff --git a/streamingvisitors/src/tests/query_term_filter_factory/CMakeLists.txt b/streamingvisitors/src/tests/query_term_filter_factory/CMakeLists.txt index a4359e391f4..9e7d95c6a87 100644 --- a/streamingvisitors/src/tests/query_term_filter_factory/CMakeLists.txt +++ b/streamingvisitors/src/tests/query_term_filter_factory/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_query_term_filter_factory_test_app TEST SOURCES query_term_filter_factory_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors GTest::GTest ) vespa_add_test(NAME streamingvisitors_query_term_filter_factory_test_app COMMAND streamingvisitors_query_term_filter_factory_test_app) diff --git a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt index 9fa9f75f047..0de6c12cc75 100644 --- a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt +++ b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(streamingvisitors_querywrapper_test_app TEST SOURCES querywrapper_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors + GTest::gtest ) vespa_add_test(NAME streamingvisitors_querywrapper_test_app COMMAND streamingvisitors_querywrapper_test_app) diff --git a/streamingvisitors/src/tests/querywrapper/querywrapper_test.cpp b/streamingvisitors/src/tests/querywrapper/querywrapper_test.cpp index 2a4b9e1f869..6deb0d4cda4 100644 --- a/streamingvisitors/src/tests/querywrapper/querywrapper_test.cpp +++ b/streamingvisitors/src/tests/querywrapper/querywrapper_test.cpp @@ -1,10 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/query/tree/querybuilder.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/tree/stackdumpcreator.h> #include <vespa/searchvisitor/querywrapper.h> +#include <vespa/vespalib/gtest/gtest.h> #include <iostream> using namespace search; @@ -13,17 +13,7 @@ using namespace search::streaming; namespace streaming { -class QueryWrapperTest : public vespalib::TestApp -{ -private: - void testQueryWrapper(); - -public: - int Main() override; -}; - -void -QueryWrapperTest::testQueryWrapper() +TEST(QueryWrapperTest, test_query_wrapper) { QueryNodeResultFactory empty; { @@ -49,27 +39,17 @@ QueryWrapperTest::testQueryWrapper() q.getLeaves(terms); ASSERT_TRUE(tl.size() == 3 && terms.size() == 3); for (size_t i = 0; i < 3; ++i) { - EXPECT_EQUAL(tl[i], terms[i]); + EXPECT_EQ(tl[i], terms[i]); std::cout << "t[" << i << "]:" << terms[i] << std::endl; auto phrase = dynamic_cast<PhraseQueryNode*>(terms[i]); - EXPECT_EQUAL(i == 1, phrase != nullptr); + EXPECT_EQ(i == 1, phrase != nullptr); if (i == 1) { - EXPECT_EQUAL(3u, phrase->get_terms().size()); + EXPECT_EQ(3u, phrase->get_terms().size()); } } } } -int -QueryWrapperTest::Main() -{ - TEST_INIT("querywrapper_test"); - - testQueryWrapper(); - - TEST_DONE(); } -} // namespace streaming - -TEST_APPHOOK(::streaming::QueryWrapperTest) +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/streamingvisitors/src/tests/rank_processor/CMakeLists.txt b/streamingvisitors/src/tests/rank_processor/CMakeLists.txt index 2c3e60b7d08..35186ff2cd6 100644 --- a/streamingvisitors/src/tests/rank_processor/CMakeLists.txt +++ b/streamingvisitors/src/tests/rank_processor/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_rank_processor_test_app TEST SOURCES rank_processor_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors GTest::GTest ) vespa_add_test(NAME streamingvisitors_rank_processor_test_app COMMAND streamingvisitors_rank_processor_test_app) diff --git a/streamingvisitors/src/tests/searcher/CMakeLists.txt b/streamingvisitors/src/tests/searcher/CMakeLists.txt index 2daf963a338..ad58db76572 100644 --- a/streamingvisitors/src/tests/searcher/CMakeLists.txt +++ b/streamingvisitors/src/tests/searcher/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(vsm_searcher_test_app TEST SOURCES searcher_test.cpp DEPENDS - searchlib + vespa_searchlib searchlib_test - streamingvisitors + vespa_streamingvisitors ) vespa_add_test(NAME vsm_searcher_test_app COMMAND vsm_searcher_test_app) diff --git a/streamingvisitors/src/tests/searcher/searcher_test.cpp b/streamingvisitors/src/tests/searcher/searcher_test.cpp index 84c3a542661..b3e0ad304fe 100644 --- a/streamingvisitors/src/tests/searcher/searcher_test.cpp +++ b/streamingvisitors/src/tests/searcher/searcher_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/searchlib/query/streaming/fuzzy_term.h> diff --git a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt index 87a1098f52d..83122503bf5 100644 --- a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt +++ b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_searchvisitor_test_app TEST SOURCES searchvisitor_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors GTest::GTest ) vespa_add_test(NAME streamingvisitors_searchvisitor_test_app COMMAND streamingvisitors_searchvisitor_test_app) diff --git a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp index 7cf333a4bee..0c4fb4d1864 100644 --- a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp +++ b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp @@ -15,6 +15,7 @@ #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/log/log.h> LOG_SETUP("searchvisitor_test"); @@ -30,6 +31,10 @@ vespalib::string get_doc_id(int id) { return "id:test:test::" + std::to_string(id); } +vespalib::string src_cfg(vespalib::stringref prefix, vespalib::stringref suffix) { + return prefix + TEST_PATH("cfg") + suffix; +} + /** * This class reflects the document type defined in cfg/test.sd. */ @@ -160,9 +165,9 @@ public: SearchVisitorTest::SearchVisitorTest() : _componentRegister(), - _env(::config::ConfigUri("dir:cfg"), nullptr, ""), - _factory(::config::ConfigUri("dir:cfg"), nullptr, ""), - _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig("cfg/documenttypes.cfg"))), + _env(::config::ConfigUri(src_cfg("dir:", "")), nullptr, ""), + _factory(::config::ConfigUri(src_cfg("dir:", "")), nullptr, ""), + _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig(src_cfg("", "/documenttypes.cfg")))), _doc_type(_repo->getDocumentType("test")) { assert(_doc_type != nullptr); diff --git a/streamingvisitors/src/tests/textutil/CMakeLists.txt b/streamingvisitors/src/tests/textutil/CMakeLists.txt index da99850e43e..267b67f3b97 100644 --- a/streamingvisitors/src/tests/textutil/CMakeLists.txt +++ b/streamingvisitors/src/tests/textutil/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(vsm_textutil_test_app TEST SOURCES textutil_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors + GTest::gtest ) vespa_add_test(NAME vsm_textutil_test_app COMMAND vsm_textutil_test_app) diff --git a/streamingvisitors/src/tests/textutil/textutil_test.cpp b/streamingvisitors/src/tests/textutil/textutil_test.cpp index f7f340a2182..1dbabf1f0af 100644 --- a/streamingvisitors/src/tests/textutil/textutil_test.cpp +++ b/streamingvisitors/src/tests/textutil/textutil_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/fastlib/text/normwordfolder.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vsm/searcher/fold.h> #include <vespa/vsm/searcher/futf8strchrfieldsearcher.h> #include <vespa/vsm/searcher/utf8stringfieldsearcherbase.h> @@ -23,9 +23,9 @@ using SizeV = Vector<size_t>; using SFSB = UTF8StringFieldSearcherBase; using FSFS = FUTF8StrChrFieldSearcher; -class TextUtilTest : public vespalib::TestApp +class TextUtilTest : public ::testing::Test { -private: +protected: ucs4_t getUTF8Char(const char * src); template <typename BW, bool OFF> void assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets); @@ -38,23 +38,23 @@ private: template <typename BW, bool OFF> void testSkipSeparators(); - void testSkipSeparators(); - void testSeparatorCharacter(); - void testAnsiFold(); - void test_lfoldua(); -#ifdef __x86_64__ - void test_sse2_foldua(); -#endif -public: - int Main() override; + TextUtilTest(); + ~TextUtilTest() override; }; +TextUtilTest::TextUtilTest() + : ::testing::Test() +{ +} + +TextUtilTest::~TextUtilTest() = default; + ucs4_t TextUtilTest::getUTF8Char(const char * src) { ucs4_t retval = Fast_UnicodeUtil::GetUTF8Char(src); - ASSERT_TRUE(retval != Fast_UnicodeUtil::_BadUTF8Char); + EXPECT_TRUE(retval != Fast_UnicodeUtil::_BadUTF8Char); return retval; } @@ -68,12 +68,12 @@ TextUtilTest::assertSkipSeparators(const char * input, size_t len, const UCS4V & UTF8StrChrFieldSearcher fs(0); BW bw(dstbuf.get(), offsets.get()); size_t dstlen = fs.skipSeparators(srcbuf, len, bw); - EXPECT_EQUAL(dstlen, expdstbuf.size()); + EXPECT_EQ(dstlen, expdstbuf.size()); ASSERT_TRUE(dstlen == expdstbuf.size()); for (size_t i = 0; i < dstlen; ++i) { - EXPECT_EQUAL(dstbuf[i], expdstbuf[i]); + EXPECT_EQ(dstbuf[i], expdstbuf[i]); if (OFF) { - EXPECT_EQUAL(offsets[i], expoffsets[i]); + EXPECT_EQ(offsets[i], expoffsets[i]); } } } @@ -83,7 +83,7 @@ TextUtilTest::assertAnsiFold(const std::string & toFold, const std::string & exp { char folded[256]; EXPECT_TRUE(FSFS::ansiFold(toFold.c_str(), toFold.size(), folded)); - EXPECT_EQUAL(std::string(folded, toFold.size()), exp); + EXPECT_EQ(std::string(folded, toFold.size()), exp); } void @@ -91,7 +91,7 @@ TextUtilTest::assertAnsiFold(char c, char exp) { char folded; EXPECT_TRUE(FSFS::ansiFold(&c, 1, &folded)); - EXPECT_EQUAL((int32_t)folded, (int32_t)exp); + EXPECT_EQ((int32_t)folded, (int32_t)exp); } #ifdef __x86_64__ @@ -103,8 +103,8 @@ TextUtilTest::assert_sse2_foldua(const std::string & toFold, size_t charFolded, const unsigned char * toFoldOrg = reinterpret_cast<const unsigned char *>(toFold.c_str()); const unsigned char * retval = sse2_foldua(toFoldOrg, toFold.size(), reinterpret_cast<unsigned char *>(folded + alignedStart)); - EXPECT_EQUAL((size_t)(retval - toFoldOrg), charFolded); - EXPECT_EQUAL(std::string(folded + alignedStart, charFolded), exp); + EXPECT_EQ((size_t)(retval - toFoldOrg), charFolded); + EXPECT_EQ(std::string(folded + alignedStart, charFolded), exp); } void @@ -115,9 +115,9 @@ TextUtilTest::assert_sse2_foldua(unsigned char c, unsigned char exp, size_t char unsigned char folded[32]; size_t alignedStart = 0xF - (size_t(folded + 0xF) % 0x10); const unsigned char * retval = sse2_foldua(toFold, 16, folded + alignedStart); - EXPECT_EQUAL((size_t)(retval - toFold), charFolded); + EXPECT_EQ((size_t)(retval - toFold), charFolded); for (size_t i = 0; i < charFolded; ++i) { - EXPECT_EQUAL((int32_t)folded[i + alignedStart], (int32_t)exp); + EXPECT_EQ((int32_t)folded[i + alignedStart], (int32_t)exp); } } #endif @@ -145,8 +145,7 @@ TextUtilTest::testSkipSeparators() SizeV().a(0).a(0).a(2).a(3).a(3)); } -void -TextUtilTest::testSkipSeparators() +TEST_F(TextUtilTest, skip_separators) { Fast_NormalizeWordFolder::Setup(Fast_NormalizeWordFolder::DO_SHARP_S_SUBSTITUTION); @@ -154,8 +153,7 @@ TextUtilTest::testSkipSeparators() testSkipSeparators<SFSB::OffsetWrapper, true>(); } -void -TextUtilTest::testSeparatorCharacter() +TEST_F(TextUtilTest, separator_character) { EXPECT_TRUE(SFSB::isSeparatorCharacter('\x00')); EXPECT_TRUE(SFSB::isSeparatorCharacter('\x01')); @@ -194,8 +192,7 @@ TextUtilTest::testSeparatorCharacter() EXPECT_TRUE(! SFSB::isSeparatorCharacter('\x20')); // space } -void -TextUtilTest::testAnsiFold() +TEST_F(TextUtilTest, ansi_fold) { FieldSearcher::init(); assertAnsiFold("", ""); @@ -220,8 +217,7 @@ TextUtilTest::testAnsiFold() } } -void -TextUtilTest::test_lfoldua() +TEST_F(TextUtilTest, lfoldua) { FieldSearcher::init(); char folded[256]; @@ -229,12 +225,11 @@ TextUtilTest::test_lfoldua() const char * toFold = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; size_t len = strlen(toFold); EXPECT_TRUE(FSFS::lfoldua(toFold, len, folded, alignedStart)); - EXPECT_EQUAL(std::string(folded + alignedStart, len), "abcdefghijklmnopqrstuvwxyz"); + EXPECT_EQ(std::string(folded + alignedStart, len), "abcdefghijklmnopqrstuvwxyz"); } #ifdef __x86_64__ -void -TextUtilTest::test_sse2_foldua() +TEST_F(TextUtilTest, sse2_foldua) { assert_sse2_foldua("", 0, ""); assert_sse2_foldua("ABCD", 0, ""); @@ -263,22 +258,6 @@ TextUtilTest::test_sse2_foldua() } #endif -int -TextUtilTest::Main() -{ - TEST_INIT("textutil_test"); - - testSkipSeparators(); - testSeparatorCharacter(); - testAnsiFold(); - test_lfoldua(); -#ifdef __x86_64__ - test_sse2_foldua(); -#endif - - TEST_DONE(); -} - } -TEST_APPHOOK(vsm::TextUtilTest); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/streamingvisitors/src/tests/tokens_converter/CMakeLists.txt b/streamingvisitors/src/tests/tokens_converter/CMakeLists.txt index 01a1fc965af..848e52d275b 100644 --- a/streamingvisitors/src/tests/tokens_converter/CMakeLists.txt +++ b/streamingvisitors/src/tests/tokens_converter/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_tokens_converter_test_app TEST SOURCES tokens_converter_test.cpp DEPENDS - streamingvisitors + vespa_streamingvisitors GTest::gtest ) diff --git a/streamingvisitors/src/vespa/searchvisitor/CMakeLists.txt b/streamingvisitors/src/vespa/searchvisitor/CMakeLists.txt index 4112c76bdaa..855038058d0 100644 --- a/streamingvisitors/src/vespa/searchvisitor/CMakeLists.txt +++ b/streamingvisitors/src/vespa/searchvisitor/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(streamingvisitors +vespa_add_library(vespa_streamingvisitors SOURCES attribute_access_recorder.cpp hitcollector.cpp diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 7ecda0e82f1..926da9438a1 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -684,7 +684,7 @@ SearchVisitor::RankController::processAccessedAttributes(const QueryEnvironment SearchVisitor::RankController::RankController() : _rankProfile("default"), _rankManagerSnapshot(nullptr), - _rank_score_drop_limit(std::numeric_limits<search::feature_t>::min()), + _rank_score_drop_limit(), _hasRanking(false), _hasSummaryFeatures(false), _dumpFeatures(false), @@ -703,9 +703,9 @@ SearchVisitor::RankController::setupRankProcessors(Query & query, const search::IAttributeManager & attrMan, std::vector<AttrInfo> & attributeFields) { - using RankScoreDropLimit = search::fef::indexproperties::hitcollector::RankScoreDropLimit; + using FirstPhaseRankScoreDropLimit = search::fef::indexproperties::hitcollector::FirstPhaseRankScoreDropLimit; const search::fef::RankSetup & rankSetup = _rankManagerSnapshot->getRankSetup(_rankProfile); - _rank_score_drop_limit = RankScoreDropLimit::lookup(_queryProperties, rankSetup.getRankScoreDropLimit()); + _rank_score_drop_limit = FirstPhaseRankScoreDropLimit::lookup(_queryProperties, rankSetup.get_first_phase_rank_score_drop_limit()); _rankProcessor = std::make_unique<RankProcessor>(_rankManagerSnapshot, _rankProfile, query, location, _queryProperties, _featureOverrides, &attrMan); _rankProcessor->initForRanking(wantedHitCount, use_sort_blob); // register attribute vectors needed for ranking @@ -751,8 +751,11 @@ SearchVisitor::RankController::rankMatchedDocument(uint32_t docId) bool SearchVisitor::RankController::keepMatchedDocument() { + if (!_rank_score_drop_limit.has_value()) { + return true; + } // also make sure that NaN scores are added - return (!(_rankProcessor->getRankScore() <= _rank_score_drop_limit)); + return (!(_rankProcessor->getRankScore() <= _rank_score_drop_limit.value())); } void @@ -1139,7 +1142,7 @@ SearchVisitor::handleDocument(StorageDocument::SP documentSP) } else { _hitsRejectedCount++; LOG(debug, "Do not keep document with id '%s' because rank score (%f) <= rank score drop limit (%f)", - documentId.c_str(), rp.getRankScore(), _rankController.rank_score_drop_limit()); + documentId.c_str(), rp.getRankScore(), _rankController.rank_score_drop_limit().value()); } } else { LOG(debug, "Did not match document with id '%s'", document.docDoc().getId().getScheme().toString().c_str()); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 7d73a159f6f..2f6673abd2d 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -30,6 +30,7 @@ #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/documentapi/messagebus/messages/queryresultmessage.h> #include <vespa/document/fieldvalue/iteratorhandler.h> +#include <optional> using namespace search::aggregation; @@ -127,7 +128,7 @@ private: private: vespalib::string _rankProfile; std::shared_ptr<const RankManager::Snapshot> _rankManagerSnapshot; - search::feature_t _rank_score_drop_limit; + std::optional<search::feature_t> _rank_score_drop_limit; bool _hasRanking; bool _hasSummaryFeatures; bool _dumpFeatures; @@ -157,7 +158,7 @@ private: RankProcessor * getRankProcessor() { return _rankProcessor.get(); } void setDumpFeatures(bool dumpFeatures) { _dumpFeatures = dumpFeatures; } bool getDumpFeatures() const { return _dumpFeatures; } - search::feature_t rank_score_drop_limit() const noexcept { return _rank_score_drop_limit; } + std::optional<search::feature_t> rank_score_drop_limit() const noexcept { return _rank_score_drop_limit; } /** * Setup rank processors used for ranking and dumping. diff --git a/valgrind-suppressions.txt b/valgrind-suppressions.txt index 820fc17c155..36cc61701b4 100644 --- a/valgrind-suppressions.txt +++ b/valgrind-suppressions.txt @@ -531,10 +531,9 @@ fun:_ZN6google8protobuf14DescriptorPool23internal_generated_poolEv fun:_ZN6google8protobuf14DescriptorPool24InternalAddGeneratedFileEPKvi fun:_ZN6google8protobuf8internal14AddDescriptorsEPKNS1_15DescriptorTableE - fun:call_init.part.0 + ... fun:call_init fun:_dl_init - obj:/usr/lib64/ld-2.28.so } { Protobuf 5.26.1 suppression 5 @@ -549,10 +548,9 @@ fun:_ZN6google8protobuf14DescriptorPool23internal_generated_poolEv fun:_ZN6google8protobuf14DescriptorPool24InternalAddGeneratedFileEPKvi fun:_ZN6google8protobuf8internal14AddDescriptorsEPKNS1_15DescriptorTableE - fun:call_init.part.0 + ... fun:call_init fun:_dl_init - obj:/usr/lib64/ld-2.28.so } { Protobuf 5.26.1 suppression 6 @@ -563,8 +561,59 @@ fun:_ZN4absl12lts_2024011618container_internal12raw_hash_setINS1_17FlatHashSetPolicyIPKN6google8protobuf8internal15DescriptorTableEEENS5_12_GLOBAL__N_123GeneratedMessageFactory20DescriptorByNameHashENSC_18DescriptorByNameEqESaIS9_EE6resizeEm fun:_ZN4absl12lts_2024011618container_internal12raw_hash_setINS1_17FlatHashSetPolicyIPKN6google8protobuf8internal15DescriptorTableEEENS5_12_GLOBAL__N_123GeneratedMessageFactory20DescriptorByNameHashENSC_18DescriptorByNameEqESaIS9_EE14prepare_insertEm fun:_ZN6google8protobuf14MessageFactory29InternalRegisterGeneratedFileEPKNS0_8internal15DescriptorTableE - fun:call_init.part.0 + ... fun:call_init fun:_dl_init - obj:/usr/lib64/ld-2.28.so +} +{ + NPTL keeps a cache of thread stacks, and metadata for thread local storage is not freed for threads in that cache + Memcheck:Leak + match-leak-kinds: possible + fun:calloc + fun:UnknownInlinedFun + fun:allocate_dtv + fun:_dl_allocate_tls + fun:allocate_stack + fun:pthread_create@@GLIBC_2.34 +} +{ + Fedora 40 dlopen suppression + Memcheck:Leak + match-leak-kinds: possible + fun:malloc + fun:UnknownInlinedFun + fun:_dlfo_mappings_segment_allocate + fun:_dl_find_object_update_1 + fun:_dl_find_object_update + fun:dl_open_worker_begin + fun:_dl_catch_exception + fun:dl_open_worker + fun:_dl_catch_exception + fun:_dl_open + fun:do_dlopen + fun:_dl_catch_exception + fun:_dl_catch_error + fun:dlerror_run + fun:__libc_dlopen_mode +} +{ + Fedora 40 onnxruntime 1.18.0 initialization suppression + Memcheck:Leak + match-leak-kinds: definite + fun:calloc + fun:cpuinfo_x86_linux_init + ... + fun:pthread_once@@GLIBC_2.34 + ... +} +{ + Apparent memory leak on Fedora 40. + Memcheck:Leak + match-leak-kinds: possible + fun:malloc + fun:__tsearch + fun:tsearch + fun:__add_to_environ + fun:setenv + ... } diff --git a/vbench/src/apps/dumpurl/CMakeLists.txt b/vbench/src/apps/dumpurl/CMakeLists.txt index 4e76e9220ce..b6cd5e9a7b6 100644 --- a/vbench/src/apps/dumpurl/CMakeLists.txt +++ b/vbench/src/apps/dumpurl/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_executable(vbench_dumpurl_app SOURCES dumpurl.cpp DEPENDS - vbench + vespa_vbench ) diff --git a/vbench/src/apps/vbench/CMakeLists.txt b/vbench/src/apps/vbench/CMakeLists.txt index 00bec68b1ab..a7e7c70b1b6 100644 --- a/vbench/src/apps/vbench/CMakeLists.txt +++ b/vbench/src/apps/vbench/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_executable(vbench_app SOURCES vbench.cpp DEPENDS - vbench + vespa_vbench ) diff --git a/vbench/src/tests/app_dumpurl/CMakeLists.txt b/vbench/src/tests/app_dumpurl/CMakeLists.txt index a97a3d9a7d6..a2b83138521 100644 --- a/vbench/src/tests/app_dumpurl/CMakeLists.txt +++ b/vbench/src/tests/app_dumpurl/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_app_dumpurl_test_app TEST app_dumpurl_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_app_dumpurl_test_app NO_VALGRIND COMMAND vbench_app_dumpurl_test_app) diff --git a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp index 8fecff0531d..dd687ec7c31 100644 --- a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp +++ b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/process/process.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/app_vbench/CMakeLists.txt b/vbench/src/tests/app_vbench/CMakeLists.txt index 0df03d7a3fa..45a56dc9ee5 100644 --- a/vbench/src/tests/app_vbench/CMakeLists.txt +++ b/vbench/src/tests/app_vbench/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(vbench_app_vbench_test_app TEST app_vbench_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_app_vbench_test_app NO_VALGRIND COMMAND vbench_app_vbench_test_app) configure_file(vbench.cfg.template.template vbench.cfg.template @ONLY) diff --git a/vbench/src/tests/app_vbench/app_vbench_test.cpp b/vbench/src/tests/app_vbench/app_vbench_test.cpp index 1a166f164e2..97ab828fc66 100644 --- a/vbench/src/tests/app_vbench/app_vbench_test.cpp +++ b/vbench/src/tests/app_vbench/app_vbench_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/process/process.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/benchmark_headers/CMakeLists.txt b/vbench/src/tests/benchmark_headers/CMakeLists.txt index c9e0b86b7aa..82efa87b9e5 100644 --- a/vbench/src/tests/benchmark_headers/CMakeLists.txt +++ b/vbench/src/tests/benchmark_headers/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_benchmark_headers_test_app TEST benchmark_headers_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_benchmark_headers_test_app COMMAND vbench_benchmark_headers_test_app) diff --git a/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp b/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp index c9af4138142..aa4e9b0b832 100644 --- a/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp +++ b/vbench/src/tests/benchmark_headers/benchmark_headers_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/dispatcher/CMakeLists.txt b/vbench/src/tests/dispatcher/CMakeLists.txt index aba126394b9..999e40c0f21 100644 --- a/vbench/src/tests/dispatcher/CMakeLists.txt +++ b/vbench/src/tests/dispatcher/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_dispatcher_test_app TEST dispatcher_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_dispatcher_test_app COMMAND vbench_dispatcher_test_app) diff --git a/vbench/src/tests/dispatcher/dispatcher_test.cpp b/vbench/src/tests/dispatcher/dispatcher_test.cpp index a3978ec5111..a6a0438b232 100644 --- a/vbench/src/tests/dispatcher/dispatcher_test.cpp +++ b/vbench/src/tests/dispatcher/dispatcher_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/dropped_tagger/CMakeLists.txt b/vbench/src/tests/dropped_tagger/CMakeLists.txt index 7aabb2ce989..37810842ada 100644 --- a/vbench/src/tests/dropped_tagger/CMakeLists.txt +++ b/vbench/src/tests/dropped_tagger/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_dropped_tagger_test_app TEST dropped_tagger_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_dropped_tagger_test_app COMMAND vbench_dropped_tagger_test_app) diff --git a/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp b/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp index 88a852785be..09c2d475224 100644 --- a/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp +++ b/vbench/src/tests/dropped_tagger/dropped_tagger_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/handler_thread/CMakeLists.txt b/vbench/src/tests/handler_thread/CMakeLists.txt index b994a8b72ea..1eee194f557 100644 --- a/vbench/src/tests/handler_thread/CMakeLists.txt +++ b/vbench/src/tests/handler_thread/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_handler_thread_test_app TEST handler_thread_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_handler_thread_test_app COMMAND vbench_handler_thread_test_app) diff --git a/vbench/src/tests/handler_thread/handler_thread_test.cpp b/vbench/src/tests/handler_thread/handler_thread_test.cpp index bd36556efa2..637166e6d1a 100644 --- a/vbench/src/tests/handler_thread/handler_thread_test.cpp +++ b/vbench/src/tests/handler_thread/handler_thread_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/util/time.h> diff --git a/vbench/src/tests/hex_number/CMakeLists.txt b/vbench/src/tests/hex_number/CMakeLists.txt index 4ae7f2d6ba4..9879e6ce5c5 100644 --- a/vbench/src/tests/hex_number/CMakeLists.txt +++ b/vbench/src/tests/hex_number/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_hex_number_test_app TEST hex_number_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_hex_number_test_app COMMAND vbench_hex_number_test_app) diff --git a/vbench/src/tests/hex_number/hex_number_test.cpp b/vbench/src/tests/hex_number/hex_number_test.cpp index 96b55a5791d..7d535116a19 100644 --- a/vbench/src/tests/hex_number/hex_number_test.cpp +++ b/vbench/src/tests/hex_number/hex_number_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/http_client/CMakeLists.txt b/vbench/src/tests/http_client/CMakeLists.txt index 818317dba2e..c594d830928 100644 --- a/vbench/src/tests/http_client/CMakeLists.txt +++ b/vbench/src/tests/http_client/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_http_client_test_app TEST http_client_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_http_client_test_app COMMAND vbench_http_client_test_app) diff --git a/vbench/src/tests/http_client/http_client_test.cpp b/vbench/src/tests/http_client/http_client_test.cpp index fcbea2edd58..468207f4742 100644 --- a/vbench/src/tests/http_client/http_client_test.cpp +++ b/vbench/src/tests/http_client/http_client_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/http_connection/CMakeLists.txt b/vbench/src/tests/http_connection/CMakeLists.txt index a45ed7c78bc..cd6b2201559 100644 --- a/vbench/src/tests/http_connection/CMakeLists.txt +++ b/vbench/src/tests/http_connection/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_http_connection_test_app TEST http_connection_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_http_connection_test_app COMMAND vbench_http_connection_test_app) diff --git a/vbench/src/tests/http_connection/http_connection_test.cpp b/vbench/src/tests/http_connection/http_connection_test.cpp index 0e993eef0b1..f3cc45a58d9 100644 --- a/vbench/src/tests/http_connection/http_connection_test.cpp +++ b/vbench/src/tests/http_connection/http_connection_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/http_connection_pool/CMakeLists.txt b/vbench/src/tests/http_connection_pool/CMakeLists.txt index f533393daa2..40e9448d5b7 100644 --- a/vbench/src/tests/http_connection_pool/CMakeLists.txt +++ b/vbench/src/tests/http_connection_pool/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_http_connection_pool_test_app TEST http_connection_pool_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_http_connection_pool_test_app COMMAND vbench_http_connection_pool_test_app) diff --git a/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp b/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp index 7ed011866e1..66c8899d84e 100644 --- a/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp +++ b/vbench/src/tests/http_connection_pool/http_connection_pool_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/net/crypto_engine.h> diff --git a/vbench/src/tests/input_file_reader/CMakeLists.txt b/vbench/src/tests/input_file_reader/CMakeLists.txt index ae36122bc42..1f23ca4fa89 100644 --- a/vbench/src/tests/input_file_reader/CMakeLists.txt +++ b/vbench/src/tests/input_file_reader/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_input_file_reader_test_app TEST input_file_reader_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_input_file_reader_test_app COMMAND vbench_input_file_reader_test_app) diff --git a/vbench/src/tests/latency_analyzer/CMakeLists.txt b/vbench/src/tests/latency_analyzer/CMakeLists.txt index cd16eec0ee9..ca472dcd48e 100644 --- a/vbench/src/tests/latency_analyzer/CMakeLists.txt +++ b/vbench/src/tests/latency_analyzer/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_latency_analyzer_test_app TEST latency_analyzer_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_latency_analyzer_test_app COMMAND vbench_latency_analyzer_test_app) diff --git a/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp b/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp index 7c25f255e14..47d5df354ca 100644 --- a/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp +++ b/vbench/src/tests/latency_analyzer/latency_analyzer_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/line_reader/CMakeLists.txt b/vbench/src/tests/line_reader/CMakeLists.txt index 834c25b278d..6c06a90a3ae 100644 --- a/vbench/src/tests/line_reader/CMakeLists.txt +++ b/vbench/src/tests/line_reader/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_line_reader_test_app TEST line_reader_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_line_reader_test_app COMMAND vbench_line_reader_test_app) diff --git a/vbench/src/tests/line_reader/line_reader_test.cpp b/vbench/src/tests/line_reader/line_reader_test.cpp index 7cf81bd10f0..48082f5be78 100644 --- a/vbench/src/tests/line_reader/line_reader_test.cpp +++ b/vbench/src/tests/line_reader/line_reader_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/qps_analyzer/CMakeLists.txt b/vbench/src/tests/qps_analyzer/CMakeLists.txt index 875112a8105..1be54e57151 100644 --- a/vbench/src/tests/qps_analyzer/CMakeLists.txt +++ b/vbench/src/tests/qps_analyzer/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_qps_analyzer_test_app TEST qps_analyzer_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_qps_analyzer_test_app COMMAND vbench_qps_analyzer_test_app) diff --git a/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp b/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp index 54cf94a57b1..b14508778f3 100644 --- a/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp +++ b/vbench/src/tests/qps_analyzer/qps_analyzer_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/qps_tagger/CMakeLists.txt b/vbench/src/tests/qps_tagger/CMakeLists.txt index 15621f36da6..6254b433b8a 100644 --- a/vbench/src/tests/qps_tagger/CMakeLists.txt +++ b/vbench/src/tests/qps_tagger/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_qps_tagger_test_app TEST qps_tagger_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_qps_tagger_test_app COMMAND vbench_qps_tagger_test_app) diff --git a/vbench/src/tests/qps_tagger/qps_tagger_test.cpp b/vbench/src/tests/qps_tagger/qps_tagger_test.cpp index 1bec90042e7..80eace04f12 100644 --- a/vbench/src/tests/qps_tagger/qps_tagger_test.cpp +++ b/vbench/src/tests/qps_tagger/qps_tagger_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/request_dumper/CMakeLists.txt b/vbench/src/tests/request_dumper/CMakeLists.txt index 307d0e686ba..68b1619d560 100644 --- a/vbench/src/tests/request_dumper/CMakeLists.txt +++ b/vbench/src/tests/request_dumper/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_request_dumper_test_app TEST request_dumper_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_request_dumper_test_app COMMAND vbench_request_dumper_test_app) diff --git a/vbench/src/tests/request_dumper/request_dumper_test.cpp b/vbench/src/tests/request_dumper/request_dumper_test.cpp index 71925767582..a4cf7dc8f1d 100644 --- a/vbench/src/tests/request_dumper/request_dumper_test.cpp +++ b/vbench/src/tests/request_dumper/request_dumper_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/request_generator/CMakeLists.txt b/vbench/src/tests/request_generator/CMakeLists.txt index b2e7567fd8e..39bf67a07ab 100644 --- a/vbench/src/tests/request_generator/CMakeLists.txt +++ b/vbench/src/tests/request_generator/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_request_generator_test_app TEST request_generator_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_request_generator_test_app COMMAND vbench_request_generator_test_app) diff --git a/vbench/src/tests/request_sink/CMakeLists.txt b/vbench/src/tests/request_sink/CMakeLists.txt index 1de0ce66e17..d5b45cd600b 100644 --- a/vbench/src/tests/request_sink/CMakeLists.txt +++ b/vbench/src/tests/request_sink/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_request_sink_test_app TEST request_sink_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_request_sink_test_app COMMAND vbench_request_sink_test_app) diff --git a/vbench/src/tests/request_sink/request_sink_test.cpp b/vbench/src/tests/request_sink/request_sink_test.cpp index 334dfdd5142..606aa0577c2 100644 --- a/vbench/src/tests/request_sink/request_sink_test.cpp +++ b/vbench/src/tests/request_sink/request_sink_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/server_spec/CMakeLists.txt b/vbench/src/tests/server_spec/CMakeLists.txt index 0156c7b82c4..d7e86b90e36 100644 --- a/vbench/src/tests/server_spec/CMakeLists.txt +++ b/vbench/src/tests/server_spec/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_server_spec_test_app TEST server_spec_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_server_spec_test_app COMMAND vbench_server_spec_test_app) diff --git a/vbench/src/tests/server_spec/server_spec_test.cpp b/vbench/src/tests/server_spec/server_spec_test.cpp index 930d14f824a..49a430184ce 100644 --- a/vbench/src/tests/server_spec/server_spec_test.cpp +++ b/vbench/src/tests/server_spec/server_spec_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/server_tagger/CMakeLists.txt b/vbench/src/tests/server_tagger/CMakeLists.txt index 1e6db604318..474babaa8e4 100644 --- a/vbench/src/tests/server_tagger/CMakeLists.txt +++ b/vbench/src/tests/server_tagger/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_server_tagger_test_app TEST server_tagger_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_server_tagger_test_app COMMAND vbench_server_tagger_test_app) diff --git a/vbench/src/tests/server_tagger/server_tagger_test.cpp b/vbench/src/tests/server_tagger/server_tagger_test.cpp index e824b9fa0f1..32c59aa3177 100644 --- a/vbench/src/tests/server_tagger/server_tagger_test.cpp +++ b/vbench/src/tests/server_tagger/server_tagger_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/socket/CMakeLists.txt b/vbench/src/tests/socket/CMakeLists.txt index 421db519574..013c456e754 100644 --- a/vbench/src/tests/socket/CMakeLists.txt +++ b/vbench/src/tests/socket/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_socket_test_app TEST socket_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_socket_test_app COMMAND vbench_socket_test_app) diff --git a/vbench/src/tests/socket/socket_test.cpp b/vbench/src/tests/socket/socket_test.cpp index 67588896b28..8ab0bdd9d65 100644 --- a/vbench/src/tests/socket/socket_test.cpp +++ b/vbench/src/tests/socket/socket_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> diff --git a/vbench/src/tests/taint/CMakeLists.txt b/vbench/src/tests/taint/CMakeLists.txt index 0e938f64795..90ab35bde60 100644 --- a/vbench/src/tests/taint/CMakeLists.txt +++ b/vbench/src/tests/taint/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_taint_test_app TEST taint_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_taint_test_app COMMAND vbench_taint_test_app) diff --git a/vbench/src/tests/taint/taint_test.cpp b/vbench/src/tests/taint/taint_test.cpp index 4596f824ffd..cb95dc7da7e 100644 --- a/vbench/src/tests/taint/taint_test.cpp +++ b/vbench/src/tests/taint/taint_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/time_queue/CMakeLists.txt b/vbench/src/tests/time_queue/CMakeLists.txt index 108b054143b..dcd49516629 100644 --- a/vbench/src/tests/time_queue/CMakeLists.txt +++ b/vbench/src/tests/time_queue/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_time_queue_test_app TEST time_queue_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_time_queue_test_app COMMAND vbench_time_queue_test_app) diff --git a/vbench/src/tests/time_queue/time_queue_test.cpp b/vbench/src/tests/time_queue/time_queue_test.cpp index 4e505822105..46b2d67f684 100644 --- a/vbench/src/tests/time_queue/time_queue_test.cpp +++ b/vbench/src/tests/time_queue/time_queue_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> using namespace vbench; diff --git a/vbench/src/tests/timer/CMakeLists.txt b/vbench/src/tests/timer/CMakeLists.txt index 5197a2a5aef..52903bf1758 100644 --- a/vbench/src/tests/timer/CMakeLists.txt +++ b/vbench/src/tests/timer/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(vbench_timer_test_app TEST timer_test.cpp DEPENDS vbench_test - vbench + vespa_vbench ) vespa_add_test(NAME vbench_timer_test_app COMMAND vbench_timer_test_app) diff --git a/vbench/src/tests/timer/timer_test.cpp b/vbench/src/tests/timer/timer_test.cpp index 18defccee26..3cc15173135 100644 --- a/vbench/src/tests/timer/timer_test.cpp +++ b/vbench/src/tests/timer/timer_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vbench/test/all.h> #include <vespa/vespalib/util/time.h> diff --git a/vbench/src/vbench/CMakeLists.txt b/vbench/src/vbench/CMakeLists.txt index 7fdf0e900af..d40ff0bfa93 100644 --- a/vbench/src/vbench/CMakeLists.txt +++ b/vbench/src/vbench/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(vbench +vespa_add_library(vespa_vbench SOURCES $<TARGET_OBJECTS:vbench_core> $<TARGET_OBJECTS:vbench_http> diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index f3917691af1..305854608f5 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -3,9 +3,9 @@ vespa_define_module( DEPENDS vespalog vespalib - config_cloudconfig - configdefinitions - document + vespa_config + vespa_configdefinitions + vespa_document LIBS src/vespa/vdslib diff --git a/vdslib/src/tests/container/CMakeLists.txt b/vdslib/src/tests/container/CMakeLists.txt index 4b5b935092d..d257c327439 100644 --- a/vdslib/src/tests/container/CMakeLists.txt +++ b/vdslib/src/tests/container/CMakeLists.txt @@ -5,6 +5,6 @@ vespa_add_library(vdslib_containertest searchresulttest.cpp documentsummarytest.cpp DEPENDS - vdslib + vespa_vdslib GTest::GTest ) diff --git a/vdslib/src/tests/distribution/CMakeLists.txt b/vdslib/src/tests/distribution/CMakeLists.txt index ad9f9722a0c..2fee0838feb 100644 --- a/vdslib/src/tests/distribution/CMakeLists.txt +++ b/vdslib/src/tests/distribution/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_library(vdslib_testdistribution distributiontest.cpp grouptest.cpp DEPENDS - vdslib + vespa_vdslib GTest::GTest ) diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index 001240491d7..9d590c72559 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -11,11 +11,14 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/lexical_cast.h> +#include <vespa/vespalib/test/test_data.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/size_literals.h> #include <gmock/gmock.h> #include <chrono> +#include <filesystem> #include <fstream> #include <iterator> #include <thread> @@ -30,7 +33,35 @@ T readConfig(const config::ConfigUri & uri) return *config::ConfigGetter<T>::getConfig(uri.getConfigId(), uri.getContext()); } -TEST(DistributionTest, test_verify_java_distributions) +class DistributionTest : public ::testing::Test, public vespalib::test::TestData<DistributionTest> { +protected: + DistributionTest(); + ~DistributionTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +DistributionTest::DistributionTest() + : ::testing::Test(), + vespalib::test::TestData<DistributionTest>() +{ +} + +DistributionTest::~DistributionTest() = default; + +void +DistributionTest::SetUpTestSuite() +{ + setup_test_data(TEST_PATH("distribution/testdata"), "distribution-testdata"); +} + +void +DistributionTest::TearDownTestSuite() +{ + tear_down_test_data(); +} + +TEST_F(DistributionTest, test_verify_java_distributions) { std::vector<std::string> tests; tests.push_back("capacity"); @@ -39,16 +70,19 @@ TEST(DistributionTest, test_verify_java_distributions) tests.push_back("retired"); for (uint32_t i=0; i<tests.size(); ++i) { std::string test = tests[i]; + std::string java_cfg(source_testdata() + "/java_" + test + ".cfg"); + std::string java_testfile(source_testdata() + "/java_" + test + ".distribution"); + std::string cpp_testfile(build_testdata() + "/cpp_" + test + ".distribution"); std::string mystate; { - std::ifstream in("distribution/testdata/java_" + test + ".state"); + std::ifstream in(source_testdata() + "/java_" + test + ".state"); in >> mystate; in.close(); } ClusterState state(mystate); Distribution distr(readConfig<vespa::config::content::StorDistributionConfig>( - config::ConfigUri("file:distribution/testdata/java_" + test + ".cfg"))); - std::ofstream of("distribution/testdata/cpp_" + test + ".distribution"); + config::ConfigUri("file:" + java_cfg))); + std::ofstream of(cpp_testfile); long maxBucket = 1; long mask = 0; @@ -74,10 +108,13 @@ TEST(DistributionTest, test_verify_java_distributions) } of.close(); std::ostringstream cmd; - cmd << "diff -u " << "distribution/testdata/cpp_" << test << ".distribution " - << "distribution/testdata/java_" << test << ".distribution"; + + cmd << "diff -u " << cpp_testfile << " " << java_testfile; int result = system(cmd.str().c_str()); - EXPECT_EQ(0, result) << "Failed distribution sync test: " + test; + EXPECT_EQ(0, result) << "Failed distribution sync test: " << test; + if (result == 0) { + std::filesystem::remove(cpp_testfile); + } } } @@ -186,9 +223,9 @@ auto readFile(const std::string & filename) { return buf; } -TEST(DistributionTest, test_verify_java_distributions_2) +TEST_F(DistributionTest, test_verify_java_distributions_2) { - vespalib::DirectoryList files(vespalib::listDirectory("distribution/testdata")); + vespalib::DirectoryList files(vespalib::listDirectory(source_testdata())); for (uint32_t i=0, n=files.size(); i<n; ++i) { size_t pos = files[i].find(".java.results"); if (pos == vespalib::string::npos || pos + 13 != files[i].size()) { @@ -200,14 +237,14 @@ TEST(DistributionTest, test_verify_java_distributions_2) using namespace vespalib::slime; vespalib::Slime slime; - auto buf = readFile("distribution/testdata/" + files[i]); + auto buf = readFile(source_testdata() + "/" + files[i]); auto size = JsonFormat::decode({&buf[0], buf.size()}, slime); if (size == 0) { std::cerr << "\n\nSize of " << files[i] << " is 0. Maybe is not generated yet? Taking a 5 second nap!"; std::this_thread::sleep_for(std::chrono::seconds(5)); - buf = readFile("distribution/testdata/" + files[i]); + buf = readFile(source_testdata() + "/" + files[i]); size = JsonFormat::decode({&buf[0], buf.size()}, slime); if (size == 0) { @@ -245,12 +282,12 @@ TEST(DistributionTest, test_verify_java_distributions_2) } } -TEST(DistributionTest, test_unchanged_distribution) +TEST_F(DistributionTest, test_unchanged_distribution) { ClusterState state("distributor:10 storage:10"); Distribution distr(Distribution::getDefaultDistributionConfig(3, 10)); - std::ifstream in("distribution/testdata/41-distributordistribution"); + std::ifstream in(source_testdata() + "/41-distributordistribution"); for (unsigned i = 0; i < 64_Ki; i++) { uint16_t node = distr.getIdealDistributorNode(state, document::BucketId(16, i), "u"); @@ -353,7 +390,7 @@ std::vector<uint16_t> createNodeCountList(const std::string& source, std::vector EXPECT_EQ(exp123, cnt123); \ } -TEST(DistributionTest, test_down) +TEST_F(DistributionTest, test_down) { ASSERT_BUCKET_NODE_COUNTS( MyTest().state("storage:10 .4.s:m .5.s:m .6.s:d .7.s:d .9.s:r") @@ -366,15 +403,15 @@ TEST(DistributionTest, test_down) "0:+ 1:+ 2:+ 3:+ 8:+ 9:+"); } -TEST(DistributionTest, test_serialize_deserialize) +TEST_F(DistributionTest, test_serialize_deserialize) { MyTest t1; MyTest t2; - t2.distribution(new Distribution(t1._distribution->serialize())); + t2.distribution(new Distribution(t1._distribution->serialized())); EXPECT_EQ(t1.getNodeCounts(), t2.getNodeCounts()); } -TEST(DistributionTest, test_initializing) +TEST_F(DistributionTest, test_initializing) { ASSERT_BUCKET_NODE_COUNTS( MyTest().state("distributor:3 .0.s:i .1.s:i .2.s:i") @@ -383,7 +420,7 @@ TEST(DistributionTest, test_initializing) "0:+ 1:+ 2:+"); } -TEST(DistributionTest, testHighSplitBit) +TEST_F(DistributionTest, testHighSplitBit) { // Only 3 nodes of 10 are up => all copies should end on the 3 nodes and // none on the down nodes @@ -422,7 +459,7 @@ TEST(DistributionTest, testHighSplitBit) EXPECT_EQ(ost1.str(), ost2.str()); } -TEST(DistributionTest, test_distribution) +TEST_F(DistributionTest, test_distribution) { const int min_buckets = 64_Ki; const int max_buckets = 64_Ki; @@ -478,7 +515,7 @@ TEST(DistributionTest, test_distribution) } } -TEST(DistributionTest, test_move) +TEST_F(DistributionTest, test_move) { // This test is quite fragile, it will break if the ideal state algorithm is // changed in such a way that Bucket 0x8b4f67ae remains on node 0 and 1 if @@ -511,7 +548,7 @@ TEST(DistributionTest, test_move) EXPECT_EQ(1, int(it-diff.begin())); } -TEST(DistributionTest, test_move_constraints) +TEST_F(DistributionTest, test_move_constraints) { ClusterState clusterState("storage:10"); @@ -598,7 +635,7 @@ TEST(DistributionTest, test_move_constraints) } } -TEST(DistributionTest, test_distribution_bits) +TEST_F(DistributionTest, test_distribution_bits) { ClusterState state1("bits:16 distributor:10"); ClusterState state2("bits:19 distributor:10"); @@ -619,7 +656,7 @@ TEST(DistributionTest, test_distribution_bits) EXPECT_NE(ost1.str(), ost2.str()); } -TEST(DistributionTest, test_redundancy_hierarchical_distribution) +TEST_F(DistributionTest, test_redundancy_hierarchical_distribution) { ClusterState state("storage:10 distributor:10"); @@ -633,7 +670,7 @@ TEST(DistributionTest, test_redundancy_hierarchical_distribution) } } -TEST(DistributionTest, test_hierarchical_distribution) +TEST_F(DistributionTest, test_hierarchical_distribution) { std::string distConfig( "redundancy 4\n" @@ -683,7 +720,7 @@ TEST(DistributionTest, test_hierarchical_distribution) EXPECT_EQ(expectedMains, mainNode); } -TEST(DistributionTest, test_group_capacity) +TEST_F(DistributionTest, test_group_capacity) { std::string distConfig( "redundancy 1\n" @@ -727,7 +764,7 @@ TEST(DistributionTest, test_group_capacity) EXPECT_EQ(1000 - group0count, group1count); } -TEST(DistributionTest, test_hierarchical_no_redistribution) +TEST_F(DistributionTest, test_hierarchical_no_redistribution) { std::string distConfig( "redundancy 2\n" @@ -871,7 +908,7 @@ std::string groupConfig("group[3]\n" "group[2].nodes[2].index 5\n"); } -TEST(DistributionTest, test_active_per_group) +TEST_F(DistributionTest, test_active_per_group) { using IndexList = Distribution::IndexList; // Disabled feature @@ -946,7 +983,7 @@ TEST(DistributionTest, test_active_per_group) } } -TEST(DistributionTest, test_hierarchical_distribute_less_than_redundancy) +TEST_F(DistributionTest, test_hierarchical_distribute_less_than_redundancy) { Distribution distr("redundancy 4\nactive_per_leaf_group true\n" + groupConfig); ClusterState state("storage:6"); @@ -974,7 +1011,7 @@ TEST(DistributionTest, test_hierarchical_distribute_less_than_redundancy) } } -TEST(DistributionTest, wildcard_top_level_distribution_gives_expected_node_results) { +TEST_F(DistributionTest, wildcard_top_level_distribution_gives_expected_node_results) { std::string raw_config = R"(redundancy 2 initial_redundancy 2 ensure_primary_persisted true @@ -1062,7 +1099,7 @@ std::string generate_state_with_n_nodes_up(int n_nodes) { } -TEST(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) { +TEST_F(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) { const int n_groups = 150; Distribution distr(generate_config_with_n_1node_groups(n_groups)); ClusterState state(generate_state_with_n_nodes_up(n_groups)); @@ -1075,7 +1112,7 @@ TEST(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) { fprintf(stderr, "%.10f seconds\n", min_time); } -TEST(DistributionTest, control_size_of_IndexList) { +TEST_F(DistributionTest, control_size_of_IndexList) { EXPECT_EQ(24u, sizeof(Distribution::IndexList)); } diff --git a/storage/src/tests/common/global_bucket_space_distribution_converter_test.cpp b/vdslib/src/tests/distribution/global_bucket_space_distribution_converter_test.cpp index 774f90821fa..7d79d3bd5b4 100644 --- a/storage/src/tests/common/global_bucket_space_distribution_converter_test.cpp +++ b/vdslib/src/tests/distribution/global_bucket_space_distribution_converter_test.cpp @@ -1,13 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/vdslib/distribution/distribution.h> +#include <vespa/vdslib/distribution/global_bucket_space_distribution_converter.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/config-stor-distribution.h> #include <vespa/vespalib/gtest/gtest.h> using namespace ::testing; -namespace storage { +namespace storage::lib { using DistributionConfig = vespa::config::content::StorDistributionConfig; @@ -254,7 +255,7 @@ TEST(GlobalBucketSpaceDistributionConverterTest, can_transform_concrete_distribu auto default_cfg = GlobalBucketSpaceDistributionConverter::string_to_config(default_flat_config); lib::Distribution flat_distr(*default_cfg); auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(flat_distr); - EXPECT_EQ(expected_flat_global_config, global_distr->serialize()); + EXPECT_EQ(expected_flat_global_config, global_distr->serialized()); } TEST(GlobalBucketSpaceDistributionConverterTest, config_retired_state_is_propagated) { diff --git a/vdslib/src/tests/state/CMakeLists.txt b/vdslib/src/tests/state/CMakeLists.txt index 978b037973a..d8065a87879 100644 --- a/vdslib/src/tests/state/CMakeLists.txt +++ b/vdslib/src/tests/state/CMakeLists.txt @@ -6,6 +6,6 @@ vespa_add_library(vdslib_teststate grouptest.cpp nodestatetest.cpp DEPENDS - vdslib + vespa_vdslib GTest::GTest ) diff --git a/vdslib/src/tests/state/cluster_state_bundle_test.cpp b/vdslib/src/tests/state/cluster_state_bundle_test.cpp index d3ab516cdcd..7e5e832b186 100644 --- a/vdslib/src/tests/state/cluster_state_bundle_test.cpp +++ b/vdslib/src/tests/state/cluster_state_bundle_test.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/config-stor-distribution.h> #include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/gtest/gtest.h> @@ -19,17 +20,17 @@ struct Fixture { derivedState(std::make_shared<const ClusterState>("storage:2 .1.s:m")), bundle(baselineState, {{BucketSpace(1), derivedState}}) {} - ~Fixture() {} + ~Fixture(); }; -TEST(ClusterStateBundleTest, derived_state_is_returned_if_bucket_space_is_found) -{ +Fixture::~Fixture() = default; + +TEST(ClusterStateBundleTest, derived_state_is_returned_if_bucket_space_is_found) { Fixture f; EXPECT_EQ(*f.derivedState, *f.bundle.getDerivedClusterState(BucketSpace(1))); } -TEST(ClusterStateBundleTest, baseline_state_is_returned_if_bucket_space_is_not_found) -{ +TEST(ClusterStateBundleTest, baseline_state_is_returned_if_bucket_space_is_not_found) { Fixture f; EXPECT_EQ(f.baselineState, *f.bundle.getDerivedClusterState(BucketSpace(2))); } @@ -42,17 +43,21 @@ makeBundle(const vespalib::string &baselineState, const std::map<BucketSpace, ve for (const auto &entry : derivedStates) { derivedBucketSpaceStates[entry.first] = std::make_shared<const ClusterState>(entry.second); } - return ClusterStateBundle(ClusterState(baselineState), std::move(derivedBucketSpaceStates), deferred_activation); + return {ClusterState(baselineState), std::move(derivedBucketSpaceStates), deferred_activation}; } ClusterStateBundle -bundle_with_feed_block(const ClusterStateBundle::FeedBlock& feed_block) -{ - return ClusterStateBundle(ClusterState("storage:2"), {}, feed_block, false); +bundle_with_feed_block(const ClusterStateBundle::FeedBlock& feed_block) { + return {ClusterState("storage:2"), {}, feed_block, false}; } -TEST(ClusterStateBundleTest, verify_equality_operator) -{ +ClusterStateBundle +bundle_with_distribution(Distribution::ConfigWrapper dist_cfg_wrapper) { + auto distr_bundle = DistributionConfigBundle::of(std::move(dist_cfg_wrapper)); + return {std::make_shared<ClusterState>("storage:2"), {}, std::nullopt, std::move(distr_bundle), false}; +} + +TEST(ClusterStateBundleTest, verify_equality_operator) { Fixture f; EXPECT_NE(f.bundle, makeBundle("storage:3", {{BucketSpace(1), "storage:2 .1.s:m"}})); EXPECT_NE(f.bundle, makeBundle("storage:2", {})); @@ -63,8 +68,7 @@ TEST(ClusterStateBundleTest, verify_equality_operator) EXPECT_EQ(f.bundle, makeBundle("storage:2", {{BucketSpace(1), "storage:2 .1.s:m"}})); } -TEST(ClusterStateBundleTest, feed_block_state_is_available) -{ +TEST(ClusterStateBundleTest, feed_block_state_is_available) { auto non_blocking = makeBundle("storage:2", {}); auto blocking = bundle_with_feed_block({true, "foo"}); @@ -77,21 +81,40 @@ TEST(ClusterStateBundleTest, feed_block_state_is_available) EXPECT_EQ("foo", blocking.feed_block()->description()); } -TEST(ClusterStateBundleTest, equality_operator_considers_feed_block) -{ - EXPECT_NE(bundle_with_feed_block({true, "foo"}), bundle_with_feed_block({false, "foo"})); - EXPECT_NE(bundle_with_feed_block({true, "foo"}), bundle_with_feed_block({true, "bar"})); - EXPECT_NE(makeBundle("storage:2", {}), bundle_with_feed_block({false, "bar"})); +TEST(ClusterStateBundleTest, equality_operator_considers_feed_block) { + EXPECT_NE(bundle_with_feed_block({true, "foo"}), bundle_with_feed_block({false, "foo"})); + EXPECT_NE(bundle_with_feed_block({true, "foo"}), bundle_with_feed_block({true, "bar"})); + EXPECT_NE(makeBundle("storage:2", {}), bundle_with_feed_block({false, "bar"})); - EXPECT_EQ(bundle_with_feed_block({true, "foo"}), bundle_with_feed_block({true, "foo"})); + EXPECT_EQ(bundle_with_feed_block({true, "foo"}), bundle_with_feed_block({true, "foo"})); EXPECT_EQ(bundle_with_feed_block({false, "foo"}), bundle_with_feed_block({false, "foo"})); } -TEST(ClusterStateBundleTest, toString_with_feed_block_includes_description) -{ +TEST(ClusterStateBundleTest, equality_operator_considers_distribution_config) { + auto b1 = bundle_with_distribution(Distribution::getDefaultDistributionConfig(2, 5)); + auto b1_2 = bundle_with_distribution(Distribution::getDefaultDistributionConfig(2, 5)); + auto b2 = bundle_with_distribution(Distribution::getDefaultDistributionConfig(3, 5)); + auto b3 = bundle_with_distribution(Distribution::getDefaultDistributionConfig(2, 6)); + EXPECT_EQ(b1, b1_2); + EXPECT_EQ(b1_2, b1); + EXPECT_NE(b1, b2); + EXPECT_NE(b1, b3); + EXPECT_NE(b2, b3); + + auto no_dist = makeBundle("storage:2", {}); + EXPECT_NE(b1, no_dist); + EXPECT_NE(no_dist, b1); +} + +TEST(ClusterStateBundleTest, toString_with_feed_block_includes_description) { EXPECT_EQ("ClusterStateBundle('storage:2', feed blocked: 'full disk')", bundle_with_feed_block({true, "full disk"}).toString()); } +TEST(ClusterStateBundleTest, to_string_with_distribution_includes_high_level_summary) { + EXPECT_EQ("ClusterStateBundle('storage:2', distribution config: 1 group(s); 5 node(s); redundancy 2; searchable-copies 0)", + bundle_with_distribution(Distribution::getDefaultDistributionConfig(2, 5)).toString()); +} + } diff --git a/vdslib/src/vespa/vdslib/CMakeLists.txt b/vdslib/src/vespa/vdslib/CMakeLists.txt index b81a897390b..421283847b5 100644 --- a/vdslib/src/vespa/vdslib/CMakeLists.txt +++ b/vdslib/src/vespa/vdslib/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(vdslib +vespa_add_library(vespa_vdslib SOURCES $<TARGET_OBJECTS:vdslib_container> $<TARGET_OBJECTS:vdslib_state> diff --git a/vdslib/src/vespa/vdslib/distribution/CMakeLists.txt b/vdslib/src/vespa/vdslib/distribution/CMakeLists.txt index 30b14d8388d..89a101a2775 100644 --- a/vdslib/src/vespa/vdslib/distribution/CMakeLists.txt +++ b/vdslib/src/vespa/vdslib/distribution/CMakeLists.txt @@ -1,8 +1,11 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vdslib_distribution OBJECT SOURCES + bucket_space_distribution_configs.cpp distribution.cpp + distribution_config_bundle.cpp distribution_config_util.cpp + global_bucket_space_distribution_converter.cpp group.cpp redundancygroupdistribution.cpp DEPENDS diff --git a/storage/src/vespa/storage/distributor/bucket_space_distribution_configs.cpp b/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.cpp index 37bf8f01752..92efe8fc588 100644 --- a/storage/src/vespa/storage/distributor/bucket_space_distribution_configs.cpp +++ b/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.cpp @@ -1,10 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bucket_space_distribution_configs.h" +#include "global_bucket_space_distribution_converter.h" #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/vdslib/distribution/distribution.h> -namespace storage::distributor { +namespace storage::lib { BucketSpaceDistributionConfigs BucketSpaceDistributionConfigs::from_default_distribution(std::shared_ptr<const lib::Distribution> distribution) { diff --git a/storage/src/vespa/storage/distributor/bucket_space_distribution_configs.h b/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h index cddd21d579f..69d94869f5b 100644 --- a/storage/src/vespa/storage/distributor/bucket_space_distribution_configs.h +++ b/vdslib/src/vespa/vdslib/distribution/bucket_space_distribution_configs.h @@ -1,27 +1,27 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "distribution.h" #include <vespa/document/bucket/bucketspace.h> #include <map> #include <memory> -namespace storage::lib { class Distribution; } - -namespace storage::distributor { +namespace storage::lib { /** * Represents a complete mapping of all known bucket spaces to their appropriate, * (possibly derived) distribution config. */ struct BucketSpaceDistributionConfigs { - std::map<document::BucketSpace, std::shared_ptr<const lib::Distribution>> space_configs; + // TODO hash_map + std::map<document::BucketSpace, std::shared_ptr<const Distribution>> space_configs; - std::shared_ptr<const lib::Distribution> get_or_nullptr(document::BucketSpace space) const noexcept { + [[nodiscard]] std::shared_ptr<const Distribution> get_or_nullptr(document::BucketSpace space) const noexcept { auto iter = space_configs.find(space); - return (iter != space_configs.end()) ? iter->second : std::shared_ptr<const lib::Distribution>(); + return (iter != space_configs.end()) ? iter->second : std::shared_ptr<const Distribution>(); } - static BucketSpaceDistributionConfigs from_default_distribution(std::shared_ptr<const lib::Distribution>); + static BucketSpaceDistributionConfigs from_default_distribution(std::shared_ptr<const Distribution>); }; } diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index 4a174feffcc..694e6d68dc6 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -78,6 +78,11 @@ Distribution::ConfigWrapper::ConfigWrapper(std::unique_ptr<DistributionConfig> c Distribution::ConfigWrapper::~ConfigWrapper() = default; +std::unique_ptr<Distribution::DistributionConfig> +Distribution::ConfigWrapper::steal() noexcept { + return std::move(_cfg); +} + Distribution::Distribution(const ConfigWrapper & config) : Distribution(config.get()) { } @@ -206,7 +211,7 @@ Distribution::getStorageSeed(const document::BucketId& bucket, const ClusterStat void Distribution::print(std::ostream& out, bool, const std::string&) const { - out << serialize(); + out << serialized(); } namespace { diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.h b/vdslib/src/vespa/vdslib/distribution/distribution.h index 38ddac30fc0..4457c1831b8 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.h +++ b/vdslib/src/vespa/vdslib/distribution/distribution.h @@ -92,7 +92,8 @@ public: ConfigWrapper & operator = (ConfigWrapper && rhs) noexcept = default; explicit ConfigWrapper(std::unique_ptr<DistributionConfig> cfg) noexcept; ~ConfigWrapper(); - const DistributionConfig & get() const { return *_cfg; } + [[nodiscard]] const DistributionConfig & get() const noexcept { return *_cfg; } + [[nodiscard]] std::unique_ptr<DistributionConfig> steal() noexcept; private: std::unique_ptr<DistributionConfig> _cfg; }; @@ -105,14 +106,14 @@ public: Distribution& operator=(const Distribution&) = delete; - const vespalib::string& serialize() const noexcept { return _serialized; } + [[nodiscard]] const vespalib::string& serialized() const noexcept { return _serialized; } - const Group& getNodeGraph() const noexcept { return *_nodeGraph; } - uint16_t getRedundancy() const noexcept { return _redundancy; } - uint16_t getInitialRedundancy() const noexcept { return _initialRedundancy; } - uint16_t getReadyCopies() const noexcept { return _readyCopies; } - bool ensurePrimaryPersisted() const noexcept { return _ensurePrimaryPersisted; } - bool activePerGroup() const noexcept { return _activePerGroup; } + [[nodiscard]] const Group& getNodeGraph() const noexcept { return *_nodeGraph; } + [[nodiscard]] uint16_t getRedundancy() const noexcept { return _redundancy; } + [[nodiscard]] uint16_t getInitialRedundancy() const noexcept { return _initialRedundancy; } + [[nodiscard]] uint16_t getReadyCopies() const noexcept { return _readyCopies; } + [[nodiscard]] bool ensurePrimaryPersisted() const noexcept { return _ensurePrimaryPersisted; } + [[nodiscard]] bool activePerGroup() const noexcept { return _activePerGroup; } bool operator==(const Distribution& o) const noexcept { return (_serialized == o._serialized); } bool operator!=(const Distribution& o) const noexcept { return (_serialized != o._serialized); } @@ -120,10 +121,10 @@ public: void print(std::ostream& out, bool, const std::string&) const override; /** Simplified wrapper for getIdealNodes() */ - std::vector<uint16_t> getIdealStorageNodes(const ClusterState&, const document::BucketId&, const char* upStates = "uim") const; + [[nodiscard]] std::vector<uint16_t> getIdealStorageNodes(const ClusterState&, const document::BucketId&, const char* upStates = "uim") const; /** Simplified wrapper for getIdealNodes() */ - uint16_t getIdealDistributorNode(const ClusterState&, const document::BucketId&, const char* upStates = "uim") const; + [[nodiscard]] uint16_t getIdealDistributorNode(const ClusterState&, const document::BucketId&, const char* upStates = "uim") const; /** * @throws TooFewBucketBitsInUseException If distribution bit count is diff --git a/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.cpp b/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.cpp new file mode 100644 index 00000000000..810678af707 --- /dev/null +++ b/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.cpp @@ -0,0 +1,80 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "distribution_config_bundle.h" +#include <vespa/config/print/asciiconfigreader.hpp> +#include <vespa/config-stor-distribution.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <sstream> + +namespace storage::lib { + +namespace { + +void count_nodes_and_leaf_groups(const Group& g, uint16_t& node_count_inout, uint16_t& leaf_group_count_inout) { + if (g.isLeafGroup()) { + leaf_group_count_inout++; + node_count_inout += g.getNodes().size(); + } else { + for (const auto& sub_g : g.getSubGroups()) { + count_nodes_and_leaf_groups(*sub_g.second, node_count_inout, leaf_group_count_inout); + } + } +} + +std::unique_ptr<Distribution::DistributionConfig> config_from_existing_distribution(const Distribution& distr) { + vespalib::asciistream is(distr.serialized()); + config::AsciiConfigReader<vespa::config::content::StorDistributionConfig> reader(is); + return reader.read(); +} + +} + +// TODO de-dupe ctors +DistributionConfigBundle::DistributionConfigBundle(std::shared_ptr<const Distribution> distr) + : _config(config_from_existing_distribution(*distr)), + _default_distribution(std::move(distr)), + _bucket_space_distributions(BucketSpaceDistributionConfigs::from_default_distribution(_default_distribution)), + _total_node_count(0), + _total_leaf_group_count(0) +{ + count_nodes_and_leaf_groups(_default_distribution->getNodeGraph(), _total_node_count, _total_leaf_group_count); +} + +DistributionConfigBundle::DistributionConfigBundle(Distribution::ConfigWrapper config) + : DistributionConfigBundle(config.steal()) +{ +} + +DistributionConfigBundle::DistributionConfigBundle(std::unique_ptr<const Distribution::DistributionConfig> config) + : _config(std::move(config)), + _default_distribution(std::make_shared<Distribution>(*_config)), + _bucket_space_distributions(BucketSpaceDistributionConfigs::from_default_distribution(_default_distribution)), + _total_node_count(0), + _total_leaf_group_count(0) +{ + count_nodes_and_leaf_groups(_default_distribution->getNodeGraph(), _total_node_count, _total_leaf_group_count); +} + +DistributionConfigBundle::~DistributionConfigBundle() = default; + +bool DistributionConfigBundle::operator==(const DistributionConfigBundle& rhs) const noexcept { + // Distribution caches the raw string config format internally. + // Equality is checked using this cheap representation. + return (*_default_distribution == *rhs._default_distribution); +} + +std::shared_ptr<DistributionConfigBundle> +DistributionConfigBundle::of(std::shared_ptr<const Distribution> distr) { + return std::make_shared<DistributionConfigBundle>(std::move(distr)); +} + +std::shared_ptr<DistributionConfigBundle> +DistributionConfigBundle::of(Distribution::ConfigWrapper cfg) { + return std::make_shared<DistributionConfigBundle>(std::move(cfg)); +} + +std::shared_ptr<DistributionConfigBundle> +DistributionConfigBundle::of(std::unique_ptr<const Distribution::DistributionConfig> cfg) { + return std::make_shared<DistributionConfigBundle>(std::move(cfg)); +} + +} diff --git a/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h b/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h new file mode 100644 index 00000000000..7b68d4fb6aa --- /dev/null +++ b/vdslib/src/vespa/vdslib/distribution/distribution_config_bundle.h @@ -0,0 +1,56 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "bucket_space_distribution_configs.h" +#include "distribution.h" +#include <iosfwd> + +namespace storage::lib { + +/** + * Encapsulates immutable distribution config bound to a particular cluster state version. + * Implicitly derives (and provides) bucket space-specific distribution configs, allowing + * such transforms to be performed and stored in one location. + */ +class DistributionConfigBundle { + std::unique_ptr<const Distribution::DistributionConfig> _config; + std::shared_ptr<const Distribution> _default_distribution; + BucketSpaceDistributionConfigs _bucket_space_distributions; + uint16_t _total_node_count; + uint16_t _total_leaf_group_count; +public: + explicit DistributionConfigBundle(std::shared_ptr<const Distribution> distr); + explicit DistributionConfigBundle(std::unique_ptr<const Distribution::DistributionConfig> config); // TODO shared_ptr? + explicit DistributionConfigBundle(Distribution::ConfigWrapper config); + ~DistributionConfigBundle(); + + [[nodiscard]] const Distribution::DistributionConfig& config() const noexcept { return *_config; } + + [[nodiscard]] const Distribution& default_distribution() const noexcept { return *_default_distribution; } + [[nodiscard]] const std::shared_ptr<const Distribution>& default_distribution_sp() const noexcept { + return _default_distribution; + } + [[nodiscard]] std::shared_ptr<const Distribution> bucket_space_distribution_or_nullptr(document::BucketSpace space) const noexcept { + return _bucket_space_distributions.get_or_nullptr(space); + } + [[nodiscard]] const BucketSpaceDistributionConfigs& bucket_space_distributions() const noexcept { + return _bucket_space_distributions; + } + + [[nodiscard]] uint16_t total_node_count() const noexcept { return _total_node_count; } + [[nodiscard]] uint16_t total_leaf_group_count() const noexcept { return _total_leaf_group_count; } + // Note: these apply to the default space only + [[nodiscard]] uint16_t redundancy() const noexcept { return _default_distribution->getRedundancy(); } + [[nodiscard]] uint16_t searchable_copies() const noexcept { return _default_distribution->getReadyCopies(); } + + bool operator==(const DistributionConfigBundle& rhs) const noexcept; + bool operator!=(const DistributionConfigBundle& rhs) const noexcept { + return !(*this == rhs); + } + + [[nodiscard]] static std::shared_ptr<DistributionConfigBundle> of(std::shared_ptr<const Distribution> cfg); + [[nodiscard]] static std::shared_ptr<DistributionConfigBundle> of(Distribution::ConfigWrapper cfg); + [[nodiscard]] static std::shared_ptr<DistributionConfigBundle> of(std::unique_ptr<const Distribution::DistributionConfig> cfg); +}; + +} diff --git a/storage/src/vespa/storage/common/global_bucket_space_distribution_converter.cpp b/vdslib/src/vespa/vdslib/distribution/global_bucket_space_distribution_converter.cpp index eb42f19a5e8..ae9f97fd7b4 100644 --- a/storage/src/vespa/storage/common/global_bucket_space_distribution_converter.cpp +++ b/vdslib/src/vespa/vdslib/distribution/global_bucket_space_distribution_converter.cpp @@ -1,15 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "distribution_config_util.h" #include "global_bucket_space_distribution_converter.h" -#include <vespa/vdslib/distribution/distribution.h> +#include <vespa/config-stor-distribution.h> #include <vespa/config/print/asciiconfigwriter.h> #include <vespa/config/print/asciiconfigreader.hpp> -#include <vespa/vdslib/distribution/distribution_config_util.h> #include <vespa/vespalib/stllike/asciistream.h> #include <cassert> #include <map> -namespace storage { +namespace storage::lib { using DistributionConfig = vespa::config::content::StorDistributionConfig; using DistributionConfigBuilder = vespa::config::content::StorDistributionConfigBuilder; @@ -159,7 +159,7 @@ GlobalBucketSpaceDistributionConverter::convert_to_global(const DistributionConf std::shared_ptr<lib::Distribution> GlobalBucketSpaceDistributionConverter::convert_to_global(const lib::Distribution& distr) { - const auto src_config = distr.serialize(); + const auto& src_config = distr.serialized(); auto global_config = convert_to_global(*string_to_config(src_config)); return std::make_shared<lib::Distribution>(*global_config); } diff --git a/storage/src/vespa/storage/common/global_bucket_space_distribution_converter.h b/vdslib/src/vespa/vdslib/distribution/global_bucket_space_distribution_converter.h index c530922ad18..e0df5fab554 100644 --- a/storage/src/vespa/storage/common/global_bucket_space_distribution_converter.h +++ b/vdslib/src/vespa/vdslib/distribution/global_bucket_space_distribution_converter.h @@ -2,14 +2,13 @@ #pragma once -#include <vespa/config-stor-distribution.h> +#include "distribution.h" #include <memory> -namespace storage::lib { class Distribution; } -namespace storage { +namespace storage::lib { struct GlobalBucketSpaceDistributionConverter { - using DistributionConfig = vespa::config::content::StorDistributionConfig; + using DistributionConfig = Distribution::DistributionConfig; static std::shared_ptr<DistributionConfig> convert_to_global(const DistributionConfig&); static std::shared_ptr<lib::Distribution> convert_to_global(const lib::Distribution&); diff --git a/vdslib/src/vespa/vdslib/distribution/group.h b/vdslib/src/vespa/vdslib/distribution/group.h index 1c4daf6bdb3..10370f08953 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.h +++ b/vdslib/src/vespa/vdslib/distribution/group.h @@ -48,14 +48,14 @@ private: void getConfigHash(vespalib::asciistream & out) const; public: - // Create leaf node + // Create leaf node Group(uint16_t index, vespalib::stringref name) noexcept; - // Create branch node + // Create branch node Group(uint16_t index, vespalib::stringref name, const Distribution&, uint16_t redundancy); - virtual ~Group(); + ~Group() override; - bool isLeafGroup() const noexcept { return ! _nodes.empty(); } + [[nodiscard]] bool isLeafGroup() const noexcept { return ! _nodes.empty(); } bool operator==(const Group& other) const noexcept; void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp index 1f6c4bc4428..6ef971b4c46 100644 --- a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp +++ b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.cpp @@ -16,16 +16,18 @@ ClusterStateBundle::FeedBlock::FeedBlock(bool block_feed_in_cluster_in, } bool -ClusterStateBundle::FeedBlock::operator==(const FeedBlock& rhs) const +ClusterStateBundle::FeedBlock::operator==(const FeedBlock& rhs) const noexcept { return (_block_feed_in_cluster == rhs._block_feed_in_cluster) && (_description == rhs._description); } -ClusterStateBundle::ClusterStateBundle(const ClusterState &baselineClusterState) +// TODO implement with ctor fwd +ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState) : _baselineClusterState(std::make_shared<const ClusterState>(baselineClusterState)), _derivedBucketSpaceStates(), _feed_block(), + _distribution_bundle(), _deferredActivation(false) { } @@ -35,6 +37,7 @@ ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState, : _baselineClusterState(std::make_shared<const ClusterState>(baselineClusterState)), _derivedBucketSpaceStates(std::move(derivedBucketSpaceStates)), _feed_block(), + _distribution_bundle(), _deferredActivation(false) { } @@ -45,6 +48,7 @@ ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState, : _baselineClusterState(std::make_shared<const ClusterState>(baselineClusterState)), _derivedBucketSpaceStates(std::move(derivedBucketSpaceStates)), _feed_block(), + _distribution_bundle(), _deferredActivation(deferredActivation) { } @@ -56,10 +60,24 @@ ClusterStateBundle::ClusterStateBundle(const ClusterState& baselineClusterState, : _baselineClusterState(std::make_shared<const ClusterState>(baselineClusterState)), _derivedBucketSpaceStates(std::move(derivedBucketSpaceStates)), _feed_block(feed_block_in), + _distribution_bundle(), _deferredActivation(deferredActivation) { } +ClusterStateBundle::ClusterStateBundle(std::shared_ptr<const ClusterState> baseline_cluster_state, + BucketSpaceStateMapping derived_bucket_space_states, + std::optional<FeedBlock> feed_block_in, + std::shared_ptr<const DistributionConfigBundle> distribution_bundle, + bool deferred_activation) + : _baselineClusterState(std::move(baseline_cluster_state)), + _derivedBucketSpaceStates(std::move(derived_bucket_space_states)), + _feed_block(std::move(feed_block_in)), + _distribution_bundle(std::move(distribution_bundle)), + _deferredActivation(deferred_activation) +{ +} + ClusterStateBundle::ClusterStateBundle(const ClusterStateBundle&) = default; ClusterStateBundle& ClusterStateBundle::operator=(const ClusterStateBundle&) = default; ClusterStateBundle::ClusterStateBundle(ClusterStateBundle&&) noexcept = default; @@ -67,13 +85,20 @@ ClusterStateBundle& ClusterStateBundle::operator=(ClusterStateBundle&&) noexcept ClusterStateBundle::~ClusterStateBundle() = default; -const std::shared_ptr<const lib::ClusterState> & +std::shared_ptr<const ClusterStateBundle> +ClusterStateBundle::clone_with_new_distribution(std::shared_ptr<const DistributionConfigBundle> distribution) const +{ + return std::make_shared<const ClusterStateBundle>(_baselineClusterState, _derivedBucketSpaceStates, _feed_block, + std::move(distribution), _deferredActivation); +} + +const std::shared_ptr<const lib::ClusterState>& ClusterStateBundle::getBaselineClusterState() const { return _baselineClusterState; } -const std::shared_ptr<const lib::ClusterState> & +const std::shared_ptr<const lib::ClusterState>& ClusterStateBundle::getDerivedClusterState(document::BucketSpace bucketSpace) const { auto itr = _derivedBucketSpaceStates.find(bucketSpace); @@ -90,7 +115,7 @@ ClusterStateBundle::getVersion() const } bool -ClusterStateBundle::operator==(const ClusterStateBundle &rhs) const noexcept +ClusterStateBundle::operator==(const ClusterStateBundle& rhs) const noexcept { if (!(*_baselineClusterState == *rhs._baselineClusterState)) { return false; @@ -98,6 +123,13 @@ ClusterStateBundle::operator==(const ClusterStateBundle &rhs) const noexcept if (_derivedBucketSpaceStates.size() != rhs._derivedBucketSpaceStates.size()) { return false; } + if (_distribution_bundle && rhs._distribution_bundle) { + if (*_distribution_bundle != *rhs._distribution_bundle) { + return false; + } + } else if (_distribution_bundle || rhs._distribution_bundle) { + return false; // either side, but not both, had distribution config set + } if (_feed_block != rhs._feed_block) { return false; } @@ -138,6 +170,11 @@ std::ostream& operator<<(std::ostream& os, const ClusterStateBundle& bundle) { if (bundle.block_feed_in_cluster()) { os << ", feed blocked: '" << bundle.feed_block()->description() << "'"; } + if (auto* distr = bundle.distribution_config_bundle_or_nullptr()) { + os << ", distribution config: " << distr->total_leaf_group_count() << " group(s); " + << distr->total_node_count() << " node(s); redundancy " + << distr->redundancy() << "; searchable-copies " << distr->searchable_copies(); + } if (bundle.deferredActivation()) { os << " (deferred activation)"; } diff --git a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h index c8889ad7da4..18f82176dd5 100644 --- a/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h +++ b/vdslib/src/vespa/vdslib/state/cluster_state_bundle.h @@ -3,11 +3,12 @@ #pragma once #include <vespa/document/bucket/bucketspace.h> +#include <vespa/vdslib/distribution/distribution_config_bundle.h> #include <iosfwd> +#include <memory> #include <optional> #include <string> #include <unordered_map> -#include <memory> namespace storage::lib { @@ -17,10 +18,8 @@ class ClusterState; * Class representing the baseline cluster state and the derived cluster * state for each bucket space. */ -class ClusterStateBundle -{ +class ClusterStateBundle { public: - /** * Represents feed blocking status of the entire cluster. * @@ -36,10 +35,10 @@ public: public: FeedBlock(bool block_feed_in_cluster_in, const vespalib::string& description_in); - bool block_feed_in_cluster() const { return _block_feed_in_cluster; } - const vespalib::string& description() const { return _description; } - bool operator==(const FeedBlock& rhs) const; - bool operator!=(const FeedBlock& rhs) const { return !operator==(rhs); } + [[nodiscard]] bool block_feed_in_cluster() const noexcept { return _block_feed_in_cluster; } + [[nodiscard]] const vespalib::string& description() const noexcept { return _description; } + bool operator==(const FeedBlock& rhs) const noexcept; + bool operator!=(const FeedBlock& rhs) const noexcept { return !operator==(rhs); } }; using BucketSpaceStateMapping = std::unordered_map< @@ -47,12 +46,13 @@ public: std::shared_ptr<const ClusterState>, document::BucketSpace::hash >; - std::shared_ptr<const ClusterState> _baselineClusterState; - BucketSpaceStateMapping _derivedBucketSpaceStates; - std::optional<FeedBlock> _feed_block; - bool _deferredActivation; + std::shared_ptr<const ClusterState> _baselineClusterState; + BucketSpaceStateMapping _derivedBucketSpaceStates; + std::optional<FeedBlock> _feed_block; + std::shared_ptr<const DistributionConfigBundle> _distribution_bundle; + bool _deferredActivation; public: - explicit ClusterStateBundle(const ClusterState &baselineClusterState); + explicit ClusterStateBundle(const ClusterState& baselineClusterState); ClusterStateBundle(const ClusterState& baselineClusterState, BucketSpaceStateMapping derivedBucketSpaceStates); ClusterStateBundle(const ClusterState& baselineClusterState, @@ -62,6 +62,11 @@ public: BucketSpaceStateMapping derivedBucketSpaceStates, const FeedBlock& feed_block_in, bool deferredActivation); + ClusterStateBundle(std::shared_ptr<const ClusterState> baseline_cluster_state, + BucketSpaceStateMapping derived_bucket_space_states, + std::optional<FeedBlock> feed_block_in, + std::shared_ptr<const DistributionConfigBundle> distribution_bundle, + bool deferred_activation); ClusterStateBundle(const ClusterStateBundle&); ClusterStateBundle& operator=(const ClusterStateBundle&); @@ -69,20 +74,36 @@ public: ClusterStateBundle& operator=(ClusterStateBundle&&) noexcept; ~ClusterStateBundle(); - const std::shared_ptr<const ClusterState> &getBaselineClusterState() const; - const std::shared_ptr<const ClusterState> &getDerivedClusterState(document::BucketSpace bucketSpace) const; + + // Factory function for "simulating" atomic cluster states with distribution config attached + // when the cluster controller is not yet on version that sends bundled config. + [[nodiscard]] std::shared_ptr<const ClusterStateBundle> clone_with_new_distribution( + std::shared_ptr<const DistributionConfigBundle>) const; + + const std::shared_ptr<const ClusterState>& getBaselineClusterState() const; + const std::shared_ptr<const ClusterState>& getDerivedClusterState(document::BucketSpace bucketSpace) const; const BucketSpaceStateMapping& getDerivedClusterStates() const noexcept { return _derivedBucketSpaceStates; } [[nodiscard]] bool block_feed_in_cluster() const noexcept { return _feed_block.has_value() && _feed_block->block_feed_in_cluster(); } - const std::optional<FeedBlock>& feed_block() const { return _feed_block; } - uint32_t getVersion() const; - bool deferredActivation() const noexcept { return _deferredActivation; } - std::string toString() const; - bool operator==(const ClusterStateBundle &rhs) const noexcept; - bool operator!=(const ClusterStateBundle &rhs) const noexcept { return !operator==(rhs); } + [[nodiscard]] bool has_distribution_config() const noexcept { + return static_cast<bool>(_distribution_bundle); + } + [[nodiscard]] const DistributionConfigBundle* distribution_config_bundle_or_nullptr() const noexcept { + return _distribution_bundle.get(); + } + // Only guaranteed to not be nullptr iff has_distribution_config() == true + [[nodiscard]] const std::shared_ptr<const DistributionConfigBundle>& distribution_config_bundle() const noexcept { + return _distribution_bundle; + } + [[nodiscard]] const std::optional<FeedBlock>& feed_block() const { return _feed_block; } + [[nodiscard]] uint32_t getVersion() const; + [[nodiscard]] bool deferredActivation() const noexcept { return _deferredActivation; } + [[nodiscard]] std::string toString() const; + bool operator==(const ClusterStateBundle& rhs) const noexcept; + bool operator!=(const ClusterStateBundle& rhs) const noexcept { return !operator==(rhs); } }; std::ostream& operator<<(std::ostream&, const ClusterStateBundle&); diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index cac79c3850e..57182446a7e 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -145,6 +145,10 @@ <artifactId>slf4j-api</artifactId> </exclusion> <exclusion> + <groupId>software.amazon.awssdk</groupId> + <artifactId>ssm</artifactId> + </exclusion> + <exclusion> <groupId>org.bouncycastle</groupId> <artifactId>*</artifactId> </exclusion> diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index 2f344004780..085e9973cab 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -42,28 +42,25 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde this(new AthenzService(config.athenzDomain(), config.athenzService()), SiaUtils.getPrivateKeyFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())), SiaUtils.getCertificateFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())), - Paths.get(config.trustStorePath()), config.publicSystem()); + Paths.get(config.trustStorePath())); } public SiaIdentityProvider(AthenzIdentity service, Path siaPath, - Path clientTruststoreFile, - boolean publicSystem) { + Path clientTruststoreFile) { this(service, SiaUtils.getPrivateKeyFile(siaPath, service), SiaUtils.getCertificateFile(siaPath, service), - clientTruststoreFile, - publicSystem); + clientTruststoreFile); } public SiaIdentityProvider(AthenzIdentity service, Path privateKeyFile, Path certificateFile, - Path clientTruststoreFile, - boolean publicSystem) { + Path clientTruststoreFile) { this.service = service; this.keyManager = AutoReloadingX509KeyManager.fromPemFiles(privateKeyFile, certificateFile); - this.sslContext = createIdentitySslContext(keyManager, clientTruststoreFile, publicSystem); + this.sslContext = createIdentitySslContext(keyManager, clientTruststoreFile); this.certificateFile = certificateFile; this.privateKeyFile = privateKeyFile; } @@ -83,30 +80,23 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde @Override public Path privateKeyPath() { return privateKeyFile; } public SSLContext createIdentitySslContextWithTrustStore(Path trustStoreFile) { - return createIdentitySslContext(keyManager, trustStoreFile, false); - } - - public SSLContext createIdentitySslContextWithTrustStore(Path trustStoreFile, boolean includeDefaultTruststore) { - return createIdentitySslContext(keyManager, trustStoreFile, includeDefaultTruststore); + return createIdentitySslContext(keyManager, trustStoreFile); } /** * Create an SSL context with the given trust store and the key manager from this provider. - * If the {code includeDefaultTruststore} is true, the default trust store will be included. + * Include default trust store * * @param keyManager the key manager * @param trustStoreFile the trust store file - * @param includeDefaultTruststore whether to include the default trust store */ - private static SSLContext createIdentitySslContext(AutoReloadingX509KeyManager keyManager, Path trustStoreFile, boolean includeDefaultTruststore) { - List<X509Certificate> defaultTrustStore = List.of(); - if (includeDefaultTruststore) { - try { - // load the default java trust store and extract the certificates - defaultTrustStore = Stream.of(TrustManagerUtils.createDefaultX509TrustManager().getAcceptedIssuers()).toList(); - } catch (Exception e) { - throw new RuntimeException("Failed to load default trust store", e); - } + private static SSLContext createIdentitySslContext(AutoReloadingX509KeyManager keyManager, Path trustStoreFile) { + List<X509Certificate> defaultTrustStore; + try { + // load the default java trust store and extract the certificates + defaultTrustStore = Stream.of(TrustManagerUtils.createDefaultX509TrustManager().getAcceptedIssuers()).toList(); + } catch (Exception e) { + throw new RuntimeException("Failed to load default trust store", e); } try { List<X509Certificate> caCertList = Stream.concat( diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java deleted file mode 100644 index 9f37e3f4613..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api; - -public record DefaultSignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, - String data, IdentityDocument identityDocument) implements SignedIdentityDocument { - - public DefaultSignedIdentityDocument { - identityDocument = EntityBindingsMapper.fromIdentityDocumentData(data); - } - - public DefaultSignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, String data) { - this(signature,signingKeyVersion,documentVersion, data, null); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index 4bbdd24db32..123995721e9 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -4,12 +4,11 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.DefaultSignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.V4SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentEntity; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.LegacySignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.V5SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.yolean.Exceptions; @@ -54,57 +53,32 @@ public class EntityBindingsMapper { } public static SignedIdentityDocument toSignedIdentityDocument(SignedIdentityDocumentEntity entity) { - if (entity instanceof LegacySignedIdentityDocumentEntity docEntity) { - IdentityDocument doc = new IdentityDocument( - fromDottedString(docEntity.providerUniqueId()), - new AthenzService(docEntity.providerService()), - docEntity.configServerHostname(), - docEntity.instanceHostname(), - docEntity.createdAt(), - docEntity.ipAddresses(), - IdentityType.fromId(docEntity.identityType()), - Optional.ofNullable(docEntity.clusterType()).map(ClusterType::from).orElse(null), - docEntity.ztsUrl(), - Optional.ofNullable(docEntity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), - docEntity.unknownAttributes()); - return new LegacySignedIdentityDocument( - docEntity.signature(), + if (entity instanceof V4SignedIdentityDocumentEntity docEntity) { + return new V4SignedIdentityDocument(docEntity.signature(), docEntity.signingKeyVersion(), - entity.documentVersion(), - doc); - } else if (entity instanceof DefaultSignedIdentityDocumentEntity docEntity) { - return new DefaultSignedIdentityDocument(docEntity.signature(), - docEntity.signingKeyVersion(), - docEntity.documentVersion(), - docEntity.data()); + docEntity.documentVersion(), + docEntity.data()); + } else if (entity instanceof V5SignedIdentityDocumentEntity docEntity) { + return new V5SignedIdentityDocument(docEntity.signature(), + docEntity.signingKeyVersion(), + docEntity.documentVersion(), + docEntity.data()); } else { throw new IllegalArgumentException("Unknown signed identity document type: " + entity.getClass().getName()); } } public static SignedIdentityDocumentEntity toSignedIdentityDocumentEntity(SignedIdentityDocument model) { - if (model instanceof LegacySignedIdentityDocument legacyModel) { - IdentityDocument idDoc = legacyModel.identityDocument(); - return new LegacySignedIdentityDocumentEntity( - legacyModel.signature(), - legacyModel.signingKeyVersion(), - idDoc.providerUniqueId().asDottedString(), - idDoc.providerService().getFullName(), - legacyModel.documentVersion(), - idDoc.configServerHostname(), - idDoc.instanceHostname(), - idDoc.createdAt(), - idDoc.ipAddresses(), - idDoc.identityType().id(), - Optional.ofNullable(idDoc.clusterType()).map(ClusterType::toConfigValue).orElse(null), - idDoc.ztsUrl(), - Optional.ofNullable(idDoc.serviceIdentity()).map(AthenzIdentity::getFullName).orElse(null), - idDoc.unknownAttributes()); - } else if (model instanceof DefaultSignedIdentityDocument defaultModel){ - return new DefaultSignedIdentityDocumentEntity(defaultModel.signature(), - defaultModel.signingKeyVersion(), - defaultModel.documentVersion(), - defaultModel.data()); + if (model instanceof V4SignedIdentityDocument defaultModel) { + return new V4SignedIdentityDocumentEntity(defaultModel.signature(), + defaultModel.v4SigningKeyVersion(), + defaultModel.documentVersion(), + defaultModel.data()); + } else if (model instanceof V5SignedIdentityDocument defaultModel){ + return new V5SignedIdentityDocumentEntity(defaultModel.signature(), + defaultModel.signingKeyVersion(), + defaultModel.documentVersion(), + defaultModel.data()); } else { throw new IllegalArgumentException("Unsupported model type: " + model.getClass().getName()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java deleted file mode 100644 index dfab93b2a28..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api; - -public record LegacySignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, - IdentityDocument identityDocument) implements SignedIdentityDocument { -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index 8ab07d97e74..56b67694af7 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -8,13 +8,14 @@ package com.yahoo.vespa.athenz.identityprovider.api; */ public interface SignedIdentityDocument { - int LEGACY_DEFAULT_DOCUMENT_VERSION = 3; - int DEFAULT_DOCUMENT_VERSION = 4; + int LEGACY_DOCUMENT_VERSION = 4; + int DEFAULT_DOCUMENT_VERSION = 5; - default boolean outdated() { return documentVersion() < LEGACY_DEFAULT_DOCUMENT_VERSION; } + default boolean outdated() { return documentVersion() < DEFAULT_DOCUMENT_VERSION; } IdentityDocument identityDocument(); String signature(); - int signingKeyVersion(); + String signingKeyVersion(); int documentVersion(); + String data(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V4SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V4SignedIdentityDocument.java new file mode 100644 index 00000000000..36836786da3 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V4SignedIdentityDocument.java @@ -0,0 +1,19 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api; + +public record V4SignedIdentityDocument(String signature, int v4SigningKeyVersion, int documentVersion, + String data, IdentityDocument identityDocument) implements SignedIdentityDocument { + + public V4SignedIdentityDocument { + identityDocument = EntityBindingsMapper.fromIdentityDocumentData(data); + } + + public V4SignedIdentityDocument(String signature, int v4SigningKeyVersion, int documentVersion, String data) { + this(signature, v4SigningKeyVersion, documentVersion, data, null); + } + + @Override + public String signingKeyVersion() { + return Integer.toString(v4SigningKeyVersion); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V5SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V5SignedIdentityDocument.java new file mode 100644 index 00000000000..644ca2eafb4 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/V5SignedIdentityDocument.java @@ -0,0 +1,16 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.athenz.identityprovider.api; + +public record V5SignedIdentityDocument(String signature, String signingKeyVersion, int documentVersion, + String data, IdentityDocument identityDocument) implements SignedIdentityDocument { + + + public V5SignedIdentityDocument { + identityDocument = EntityBindingsMapper.fromIdentityDocumentData(data); + } + + public V5SignedIdentityDocument(String signature, String signingKeyVersion, int documentVersion, String data) { + this(signature,signingKeyVersion,documentVersion, data, null); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java deleted file mode 100644 index 647ca474420..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api.bindings; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.net.URI; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * @author bjorncs - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public record LegacySignedIdentityDocumentEntity ( - String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, - String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - String identityType, String clusterType, URI ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) implements SignedIdentityDocumentEntity { - - @JsonCreator - public LegacySignedIdentityDocumentEntity(@JsonProperty("signature") String signature, - @JsonProperty("signing-key-version") int signingKeyVersion, - @JsonProperty("provider-unique-id") String providerUniqueId, - @JsonProperty("provider-service") String providerService, - @JsonProperty("document-version") int documentVersion, - @JsonProperty("configserver-hostname") String configServerHostname, - @JsonProperty("instance-hostname") String instanceHostname, - @JsonProperty("created-at") Instant createdAt, - @JsonProperty("ip-addresses") Set<String> ipAddresses, - @JsonProperty("identity-type") String identityType, - @JsonProperty("cluster-type") String clusterType, - @JsonProperty("zts-url") String ztsUrl, - @JsonProperty("service-identity") String serviceIdentity) { - this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, URI.create(ztsUrl), serviceIdentity, new HashMap<>()); - } - - @JsonProperty("signature") @Override public String signature() { return signature; } - @JsonProperty("signing-key-version") @Override public int signingKeyVersion() { return signingKeyVersion; } - @JsonProperty("provider-unique-id") @Override public String providerUniqueId() { return providerUniqueId; } - @JsonProperty("provider-service") @Override public String providerService() { return providerService; } - @JsonProperty("document-version") @Override public int documentVersion() { return documentVersion; } - @JsonProperty("configserver-hostname") @Override public String configServerHostname() { return configServerHostname; } - @JsonProperty("instance-hostname") @Override public String instanceHostname() { return instanceHostname; } - @JsonProperty("created-at") @Override public Instant createdAt() { return createdAt; } - @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } - @JsonProperty("identity-type") @Override public String identityType() { return identityType; } - @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } - @JsonProperty("zts-url") @Override public URI ztsUrl() { return ztsUrl; } - @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } - @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } - @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index c5c39fb6590..dc2daa530e8 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -56,9 +56,11 @@ class SignedIdentityDocumentEntityTypeResolver implements TypeIdResolver { public JavaType typeFromId(DatabindContext databindContext, String s) throws IOException { try { int version = Integer.parseInt(s); - Class<? extends SignedIdentityDocumentEntity> cls = version <= SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION - ? LegacySignedIdentityDocumentEntity.class - : DefaultSignedIdentityDocumentEntity.class; + Class<? extends SignedIdentityDocumentEntity> cls = switch (version) { + case SignedIdentityDocument.LEGACY_DOCUMENT_VERSION -> V4SignedIdentityDocumentEntity.class; + case SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION -> V5SignedIdentityDocumentEntity.class; + default -> throw new IllegalArgumentException("Unknown document version: " + version); + }; return TypeFactory.defaultInstance().constructSpecializedType(javaType,cls); } catch (NumberFormatException e) { throw new IllegalArgumentException("Unable to deserialize document with version: \"%s\"".formatted(s)); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V4SignedIdentityDocumentEntity.java index 74fd43feb35..9c6af38377a 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V4SignedIdentityDocumentEntity.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.athenz.identityprovider.api.bindings; import com.fasterxml.jackson.annotation.JsonProperty; -public record DefaultSignedIdentityDocumentEntity( +public record V4SignedIdentityDocumentEntity( @JsonProperty("signature") String signature, @JsonProperty("signing-key-version") int signingKeyVersion, @JsonProperty("document-version") int documentVersion, diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V5SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V5SignedIdentityDocumentEntity.java new file mode 100644 index 00000000000..eece4b5f066 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/V5SignedIdentityDocumentEntity.java @@ -0,0 +1,12 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api.bindings; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record V5SignedIdentityDocumentEntity( + @JsonProperty("signature") String signature, + @JsonProperty("signing-key-version") String signingKeyVersion, + @JsonProperty("document-version") int documentVersion, + @JsonProperty("data") String data) + implements SignedIdentityDocumentEntity { +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java deleted file mode 100644 index 63c966004e5..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.client; - -import com.yahoo.container.core.identity.IdentityConfig; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; -import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.vespa.athenz.utils.SiaUtils; -import com.yahoo.vespa.defaults.Defaults; - -import javax.net.ssl.SSLContext; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.time.Duration; -import java.util.Optional; - -import static java.util.Collections.singleton; - -/** - * A service that provides method for initially registering the instance and refreshing it. - * - * @author bjorncs - */ -class AthenzCredentialsService { - private static final Duration EXPIRATION_MARGIN = Duration.ofDays(2); - private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); - private static final Path IDENTITY_DOCUMENT_FILE = VESPA_SIA_DIRECTORY.resolve("vespa-tenant-identity-document.json"); - - private final AthenzService tenantIdentity; - private final URI configserverEndpoint; - private final URI ztsEndpoint; - private final AthenzService configserverIdentity; - private final ServiceIdentityProvider nodeIdentityProvider; - private final String hostname; - private final CsrGenerator csrGenerator; - private final Clock clock; - - AthenzCredentialsService(IdentityConfig identityConfig, - ServiceIdentityProvider nodeIdentityProvider, - String hostname, - Clock clock) { - this.tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service()); - this.configserverEndpoint = URI.create("https://" + identityConfig.loadBalancerAddress() + ":4443"); - this.ztsEndpoint = URI.create(identityConfig.ztsUrl()); - this.configserverIdentity = new AthenzService(identityConfig.configserverIdentityName()); - this.nodeIdentityProvider = nodeIdentityProvider; - this.hostname = hostname; - this.csrGenerator = new CsrGenerator(identityConfig.athenzDnsSuffix(), identityConfig.configserverIdentityName()); - this.clock = clock; - } - - Path certificatePath() { return SiaUtils.getCertificateFile(VESPA_SIA_DIRECTORY, tenantIdentity); } - Path privateKeyPath() { return SiaUtils.getPrivateKeyFile(VESPA_SIA_DIRECTORY, tenantIdentity); } - - AthenzCredentials registerInstance() { - Optional<AthenzCredentials> athenzCredentialsFromDisk = tryReadCredentialsFromDisk(); - if (athenzCredentialsFromDisk.isPresent()) { - return athenzCredentialsFromDisk.get(); - } - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - IdentityDocumentClient identityDocumentClient = createIdentityDocumentClient(); - // Use legacy version for now. - SignedIdentityDocument signedDocument = identityDocumentClient.getTenantIdentityDocument(hostname, SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION).orElseThrow(); - IdentityDocument document = signedDocument.identityDocument(); - Pkcs10Csr csr = csrGenerator.generateInstanceCsr( - tenantIdentity, - document.providerUniqueId(), - document.ipAddresses(), - document.clusterType(), - keyPair); - - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withIdentityProvider(nodeIdentityProvider).build()) { - InstanceIdentity instanceIdentity = - ztsClient.registerInstance( - configserverIdentity, - tenantIdentity, - EntityBindingsMapper.toAttestationData(signedDocument), - csr); - X509Certificate certificate = instanceIdentity.certificate(); - writeCredentialsToDisk(keyPair.getPrivate(), certificate, signedDocument); - return new AthenzCredentials(certificate, keyPair, signedDocument); - } - } - - AthenzCredentials updateCredentials(SignedIdentityDocument signedDocument, SSLContext sslContext) { - KeyPair newKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - IdentityDocument document = signedDocument.identityDocument(); - Pkcs10Csr csr = csrGenerator.generateInstanceCsr( - tenantIdentity, - document.providerUniqueId(), - document.ipAddresses(), - document.clusterType(), - newKeyPair); - - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withSslContext(sslContext).build()) { - InstanceIdentity instanceIdentity = - ztsClient.refreshInstance( - configserverIdentity, - tenantIdentity, - document.providerUniqueId().asDottedString(), - csr); - X509Certificate certificate = instanceIdentity.certificate(); - writeCredentialsToDisk(newKeyPair.getPrivate(), certificate, signedDocument); - return new AthenzCredentials(certificate, newKeyPair, signedDocument); - } - } - - private Optional<AthenzCredentials> tryReadCredentialsFromDisk() { - Optional<PrivateKey> privateKey = SiaUtils.readPrivateKeyFile(VESPA_SIA_DIRECTORY, tenantIdentity); - if (privateKey.isEmpty()) return Optional.empty(); - Optional<X509Certificate> certificate = SiaUtils.readCertificateFile(VESPA_SIA_DIRECTORY, tenantIdentity); - if (certificate.isEmpty()) return Optional.empty(); - if (isExpired(certificate.get())) { - return Optional.empty(); - } - if (Files.notExists(IDENTITY_DOCUMENT_FILE)) return Optional.empty(); - SignedIdentityDocument signedIdentityDocument = EntityBindingsMapper.readSignedIdentityDocumentFromFile(IDENTITY_DOCUMENT_FILE); - KeyPair keyPair = new KeyPair(KeyUtils.extractPublicKey(privateKey.get()), privateKey.get()); - return Optional.of(new AthenzCredentials(certificate.get(), keyPair, signedIdentityDocument)); - } - - private boolean isExpired(X509Certificate certificate) { - return clock.instant().isAfter(certificate.getNotAfter().toInstant().minus(EXPIRATION_MARGIN)); - } - - private void writeCredentialsToDisk(PrivateKey privateKey, - X509Certificate certificate, - SignedIdentityDocument identityDocument) { - SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, tenantIdentity, privateKey); - SiaUtils.writeCertificateFile(VESPA_SIA_DIRECTORY, tenantIdentity, certificate); - EntityBindingsMapper.writeSignedIdentityDocumentToFile(IDENTITY_DOCUMENT_FILE, identityDocument); - } - - private DefaultIdentityDocumentClient createIdentityDocumentClient() { - return new DefaultIdentityDocumentClient( - configserverEndpoint, - nodeIdentityProvider, - new AthenzIdentityVerifier(singleton(configserverIdentity))); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderProvider.java index 30ae4fff59d..dec919cada0 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderProvider.java @@ -7,24 +7,17 @@ import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.jdisc.Metric; import javax.inject.Inject; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; /** * @author olaa */ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityProvider> { - private final Path NODE_ADMIN_MANAGED_IDENTITY_DOCUMENT = Paths.get("/var/lib/sia/vespa-tenant-identity-document.json"); private final AthenzIdentityProvider athenzIdentityProvider; @Inject public AthenzIdentityProviderProvider(IdentityConfig config, Metric metric) { - if (Files.exists(NODE_ADMIN_MANAGED_IDENTITY_DOCUMENT)) - athenzIdentityProvider = new AthenzIdentityProviderImpl(config, metric); - else - athenzIdentityProvider = new LegacyAthenzIdentityProviderImpl(config, metric); + athenzIdentityProvider = new AthenzIdentityProviderImpl(config, metric); } @Override diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java index fd2cefbc93e..392faaaa339 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java @@ -2,27 +2,15 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.security.SignatureUtils; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.DefaultSignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.LegacySignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.V4SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.security.PrivateKey; import java.security.PublicKey; import java.security.Signature; -import java.security.SignatureException; -import java.time.Instant; import java.util.Base64; -import java.util.Set; -import java.util.TreeSet; -import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -44,91 +32,14 @@ public class IdentityDocumentSigner { } } - public String generateLegacySignature(IdentityDocument doc, PrivateKey privateKey) { - return generateSignature(doc.providerUniqueId(), doc.providerService(), doc.configServerHostname(), - doc.instanceHostname(), doc.createdAt(), doc.ipAddresses(), doc.identityType(), privateKey, doc.serviceIdentity()); - } - - // Cluster type is ignored due to old Vespa versions not forwarding unknown fields in signed identity document - private String generateSignature(VespaUniqueInstanceId providerUniqueId, - AthenzIdentity providerService, - String configServerHostname, - String instanceHostname, - Instant createdAt, - Set<String> ipAddresses, - IdentityType identityType, - PrivateKey privateKey, - AthenzIdentity serviceIdentity) { - try { - Signature signer = SignatureUtils.createSigner(privateKey); - signer.initSign(privateKey); - writeToSigner( - signer, providerUniqueId, providerService, configServerHostname, instanceHostname, createdAt, - ipAddresses, identityType); - writeToSigner(signer, serviceIdentity); - byte[] signature = signer.sign(); - return Base64.getEncoder().encodeToString(signature); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - } - public boolean hasValidSignature(SignedIdentityDocument doc, PublicKey publicKey) { - if (doc instanceof LegacySignedIdentityDocument signedDoc) { - return validateLegacySignature(signedDoc, publicKey); - } else if (doc instanceof DefaultSignedIdentityDocument signedDoc) { - try { - Signature signer = SignatureUtils.createVerifier(publicKey); - signer.initVerify(publicKey); - signer.update(signedDoc.data().getBytes(UTF_8)); - return signer.verify(Base64.getDecoder().decode(doc.signature())); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - } else { - throw new IllegalArgumentException("Unknown identity document type: " + doc.getClass().getName()); - } - } - - private boolean validateLegacySignature(SignedIdentityDocument doc, PublicKey publicKey) { try { - IdentityDocument iddoc = doc.identityDocument(); Signature signer = SignatureUtils.createVerifier(publicKey); signer.initVerify(publicKey); - writeToSigner( - signer, iddoc.providerUniqueId(), iddoc.providerService(), iddoc.configServerHostname(), - iddoc.instanceHostname(), iddoc.createdAt(), iddoc.ipAddresses(), iddoc.identityType()); - if (doc.documentVersion() >= LEGACY_DEFAULT_DOCUMENT_VERSION) { - writeToSigner(signer, iddoc.serviceIdentity()); - } + signer.update(doc.data().getBytes(UTF_8)); return signer.verify(Base64.getDecoder().decode(doc.signature())); } catch (GeneralSecurityException e) { throw new RuntimeException(e); } } - - private static void writeToSigner(Signature signer, - VespaUniqueInstanceId providerUniqueId, - AthenzIdentity providerService, - String configServerHostname, - String instanceHostname, - Instant createdAt, - Set<String> ipAddresses, - IdentityType identityType) throws SignatureException { - signer.update(providerUniqueId.asDottedString().getBytes(UTF_8)); - signer.update(providerService.getFullName().getBytes(UTF_8)); - signer.update(configServerHostname.getBytes(UTF_8)); - signer.update(instanceHostname.getBytes(UTF_8)); - ByteBuffer timestampAsBuffer = ByteBuffer.allocate(Long.BYTES); - timestampAsBuffer.putLong(createdAt.toEpochMilli()); - signer.update(timestampAsBuffer.array()); - for (String ipAddress : new TreeSet<>(ipAddresses)) { - signer.update(ipAddress.getBytes(UTF_8)); - } - signer.update(identityType.id().getBytes(UTF_8)); - } - - private static void writeToSigner(Signature signer, AthenzIdentity serviceIdentity) throws SignatureException{ - signer.update(serviceIdentity.getFullName().getBytes(UTF_8)); - } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java deleted file mode 100644 index 34324ef18e6..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.client; - -import ai.vespa.metrics.ContainerMetrics; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.annotation.Inject; -import com.yahoo.container.core.identity.IdentityConfig; -import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; -import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; -import com.yahoo.jdisc.Metric; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.MutableX509KeyManager; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.X509CertificateWithKey; -import com.yahoo.vespa.athenz.api.AthenzAccessToken; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.api.ZToken; -import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; -import com.yahoo.vespa.athenz.utils.SiaUtils; -import com.yahoo.vespa.defaults.Defaults; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.X509ExtendedKeyManager; -import java.net.URI; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.security.KeyStoreType.PKCS12; - -/** - * A {@link AthenzIdentityProvider} / {@link ServiceIdentityProvider} component that provides the tenant identity. - * - * @author mortent - * @author bjorncs - */ -// This class should probably not implement ServiceIdentityProvider, -// as that interface is intended for providing the node's identity, not the tenant's application identity. -public final class LegacyAthenzIdentityProviderImpl extends AbstractComponent implements AthenzIdentityProvider, ServiceIdentityProvider { - - private static final Logger log = Logger.getLogger(LegacyAthenzIdentityProviderImpl.class.getName()); - - // TODO Make some of these values configurable through config. Match requested expiration of register/update requests. - // TODO These should match the requested expiration - static final Duration UPDATE_PERIOD = Duration.ofDays(1); - static final Duration AWAIT_TERMINTATION_TIMEOUT = Duration.ofSeconds(90); - private final static Duration ROLE_SSL_CONTEXT_EXPIRY = Duration.ofHours(2); - // TODO CMS expects 10min or less token ttl. Use 10min default until we have configurable expiry - private final static Duration ROLE_TOKEN_EXPIRY = Duration.ofMinutes(10); - - // TODO Make path to trust store paths config - private static final Path CLIENT_TRUST_STORE = Paths.get("/opt/yahoo/share/ssl/certs/yahoo_certificate_bundle.pem"); - private static final Path ATHENZ_TRUST_STORE = Paths.get("/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"); - - public static final String CERTIFICATE_EXPIRY_METRIC_NAME = ContainerMetrics.ATHENZ_TENANT_CERT_EXPIRY_SECONDS.baseName(); - - private volatile AthenzCredentials credentials; - private final Metric metric; - private final Path trustStore; - private final AthenzCredentialsService athenzCredentialsService; - private final ScheduledExecutorService scheduler; - private final Clock clock; - private final AthenzService identity; - private final URI ztsEndpoint; - - private final MutableX509KeyManager identityKeyManager = new MutableX509KeyManager(); - private final SSLContext identitySslContext; - private final LoadingCache<AthenzRole, X509Certificate> roleSslCertCache; - private final Map<AthenzRole, MutableX509KeyManager> roleKeyManagerCache; - private final LoadingCache<AthenzRole, ZToken> roleSpecificRoleTokenCache; - private final LoadingCache<AthenzDomain, ZToken> domainSpecificRoleTokenCache; - private final LoadingCache<AthenzDomain, AthenzAccessToken> domainSpecificAccessTokenCache; - private final LoadingCache<List<AthenzRole>, AthenzAccessToken> roleSpecificAccessTokenCache; - private final CsrGenerator csrGenerator; - - @Inject - public LegacyAthenzIdentityProviderImpl(IdentityConfig config, Metric metric) { - this(config, - metric, - CLIENT_TRUST_STORE, - new AthenzCredentialsService(config, - createNodeIdentityProvider(config), - Defaults.getDefaults().vespaHostname(), - Clock.systemUTC()), - new ScheduledThreadPoolExecutor(1), - Clock.systemUTC()); - } - - // Test only - LegacyAthenzIdentityProviderImpl(IdentityConfig config, - Metric metric, - Path trustStore, - AthenzCredentialsService athenzCredentialsService, - ScheduledExecutorService scheduler, - Clock clock) { - this.metric = metric; - this.trustStore = trustStore; - this.athenzCredentialsService = athenzCredentialsService; - this.scheduler = scheduler; - this.clock = clock; - this.identity = new AthenzService(config.domain(), config.service()); - this.ztsEndpoint = URI.create(config.ztsUrl()); - roleSslCertCache = crateAutoReloadableCache(ROLE_SSL_CONTEXT_EXPIRY, this::requestRoleCertificate, this.scheduler); - roleKeyManagerCache = new HashMap<>(); - roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); - domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); - domainSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken); - roleSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken); - this.csrGenerator = new CsrGenerator(config.athenzDnsSuffix(), config.configserverIdentityName()); - this.identitySslContext = createIdentitySslContext(identityKeyManager, trustStore); - registerInstance(); - } - - private static <KEY, VALUE> LoadingCache<KEY, VALUE> createCache(Duration expiry, Function<KEY, VALUE> cacheLoader) { - return CacheBuilder.newBuilder() - .refreshAfterWrite(expiry.dividedBy(2).toMinutes(), TimeUnit.MINUTES) - .expireAfterWrite(expiry.toMinutes(), TimeUnit.MINUTES) - .build(new CacheLoader<KEY, VALUE>() { - @Override - public VALUE load(KEY key) { - return cacheLoader.apply(key); - } - }); - } - - private static <KEY, VALUE> LoadingCache<KEY, VALUE> crateAutoReloadableCache(Duration expiry, Function<KEY, VALUE> cacheLoader, ScheduledExecutorService scheduler) { - LoadingCache<KEY, VALUE> cache = createCache(expiry, cacheLoader); - - // The cache above will reload it's contents if and only if a request for the key is made. Scheduling - // a cache reloader to reload all keys in this cache. - scheduler.scheduleAtFixedRate(() -> { cache.asMap().keySet().forEach(cache::getUnchecked);}, - expiry.dividedBy(4).toMinutes(), - expiry.dividedBy(4).toMinutes(), - TimeUnit.MINUTES); - return cache; - } - - private static SSLContext createIdentitySslContext(X509ExtendedKeyManager keyManager, Path trustStore) { - return new SslContextBuilder() - .withKeyManager(keyManager) - .withTrustStore(trustStore) - .build(); - } - - private void registerInstance() { - try { - updateIdentityCredentials(this.athenzCredentialsService.registerInstance()); - this.scheduler.scheduleAtFixedRate(this::refreshCertificate, UPDATE_PERIOD.toMinutes(), UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); - this.scheduler.scheduleAtFixedRate(this::reportMetrics, 0, 5, TimeUnit.MINUTES); - } catch (Throwable t) { - throw new AthenzIdentityProviderException("Could not retrieve Athenz credentials", t); - } - } - - @Override - public AthenzService identity() { - return identity; - } - - @Override - public String domain() { - return identity.getDomain().getName(); - } - - @Override - public String service() { - return identity.getName(); - } - - @Override - public SSLContext getIdentitySslContext() { - return identitySslContext; - } - - @Override - public X509CertificateWithKey getIdentityCertificateWithKey() { - AthenzCredentials copy = this.credentials; - return new X509CertificateWithKey(copy.getCertificate(), copy.getKeyPair().getPrivate()); - } - - @Override public Path certificatePath() { return athenzCredentialsService.certificatePath(); } - - @Override public Path privateKeyPath() { return athenzCredentialsService.privateKeyPath(); } - - @Override - public SSLContext getRoleSslContext(String domain, String role) { - try { - AthenzRole athenzRole = new AthenzRole(new AthenzDomain(domain), role); - // Make sure to request a certificate which triggers creating a new key manager for this role - X509Certificate x509Certificate = getRoleCertificate(athenzRole); - MutableX509KeyManager keyManager = roleKeyManagerCache.get(athenzRole); - return new SslContextBuilder() - .withKeyManager(keyManager) - .withTrustStore(trustStore) - .build(); - } catch (Exception e) { - throw new AthenzIdentityProviderException("Could not retrieve role certificate: " + e.getMessage(), e); - } - } - - @Override - public String getRoleToken(String domain) { - try { - return domainSpecificRoleTokenCache.get(new AthenzDomain(domain)).getRawToken(); - } catch (Exception e) { - throw new AthenzIdentityProviderException("Could not retrieve role token: " + e.getMessage(), e); - } - } - - @Override - public String getRoleToken(String domain, String role) { - try { - return roleSpecificRoleTokenCache.get(new AthenzRole(domain, role)).getRawToken(); - } catch (Exception e) { - throw new AthenzIdentityProviderException("Could not retrieve role token: " + e.getMessage(), e); - } - } - - @Override - public String getAccessToken(String domain) { - try { - return domainSpecificAccessTokenCache.get(new AthenzDomain(domain)).value(); - } catch (Exception e) { - throw new AthenzIdentityProviderException("Could not retrieve access token: " + e.getMessage(), e); - } - } - - @Override - public String getAccessToken(String domain, List<String> roles) { - try { - List<AthenzRole> roleList = roles.stream() - .map(roleName -> new AthenzRole(domain, roleName)) - .toList(); - return roleSpecificAccessTokenCache.get(roleList).value(); - } catch (Exception e) { - throw new AthenzIdentityProviderException("Could not retrieve access token: " + e.getMessage(), e); - } - } - - @Override - public String getAccessToken(String domain, List<String> roles, List<String> proxyPrincipal) { - throw new UnsupportedOperationException("Not implemented in legacy client"); - } - - @Override - public PrivateKey getPrivateKey() { - return credentials.getKeyPair().getPrivate(); - } - - @Override - public Path trustStorePath() { - return trustStore; - } - - @Override - public List<X509Certificate> getIdentityCertificate() { - return List.of(credentials.getCertificate()); - } - - @Override - public X509Certificate getRoleCertificate(String domain, String role) { - return getRoleCertificate(new AthenzRole(new AthenzDomain(domain), role)); - } - - private X509Certificate getRoleCertificate(AthenzRole athenzRole) { - try { - return roleSslCertCache.get(athenzRole); - } catch (Exception e) { - throw new AthenzIdentityProviderException("Could not retrieve role certificate: " + e.getMessage(), e); - } - } - - private void updateIdentityCredentials(AthenzCredentials credentials) { - this.credentials = credentials; - this.identityKeyManager.updateKeystore( - KeyStoreBuilder.withType(PKCS12) - .withKeyEntry("default", credentials.getKeyPair().getPrivate(), credentials.getCertificate()) - .build(), - new char[0]); - } - - private X509Certificate requestRoleCertificate(AthenzRole role) { - var doc = credentials.getIdentityDocument().identityDocument(); - Pkcs10Csr csr = csrGenerator.generateRoleCsr( - identity, role, doc.providerUniqueId(), doc.clusterType(), credentials.getKeyPair()); - try (ZtsClient client = createZtsClient()) { - X509Certificate roleCertificate = client.getRoleCertificate(role, csr); - updateRoleKeyManager(role, roleCertificate); - log.info(String.format("Requester role certificate for role %s, expires: %s", role.toResourceNameString(), roleCertificate.getNotAfter().toInstant().toString())); - return roleCertificate; - } - } - - private void updateRoleKeyManager(AthenzRole role, X509Certificate certificate) { - MutableX509KeyManager keyManager = roleKeyManagerCache.computeIfAbsent(role, r -> new MutableX509KeyManager()); - keyManager.updateKeystore( - KeyStoreBuilder.withType(PKCS12) - .withKeyEntry("default", credentials.getKeyPair().getPrivate(), certificate) - .build(), - new char[0]); - } - - private ZToken createRoleToken(AthenzRole athenzRole) { - try (ZtsClient client = createZtsClient()) { - return client.getRoleToken(athenzRole, ROLE_TOKEN_EXPIRY); - } - } - - private ZToken createRoleToken(AthenzDomain domain) { - try (ZtsClient client = createZtsClient()) { - return client.getRoleToken(domain, ROLE_TOKEN_EXPIRY); - } - } - - private AthenzAccessToken createAccessToken(AthenzDomain domain) { - try (ZtsClient client = createZtsClient()) { - return client.getAccessToken(domain); - } - } - - private AthenzAccessToken createAccessToken(List<AthenzRole> roles) { - try (ZtsClient client = createZtsClient()) { - return client.getAccessToken(roles); - } - } - - private DefaultZtsClient createZtsClient() { - return new DefaultZtsClient.Builder(ztsEndpoint).withSslContext(getIdentitySslContext()).build(); - } - - @Override - public void deconstruct() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(AWAIT_TERMINTATION_TIMEOUT.getSeconds(), TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config) { - return new SiaIdentityProvider( - new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, CLIENT_TRUST_STORE, false); - } - - private boolean isExpired(AthenzCredentials credentials) { - return clock.instant().isAfter(getExpirationTime(credentials)); - } - - private static Instant getExpirationTime(AthenzCredentials credentials) { - return credentials.getCertificate().getNotAfter().toInstant(); - } - - void refreshCertificate() { - try { - updateIdentityCredentials(isExpired(credentials) - ? athenzCredentialsService.registerInstance() - : athenzCredentialsService.updateCredentials(credentials.getIdentityDocument(), identitySslContext)); - } catch (Throwable t) { - log.log(Level.WARNING, "Failed to update credentials: " + t.getMessage(), t); - } - } - - void reportMetrics() { - try { - Instant expirationTime = getExpirationTime(credentials); - Duration remainingLifetime = Duration.between(clock.instant(), expirationTime); - Metric.Context dimensions = metric.createContext(Map.of("implementation", this.getClassName())); - metric.set(CERTIFICATE_EXPIRY_METRIC_NAME, remainingLifetime.getSeconds(), dimensions); - } catch (Throwable t) { - log.log(Level.WARNING, "Failed to update metrics: " + t.getMessage(), t); - } - } -} - diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ServiceIdentityProviderProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ServiceIdentityProviderProvider.java index d0267d406ce..5dd49206c6d 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ServiceIdentityProviderProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ServiceIdentityProviderProvider.java @@ -22,7 +22,6 @@ public class ServiceIdentityProviderProvider implements Provider<ServiceIdentity @Override public ServiceIdentityProvider get() { if (athenzIdentityProvider instanceof AthenzIdentityProviderImpl impl) return impl; - if (athenzIdentityProvider instanceof LegacyAthenzIdentityProviderImpl legacyImpl) return legacyImpl; return null; } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java index 19a81691b76..5ca6a53a4c7 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java @@ -49,8 +49,7 @@ public class SiaIdentityProviderTest { new AthenzService("domain", "service-name"), keyFile.toPath(), certificateFile.toPath(), - trustStoreFile.toPath(), - false); + trustStoreFile.toPath()); assertNotNull(provider.getIdentitySslContext()); } @@ -73,8 +72,7 @@ public class SiaIdentityProviderTest { new AthenzService("domain", "service-name"), keyFile.toPath(), certificateFile.toPath(), - trustStoreFile.toPath(), - false); + trustStoreFile.toPath()); assertNotNull(provider.getIdentitySslContext()); } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java index 297f0c904d9..377aee22ab1 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java @@ -7,12 +7,8 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Base64; -import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author bjorncs @@ -20,38 +16,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class EntityBindingsMapperTest { @Test - public void legacy_persists_unknown_json_members() throws IOException { - var originalJson = - """ - { - "signature": "sig", - "signing-key-version": 0, - "provider-unique-id": "0.cluster.instance.app.tenant.us-west-1.test.node", - "provider-service": "domain.service", - "document-version": 2, - "configserver-hostname": "cfg", - "instance-hostname": "host", - "created-at": 12345.0, - "ip-addresses": [], - "identity-type": "node", - "cluster-type": "admin", - "zts-url": "https://zts.url/", - "unknown-string": "string-value", - "unknown-object": { "member-in-unknown-object": 123 } - } - """; - var entity = EntityBindingsMapper.fromString(originalJson); - assertInstanceOf(LegacySignedIdentityDocument.class, entity); - assertEquals(2, entity.identityDocument().unknownAttributes().size(), entity.identityDocument().unknownAttributes().toString()); - var json = EntityBindingsMapper.toAttestationData(entity); - - var expectedMemberInJson = "member-in-unknown-object"; - assertTrue(json.contains(expectedMemberInJson), - () -> "Expected JSON to contain '%s', but got \n'%s'".formatted(expectedMemberInJson, json)); - assertEquals(EntityBindingsMapper.mapper.readTree(originalJson), EntityBindingsMapper.mapper.readTree(json)); - } - - @Test public void reads_unknown_json_members() throws IOException { var iddoc = """ { diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index 2532a394f4e..3479350cf26 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -6,11 +6,10 @@ import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.DefaultSignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.V4SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.LegacySignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import org.junit.jupiter.api.Test; @@ -23,8 +22,6 @@ import java.util.List; import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.TENANT; import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; -import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -47,21 +44,6 @@ public class IdentityDocumentSignerTest { private static final AthenzIdentity serviceIdentity = new AthenzService("vespa", "node"); @Test - void legacy_generates_and_validates_signature() { - IdentityDocumentSigner signer = new IdentityDocumentSigner(); - IdentityDocument identityDocument = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); - String signature = - signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); - - SignedIdentityDocument signedIdentityDocument = new LegacySignedIdentityDocument( - signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); - - assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); - } - - @Test void generates_and_validates_signature() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); IdentityDocument identityDocument = new IdentityDocument( @@ -71,46 +53,9 @@ public class IdentityDocumentSignerTest { String signature = signer.generateSignature(data, keyPair.getPrivate()); - SignedIdentityDocument signedIdentityDocument = new DefaultSignedIdentityDocument( + SignedIdentityDocument signedIdentityDocument = new V4SignedIdentityDocument( signature, KEY_VERSION, DEFAULT_DOCUMENT_VERSION, data); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); } - - @Test - void legacy_ignores_cluster_type_and_zts_url() { - IdentityDocumentSigner signer = new IdentityDocumentSigner(); - IdentityDocument identityDocument = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); - IdentityDocument withoutIgnoredFields = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, null, null, serviceIdentity); - - String signature = - signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); - - var docWithoutIgnoredFields = new LegacySignedIdentityDocument( - signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, withoutIgnoredFields); - var docWithIgnoredFields = new LegacySignedIdentityDocument( - signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); - - assertTrue(signer.hasValidSignature(docWithoutIgnoredFields, keyPair.getPublic())); - assertEquals(docWithIgnoredFields.signature(), docWithoutIgnoredFields.signature()); - } - - @Test - void validates_signature_for_new_and_old_versions() { - IdentityDocumentSigner signer = new IdentityDocumentSigner(); - IdentityDocument identityDocument = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); - String signature = - signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); - - SignedIdentityDocument signedIdentityDocument = new LegacySignedIdentityDocument( - signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); - - assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); - } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImplTest.java deleted file mode 100644 index 90853ff7cfa..00000000000 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImplTest.java +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.client; - -import com.yahoo.container.core.identity.IdentityConfig; -import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; -import com.yahoo.jdisc.Metric; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyStoreUtils; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.Pkcs10CsrBuilder; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.test.ManualClock; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import javax.security.auth.x500.X500Principal; - -import java.io.File; -import java.io.IOException; -import java.math.BigInteger; -import java.nio.file.Path; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Date; -import java.util.concurrent.ScheduledExecutorService; -import java.util.function.Supplier; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author mortent - * @author bjorncs - */ -public class LegacyAthenzIdentityProviderImplTest { - - @TempDir - public File tempDir; - - public static final Duration certificateValidity = Duration.ofDays(30); - - private static final IdentityConfig IDENTITY_CONFIG = - new IdentityConfig(new IdentityConfig.Builder() - .service("tenantService") - .domain("tenantDomain") - .nodeIdentityName("vespa.tenant") - .configserverIdentityName("vespa.configserver") - .loadBalancerAddress("cfg") - .ztsUrl("https:localhost:4443/zts/v1") - .athenzDnsSuffix("dev-us-north-1.vespa.cloud")); - - private final KeyPair caKeypair = KeyUtils.generateKeypair(KeyAlgorithm.EC); - private Path trustStoreFile; - private X509Certificate caCertificate; - - @BeforeEach - public void createTrustStoreFile() throws IOException { - caCertificate = X509CertificateBuilder - .fromKeypair( - caKeypair, - new X500Principal("CN=mydummyca"), - Instant.EPOCH, - Instant.EPOCH.plus(10000, ChronoUnit.DAYS), - SignatureAlgorithm.SHA256_WITH_ECDSA, - BigInteger.ONE) - .build(); - trustStoreFile = File.createTempFile("junit", null, tempDir).toPath(); - KeyStoreUtils.writeKeyStoreToFile( - KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry("default", caKeypair.getPrivate(), caCertificate) - .build(), - trustStoreFile); - } - - @Test - void component_creation_fails_when_credentials_not_found() { - assertThrows(AthenzIdentityProviderException.class, () -> { - AthenzCredentialsService credentialService = mock(AthenzCredentialsService.class); - when(credentialService.registerInstance()) - .thenThrow(new RuntimeException("athenz unavailable")); - - new LegacyAthenzIdentityProviderImpl(IDENTITY_CONFIG, mock(Metric.class), trustStoreFile, credentialService, mock(ScheduledExecutorService.class), new ManualClock(Instant.EPOCH)); - }); - } - - @Test - void metrics_updated_on_refresh() { - ManualClock clock = new ManualClock(Instant.EPOCH); - Metric metric = mock(Metric.class); - - AthenzCredentialsService athenzCredentialsService = mock(AthenzCredentialsService.class); - - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC); - X509Certificate certificate = getCertificate(keyPair, getExpirationSupplier(clock)); - - when(athenzCredentialsService.registerInstance()) - .thenReturn(new AthenzCredentials(certificate, keyPair, null)); - - when(athenzCredentialsService.updateCredentials(any(), any())) - .thenThrow(new RuntimeException("#1")) - .thenThrow(new RuntimeException("#2")) - .thenReturn(new AthenzCredentials(certificate, keyPair, null)); - - LegacyAthenzIdentityProviderImpl identityProvider = - new LegacyAthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, trustStoreFile, athenzCredentialsService, mock(ScheduledExecutorService.class), clock); - - identityProvider.reportMetrics(); - verify(metric).set(eq(LegacyAthenzIdentityProviderImpl.CERTIFICATE_EXPIRY_METRIC_NAME), eq(certificateValidity.getSeconds()), any()); - - // Advance 1 day, refresh fails, cert is 1 day old - clock.advance(Duration.ofDays(1)); - identityProvider.refreshCertificate(); - identityProvider.reportMetrics(); - verify(metric).set(eq(LegacyAthenzIdentityProviderImpl.CERTIFICATE_EXPIRY_METRIC_NAME), eq(certificateValidity.minus(Duration.ofDays(1)).getSeconds()), any()); - - // Advance 1 more day, refresh fails, cert is 2 days old - clock.advance(Duration.ofDays(1)); - identityProvider.refreshCertificate(); - identityProvider.reportMetrics(); - verify(metric).set(eq(LegacyAthenzIdentityProviderImpl.CERTIFICATE_EXPIRY_METRIC_NAME), eq(certificateValidity.minus(Duration.ofDays(2)).getSeconds()), any()); - - // Advance 1 more day, refresh succeds, cert is new - clock.advance(Duration.ofDays(1)); - identityProvider.refreshCertificate(); - identityProvider.reportMetrics(); - verify(metric).set(eq(LegacyAthenzIdentityProviderImpl.CERTIFICATE_EXPIRY_METRIC_NAME), eq(certificateValidity.getSeconds()), any()); - - } - - private Supplier<Date> getExpirationSupplier(ManualClock clock) { - return () -> new Date(clock.instant().plus(certificateValidity).toEpochMilli()); - } - - private X509Certificate getCertificate(KeyPair keyPair, Supplier<Date> expiry) { - Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=dummy"), keyPair, SignatureAlgorithm.SHA256_WITH_ECDSA) - .build(); - return X509CertificateBuilder - .fromCsr(csr, - caCertificate.getSubjectX500Principal(), - Instant.EPOCH, - expiry.get().toInstant(), - caKeypair.getPrivate(), - SignatureAlgorithm.SHA256_WITH_ECDSA, - BigInteger.ONE) - .build(); - } - -} diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 7519f3a5211..65d842d287b 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -18,6 +18,7 @@ com.fasterxml.jackson.core:jackson-databind:${jackson-databind.vespa.version} com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson2.vespa.version} com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.vespa.version} com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.vespa.version} +com.fasterxml.woodstox:woodstox-core:${woodstox.vespa.version} com.github.luben:zstd-jni:${luben.zstd.vespa.version} com.google.code.findbugs:jsr305:${findbugs.vespa.version} com.google.errorprone:error_prone_annotations:${error-prone-annotations.vespa.version} @@ -74,7 +75,6 @@ io.prometheus:simpleclient_tracer_otel:${prometheus.client.vespa.version} io.prometheus:simpleclient_tracer_otel_agent:${prometheus.client.vespa.version} jakarta.inject:jakarta.inject-api:${jakarta.inject.vespa.version} javax.activation:javax.activation-api:${javax.activation-api.vespa.version} -javax.annotation:javax.annotation-api:${javax.annotation.vespa.version} javax.inject:javax.inject:${javax.inject.vespa.version} javax.servlet:javax.servlet-api:${javax.servlet-api.vespa.version} javax.ws.rs:javax.ws.rs-api:${javax.ws.rs-api.vespa.version} @@ -109,6 +109,8 @@ org.apache.maven.plugin-tools:maven-plugin-annotations:${maven-plugin-tools.vesp org.apache.maven.plugins:maven-jar-plugin:${maven-jar-plugin.vespa.version} org.apache.maven.shared:file-management:3.1.0 org.apache.maven.wagon:wagon-provider-api:${maven-wagon.vespa.version} +org.apache.maven:maven-api-meta:${maven-xml-impl.vespa.version} +org.apache.maven:maven-api-xml:${maven-xml-impl.vespa.version} org.apache.maven:maven-archiver:${maven-archiver.vespa.version} org.apache.maven:maven-artifact-manager:2.2.1 org.apache.maven:maven-artifact:${maven-core.vespa.version} @@ -119,6 +121,7 @@ org.apache.maven:maven-profile:2.2.1 org.apache.maven:maven-project:2.2.1 org.apache.maven:maven-repository-metadata:${maven-core.vespa.version} org.apache.maven:maven-settings:${maven-core.vespa.version} +org.apache.maven:maven-xml-impl:${maven-xml-impl.vespa.version} org.apache.opennlp:opennlp-tools:${opennlp.vespa.version} org.apache.velocity:velocity-engine-core:${velocity.vespa.version} org.apache.yetus:audience-annotations:0.12.0 @@ -130,12 +133,14 @@ org.bouncycastle:bcpkix-jdk18on:${bouncycastle.vespa.version} org.bouncycastle:bcprov-jdk18on:${bouncycastle.vespa.version} org.bouncycastle:bcutil-jdk18on:${bouncycastle.vespa.version} org.codehaus.plexus:plexus-archiver:${plexus-archiver.vespa.version} -org.codehaus.plexus:plexus-classworlds:2.7.0 +org.codehaus.plexus:plexus-classworlds:${plexus-classworlds.vespa.version} org.codehaus.plexus:plexus-component-annotations:2.1.0 org.codehaus.plexus:plexus-container-default:1.0-alpha-9-stable-1 org.codehaus.plexus:plexus-interpolation:${plexus-interpolation.vespa.version} org.codehaus.plexus:plexus-io:${plexus-io.vespa.version} org.codehaus.plexus:plexus-utils:${plexus-utils.vespa.version} +org.codehaus.plexus:plexus-xml:${plexus-xml.vespa.version} +org.codehaus.woodstox:stax2-api:${stax2-api.vespa.version} org.eclipse.angus:angus-activation:${eclipse-angus.vespa.version} org.eclipse.collections:eclipse-collections-api:${eclipse-collections.vespa.version} org.eclipse.collections:eclipse-collections:${eclipse-collections.vespa.version} diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java index 35faa89388b..b365446db6d 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java @@ -61,6 +61,7 @@ class CliArguments { private static final String DOOM_OPTION = "max-failure-seconds"; private static final String PROXY_OPTION = "proxy"; private static final String COMPRESSION = "compression"; + private static final String LOG_CONFIG_OPTION = "log-config"; private final CommandLine arguments; @@ -205,6 +206,8 @@ class CliArguments { } } + Optional<Path> logConfigFile() throws CliArgumentsException { return fileValue(LOG_CONFIG_OPTION); } + private OptionalInt intValue(String option) throws CliArgumentsException { try { Number number = (Number) arguments.getParsedOptionValue(option); @@ -373,6 +376,14 @@ class CliArguments { "Valid arguments are: 'auto' (default), 'none', 'gzip'") .hasArg() .type(Compression.class) + .build()) + .addOption(Option.builder() + .longOpt(LOG_CONFIG_OPTION) + .desc("Specify a path to a Java Util Logging properties file. " + + "Overrides the default configuration from " + + "VESPA_HOME/conf/vespa-feed-client/logging.properties") + .hasArg() + .type(File.class) .build()); } diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java index 8f2a5b4c5a0..9037b453e47 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLSession; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -33,6 +34,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BooleanSupplier; +import java.util.logging.LogManager; +import java.util.logging.Logger; import java.util.stream.IntStream; import static java.util.stream.Collectors.joining; @@ -44,6 +47,7 @@ import static java.util.stream.Collectors.joining; */ public class CliClient { + private static final Logger log = Logger.getLogger(CliClient.class.getName()); private static final JsonFactory factory = new JsonFactory().disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); private final PrintStream systemOut; @@ -67,7 +71,16 @@ public class CliClient { boolean verbose = false; try { CliArguments cliArgs = CliArguments.fromRawArgs(rawArgs); + var logConfigFile = cliArgs.logConfigFile().orElse(null); verbose = cliArgs.verboseSpecified(); + if (logConfigFile != null) { + try (InputStream in = new BufferedInputStream(Files.newInputStream(logConfigFile))) { + LogManager.getLogManager().readConfiguration(in); + log.fine(() -> "Log configuration overridden by " + logConfigFile); + } catch (IOException e) { + return handleException(verbose, "Failed to read log configuration from " + logConfigFile, e); + } + } if (cliArgs.helpSpecified()) { cliArgs.printHelp(systemOut); return 0; diff --git a/vespa-feed-client-cli/src/main/resources/logging.properties b/vespa-feed-client-cli/src/main/resources/logging.properties index 3f0e8b24e78..e69de29bb2d 100644 --- a/vespa-feed-client-cli/src/main/resources/logging.properties +++ b/vespa-feed-client-cli/src/main/resources/logging.properties @@ -1,2 +0,0 @@ -# Disable verbose info logging from org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec -org.apache.hc.client5.http.impl.async.level = WARNING diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt index 554c42af3dc..8e34c61c0f2 100644 --- a/vespa-feed-client-cli/src/test/resources/help.txt +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -23,6 +23,12 @@ Vespa feed client --header <arg> HTTP header on the form 'Name: value' --help + --log-config <arg> Specify a path to a Java Util + Logging properties file. + Overrides the default + configuration from + VESPA_HOME/conf/vespa-feed-clien + t/logging.properties --max-failure-seconds <arg> Exit if specified number of seconds ever pass without any successful operations. Disabled diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java index 424481b6ef2..d5eab8e17af 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java @@ -58,6 +58,7 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { Compression compression = auto; URI proxy; Duration connectionTtl = Duration.ZERO; + LongSupplier nanoClock = System::nanoTime; public FeedClientBuilderImpl() { } @@ -251,6 +252,11 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { return this; } + FeedClientBuilderImpl setNanoClock(LongSupplier nanoClock) { + this.nanoClock = requireNonNull(nanoClock); + return this; + } + /** Constructs instance of {@link ai.vespa.feed.client.FeedClient} from builder configuration */ @Override public FeedClient build() { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java index d12d72f7a70..8ee281fb38d 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java @@ -55,6 +55,7 @@ class HttpFeedClient implements FeedClient { private final RequestStrategy requestStrategy; private final AtomicBoolean closed = new AtomicBoolean(); private final boolean speedTest; + private final LongSupplier nanoClock; HttpFeedClient(FeedClientBuilderImpl builder) throws IOException { this(builder, @@ -69,6 +70,7 @@ class HttpFeedClient implements FeedClient { this.requestHeaders = new HashMap<>(builder.requestHeaders); this.requestStrategy = requestStrategy; this.speedTest = builder.speedTest; + this.nanoClock = builder.nanoClock; verifyConnection(builder, clusterFactory); } @@ -111,11 +113,12 @@ class HttpFeedClient implements FeedClient { throw new IllegalStateException("Client is closed"); HttpRequest request = new HttpRequest(method, - getPath(documentId) + getQuery(params, speedTest), + getPath(documentId), + getQuery(params, speedTest), requestHeaders, operationJson == null ? null : operationJson.getBytes(UTF_8), // TODO: make it bytes all the way? params.timeout().orElse(maxTimeout), - System::nanoTime); + nanoClock); CompletableFuture<Result> promise = new CompletableFuture<>(); requestStrategy.enqueue(documentId, request) @@ -137,11 +140,12 @@ class HttpFeedClient implements FeedClient { Instant start = Instant.now(); try (Cluster cluster = clusterFactory.create()) { HttpRequest request = new HttpRequest("POST", - getPath(DocumentId.of("feeder", "handshake", "dummy")) + getQuery(empty(), true), + getPath(DocumentId.of("feeder", "handshake", "dummy")), + getQuery(empty(), true), requestHeaders, null, Duration.ofSeconds(15), - System::nanoTime); + nanoClock); CompletableFuture<HttpResponse> future = new CompletableFuture<>(); cluster.dispatch(request, future); HttpResponse response = future.get(20, TimeUnit.SECONDS); @@ -312,7 +316,6 @@ class HttpFeedClient implements FeedClient { StringJoiner query = new StringJoiner("&", "?", "").setEmptyValue(""); if (params.createIfNonExistent()) query.add("create=true"); params.testAndSetCondition().ifPresent(condition -> query.add("condition=" + encode(condition))); - params.timeout().ifPresent(timeout -> query.add("timeout=" + timeout.toMillis() + "ms")); params.route().ifPresent(route -> query.add("route=" + encode(route))); params.tracelevel().ifPresent(tracelevel -> query.add("tracelevel=" + tracelevel)); if (speedTest) query.add("dryRun=true"); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequest.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequest.java index fdd35b74c35..22f6eaa75a4 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequest.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequest.java @@ -10,15 +10,17 @@ class HttpRequest { private final String method; private final String path; + private final String query; private final Map<String, Supplier<String>> headers; private final byte[] body; private final Duration timeout; private final long deadlineNanos; private final LongSupplier nanoClock; - public HttpRequest(String method, String path, Map<String, Supplier<String>> headers, byte[] body, Duration timeout, LongSupplier nanoClock) { + public HttpRequest(String method, String path, String query, Map<String, Supplier<String>> headers, byte[] body, Duration timeout, LongSupplier nanoClock) { this.method = method; this.path = path; + this.query = query; this.headers = headers; this.body = body; this.deadlineNanos = nanoClock.getAsLong() + timeout.toNanos(); @@ -30,8 +32,8 @@ class HttpRequest { return method; } - public String path() { - return path; + public String pathAndQuery() { + return path + (query.isEmpty() ? "?" : query + "&") + "timeout=" + Math.max(1, timeLeft().toMillis()) + "ms"; } public Map<String, Supplier<String>> headers() { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java index 5fe59647038..176c862503e 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java @@ -19,9 +19,14 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -36,6 +41,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.logging.Level.FINE; import static java.util.logging.Level.FINER; import static java.util.logging.Level.FINEST; +import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; @@ -154,7 +160,7 @@ class HttpRequestStrategy implements RequestStrategy { breaker.failure(thrown); if ( (thrown instanceof IOException) // General IO problems. // Thrown by HTTP2Session.StreamsState.reserveSlot, likely on GOAWAY from server - || (thrown instanceof IllegalStateException && thrown.getMessage().equals("session closed")) + || (thrown instanceof IllegalStateException && "session closed".equals(thrown.getMessage())) ) { log.log(FINER, thrown, () -> "Failed attempt " + attempt + " at " + request); return retry(request, attempt); @@ -217,8 +223,8 @@ class HttpRequestStrategy implements RequestStrategy { public void await() { try { - while (inflight.get() > 0) - Thread.sleep(10); + while (inflight.get() > 0) Thread.sleep(10); + resettableCluster.sync(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -334,6 +340,7 @@ class HttpRequestStrategy implements RequestStrategy { private final Object monitor = new Object(); private final ClusterFactory clusterFactory; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); private AtomicLong inflight = new AtomicLong(0); private Cluster delegate; @@ -346,15 +353,18 @@ class HttpRequestStrategy implements RequestStrategy { public void dispatch(HttpRequest request, CompletableFuture<HttpResponse> vessel) { synchronized (monitor) { AtomicLong usedCounter = inflight; - Cluster usedCluster = delegate; usedCounter.incrementAndGet(); - delegate.dispatch(request, vessel); - vessel.whenComplete((__, ___) -> { - synchronized (monitor) { - if (usedCounter.decrementAndGet() == 0 && usedCluster != delegate) - usedCluster.close(); - } - }); + Cluster usedCluster = delegate; + usedCluster.dispatch(request, vessel); + vessel.whenCompleteAsync((__, ___) -> { + synchronized (monitor) { + if (usedCounter.decrementAndGet() == 0 && usedCluster != delegate) { + log.log(INFO, "Closing old HTTP client"); + usedCluster.close(); + } + } + }, + executor); } } @@ -362,6 +372,29 @@ class HttpRequestStrategy implements RequestStrategy { public void close() { synchronized (monitor) { delegate.close(); + executor.shutdown(); + try { + if ( ! executor.awaitTermination(1, TimeUnit.MINUTES)) + log.log(WARNING, "Failed shutting down HTTP client within 1 minute"); + } + catch (InterruptedException e) { + log.log(WARNING, "Interrupted waiting for HTTP client to shut down"); + Thread.currentThread().interrupt(); + } + } + } + + private void sync() throws InterruptedException { + Future<?> sync; + synchronized (monitor) { + if (executor.isShutdown()) return; + sync = executor.submit(() -> { }); + } + try { + sync.get(); + } + catch (ExecutionException e) { + throw new RuntimeException(e); } } @@ -372,6 +405,7 @@ class HttpRequestStrategy implements RequestStrategy { void reset() throws IOException { synchronized (monitor) { + log.log(INFO, "Replacing underlying HTTP client to attempt recovery"); delegate = clusterFactory.create(); inflight = new AtomicLong(0); } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java index 18369f29f0b..28e5b5d0a21 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java @@ -48,6 +48,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; @@ -62,6 +64,7 @@ import static org.eclipse.jetty.http.MimeTypes.Type.APPLICATION_JSON; * @author bjorncs */ class JettyCluster implements Cluster { + private static final Logger log = Logger.getLogger(JettyCluster.class.getName()); // Socket timeout must be longer than the longest feasible response timeout private static final Duration IDLE_TIMEOUT = Duration.ofMinutes(15); @@ -87,7 +90,7 @@ class JettyCluster implements Cluster { vessel.completeExceptionally(new TimeoutException("operation timed out after '" + req.timeout() + "'")); return; } - Request jettyReq = client.newRequest(URI.create(endpoint.uri + req.path())) + Request jettyReq = client.newRequest(URI.create(endpoint.uri + req.pathAndQuery())) .version(HttpVersion.HTTP_2) .method(HttpMethod.fromString(req.method())) .headers(hs -> req.headers().forEach((k, v) -> hs.add(k, v.get()))) @@ -108,15 +111,23 @@ class JettyCluster implements Cluster { } jettyReq.body(new BytesRequestContent(APPLICATION_JSON.asString(), bytes)); } + log.log(Level.FINER, () -> + String.format("Dispatching request %s (%s)", req, System.identityHashCode(vessel))); jettyReq.send(new BufferingResponseListener() { @Override public void onComplete(Result result) { + log.log(Level.FINER, () -> + String.format("Completed request %s (%s): %s", + req, System.identityHashCode(vessel), + result.isFailed() + ? result.getFailure().toString() : result.getResponse().getStatus())); endpoint.inflight.decrementAndGet(); if (result.isFailed()) vessel.completeExceptionally(result.getFailure()); else vessel.complete(new JettyResponse(result.getResponse(), getContent())); } }); } catch (Exception e) { + log.log(Level.FINE, e, () -> "Failed to dispatch request: " + e.getMessage()); endpoint.inflight.decrementAndGet(); vessel.completeExceptionally(e); } diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpFeedClientTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpFeedClientTest.java index cec070c06a6..9afaeed8062 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpFeedClientTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpFeedClientTest.java @@ -43,7 +43,9 @@ class HttpFeedClientTest { @Override public void await() { throw new UnsupportedOperationException(); } @Override public CompletableFuture<HttpResponse> enqueue(DocumentId documentId, HttpRequest request) { return dispatch.get().apply(documentId, request); } } - FeedClient client = new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))).setDryrun(true), + FeedClient client = new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))) + .setDryrun(true) + .setNanoClock(() -> 0), () -> new DryrunCluster(), new MockRequestStrategy()); @@ -51,8 +53,8 @@ class HttpFeedClientTest { dispatch.set((documentId, request) -> { try { assertEquals(id, documentId); - assertEquals("/document/v1/ns/type/docid/0", - request.path()); + assertEquals("/document/v1/ns/type/docid/0?timeout=900000ms", + request.pathAndQuery()); assertEquals("PUT", request.method()); assertEquals("json", new String(request.body(), UTF_8)); @@ -83,8 +85,8 @@ class HttpFeedClientTest { dispatch.set((documentId, request) -> { try { assertEquals(id, documentId); - assertEquals("/document/v1/ns/type/docid/0?tracelevel=1", - request.path()); + assertEquals("/document/v1/ns/type/docid/0?tracelevel=1&timeout=900000ms", + request.pathAndQuery()); assertEquals("DELETE", request.method()); assertNull(request.body()); @@ -145,8 +147,8 @@ class HttpFeedClientTest { dispatch.set((documentId, request) -> { try { assertEquals(id, documentId); - assertEquals("/document/v1/ns/type/docid/0?create=true&condition=false&timeout=5000ms&route=route", - request.path()); + assertEquals("/document/v1/ns/type/docid/0?create=true&condition=false&route=route&timeout=5000ms", + request.pathAndQuery()); assertEquals("json", new String(request.body(), UTF_8)); HttpResponse response = HttpResponse.of(502, @@ -183,8 +185,8 @@ class HttpFeedClientTest { dispatch.set((documentId, request) -> { try { assertEquals(id, documentId); - assertEquals("/document/v1/ns/type/docid/0", - request.path()); + assertEquals("/document/v1/ns/type/docid/0?timeout=900000ms", + request.pathAndQuery()); assertEquals("json", new String(request.body(), UTF_8)); HttpResponse response = HttpResponse.of(500, @@ -227,7 +229,7 @@ class HttpFeedClientTest { try { assertNull(request.body()); assertEquals("POST", request.method()); - assertEquals("/document/v1/feeder/handshake/docid/dummy?dryRun=true", request.path()); + assertEquals("/document/v1/feeder/handshake/docid/dummy?dryRun=true&timeout=15000ms", request.pathAndQuery()); vessel.complete(response.get()); } catch (Throwable t) { @@ -238,19 +240,23 @@ class HttpFeedClientTest { // Old server, and speed-test. assertEquals("server does not support speed test; upgrade to a newer version", assertThrows(FeedException.class, - () -> new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))).setSpeedTest(true), + () -> new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))) + .setNanoClock(() -> 0) + .setSpeedTest(true), () -> cluster, null)) .getMessage()); // Old server. - new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))), + new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))) + .setNanoClock(() -> 0), () -> cluster, null); // New server. response.set(okResponse); - new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))), + new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))) + .setNanoClock(() -> 0), () -> cluster, null); } diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java index e067a103d59..4d8d14ad089 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java @@ -44,7 +44,7 @@ class HttpRequestStrategyTest { @Test void testConcurrency() throws IOException { int documents = 1 << 16; - HttpRequest request = new HttpRequest("PUT", "/", null, null, Duration.ofSeconds(1), () -> 0); + HttpRequest request = new HttpRequest("PUT", "/", "", null, null, Duration.ofSeconds(1), () -> 0); HttpResponse response = HttpResponse.of(200, "{}".getBytes(UTF_8)); ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); Cluster cluster = (__, vessel) -> executor.schedule(() -> vessel.complete(response), (int) (Math.random() * 2 * 10), TimeUnit.MILLISECONDS); @@ -102,7 +102,7 @@ class HttpRequestStrategyTest { DocumentId id1 = DocumentId.of("ns", "type", "1"); DocumentId id2 = DocumentId.of("ns", "type", "2"); - HttpRequest request = new HttpRequest("POST", "/", null, null, Duration.ofSeconds(180), nowNanos::get); + HttpRequest request = new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(180), nowNanos::get); // Runtime exception is not retried. cluster.expect((__, vessel) -> vessel.completeExceptionally(new RuntimeException("boom"))); @@ -146,8 +146,8 @@ class HttpRequestStrategyTest { else vessel.complete(success); }); CompletableFuture<HttpResponse> delayed = strategy.enqueue(id1, request); - CompletableFuture<HttpResponse> serialised = strategy.enqueue(id1, new HttpRequest("PUT", "/", null, null, Duration.ofSeconds(1), nowNanos::get)); - assertEquals(success, strategy.enqueue(id2, new HttpRequest("DELETE", "/", null, null, Duration.ofSeconds(1), nowNanos::get)).get()); + CompletableFuture<HttpResponse> serialised = strategy.enqueue(id1, new HttpRequest("PUT", "/", "", null, null, Duration.ofSeconds(1), nowNanos::get)); + assertEquals(success, strategy.enqueue(id2, new HttpRequest("DELETE", "/", "", null, null, Duration.ofSeconds(1), nowNanos::get)).get()); latch.await(); assertEquals(8, strategy.stats().requests()); // 3 attempts at throttled and one at id2. nowNanos.set(4_000_000_000L); @@ -168,7 +168,7 @@ class HttpRequestStrategyTest { // Error responses are not retried when not of appropriate type. cluster.expect((__, vessel) -> vessel.complete(serverError)); - assertEquals(serverError, strategy.enqueue(id1, new HttpRequest("PUT", "/", null, null, Duration.ofSeconds(1), nowNanos::get)).get()); + assertEquals(serverError, strategy.enqueue(id1, new HttpRequest("PUT", "/", "", null, null, Duration.ofSeconds(1), nowNanos::get)).get()); assertEquals(12, strategy.stats().requests()); // Some error responses are not retried. @@ -184,7 +184,7 @@ class HttpRequestStrategyTest { vessel.completeExceptionally(new IOException("retry me")); }); expected = assertThrows(ExecutionException.class, - () -> strategy.enqueue(id1, new HttpRequest("POST", "/", null, null, Duration.ofMillis(100), nowNanos::get)).get()); + () -> strategy.enqueue(id1, new HttpRequest("POST", "/", "", null, null, Duration.ofMillis(100), nowNanos::get)).get()); assertEquals("retry me", expected.getCause().getCause().getMessage()); assertEquals(15, strategy.stats().requests()); @@ -223,15 +223,15 @@ class HttpRequestStrategyTest { // First operation fails, second remains in flight, and third fails. clusters.get(0).expect((__, vessel) -> vessel.complete(HttpResponse.of(200, null))); - strategy.enqueue(DocumentId.of("ns", "type", "1"), new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), now::get)).get(); + strategy.enqueue(DocumentId.of("ns", "type", "1"), new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), now::get)).get(); Exchanger<CompletableFuture<HttpResponse>> exchanger = new Exchanger<>(); clusters.get(0).expect((__, vessel) -> { try { exchanger.exchange(vessel); } catch (InterruptedException e) { throw new RuntimeException(e); } }); - CompletableFuture<HttpResponse> secondResponse = strategy.enqueue(DocumentId.of("ns", "type", "2"), new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), now::get)); + CompletableFuture<HttpResponse> secondResponse = strategy.enqueue(DocumentId.of("ns", "type", "2"), new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), now::get)); CompletableFuture<HttpResponse> secondVessel = exchanger.exchange(null); clusters.get(0).expect((__, vessel) -> vessel.complete(HttpResponse.of(500, null))); - strategy.enqueue(DocumentId.of("ns", "type", "3"), new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), now::get)).get(); + strategy.enqueue(DocumentId.of("ns", "type", "3"), new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), now::get)).get(); // Time advances, and the circuit breaker half-opens. assertEquals(CLOSED, breaker.state()); @@ -241,20 +241,21 @@ class HttpRequestStrategyTest { // It's indeterminate which cluster gets the next request, but the second should get the next one after that. clusters.get(0).expect((__, vessel) -> vessel.complete(HttpResponse.of(500, null))); clusters.get(1).expect((__, vessel) -> vessel.complete(HttpResponse.of(500, null))); - assertEquals(500, strategy.enqueue(DocumentId.of("ns", "type", "4"), new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), now::get)).get().code()); + assertEquals(500, strategy.enqueue(DocumentId.of("ns", "type", "4"), new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), now::get)).get().code()); clusters.get(0).expect((__, vessel) -> vessel.completeExceptionally(new AssertionError("should not be called"))); clusters.get(1).expect((__, vessel) -> vessel.complete(HttpResponse.of(200, null))); - assertEquals(200, strategy.enqueue(DocumentId.of("ns", "type", "5"), new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), now::get)).get().code()); + assertEquals(200, strategy.enqueue(DocumentId.of("ns", "type", "5"), new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), now::get)).get().code()); assertFalse(clusters.get(0).closed.get()); assertFalse(clusters.get(1).closed.get()); secondVessel.complete(HttpResponse.of(504, null)); assertEquals(504, secondResponse.get().code()); + strategy.await(); assertTrue(clusters.get(0).closed.get()); assertFalse(clusters.get(1).closed.get()); - strategy.await(); strategy.destroy(); + strategy.await(); assertTrue(clusters.get(1).closed.get()); } @@ -276,10 +277,10 @@ class HttpRequestStrategyTest { DocumentId id3 = DocumentId.of("ns", "type", "3"); DocumentId id4 = DocumentId.of("ns", "type", "4"); DocumentId id5 = DocumentId.of("ns", "type", "5"); - HttpRequest failing = new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), nowNanos::get); - HttpRequest partial = new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), nowNanos::get); - HttpRequest request = new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), nowNanos::get); - HttpRequest blocking = new HttpRequest("POST", "/", null, null, Duration.ofSeconds(1), nowNanos::get); + HttpRequest failing = new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), nowNanos::get); + HttpRequest partial = new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), nowNanos::get); + HttpRequest request = new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), nowNanos::get); + HttpRequest blocking = new HttpRequest("POST", "/", "", null, null, Duration.ofSeconds(1), nowNanos::get); // Enqueue some operations to the same id, which are serialised, and then shut down while operations are in flight. Phaser phaser = new Phaser(2); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/ClientParameters.java index 326b932cabc..ad14708015b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/ClientParameters.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/ClientParameters.java @@ -26,19 +26,24 @@ public class ClientParameters { // Document type identifier public final String docType; + // Zstandard compression + public final boolean zstCompression; + public ClientParameters( boolean help, String inputFile, String outputFile, String field, String language, - String docType) { + String docType, + boolean zstCompression) { this.help = help; this.inputFile = inputFile; this.outputFile = outputFile; this.field = field; this.language = language; this.docType = docType; + this.zstCompression = zstCompression; } public static class Builder { @@ -47,8 +52,8 @@ public class ClientParameters { private String outputFile; private String field; private String language; - private String docType; + private boolean zstCompression; public Builder setHelp(boolean help) { this.help = help; @@ -79,8 +84,13 @@ public class ClientParameters { return this; } + public Builder setZstCompression(String useZstCompression) { + this.zstCompression = Boolean.parseBoolean(useZstCompression); + return this; + } + public ClientParameters build() { - return new ClientParameters(help, inputFile, outputFile, field, language, docType); + return new ClientParameters(help, inputFile, outputFile, field, language, docType, zstCompression); } } } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/CommandLineOptions.java index e5d16854647..2deaee983fe 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/CommandLineOptions.java @@ -26,6 +26,7 @@ public class CommandLineOptions { public static final String FIELD_OPTION = "field"; public static final String LANGUAGE_OPTION = "language"; public static final String DOC_TYPE_OPTION = "doc-type"; + public static final String ZST_COMPRESSION = "zst-compression"; private final Options options = createOptions(); @@ -40,35 +41,46 @@ public class CommandLineOptions { .build()); options.addOption(Option.builder("i") + .required() .hasArg(true) .desc("Input file") .longOpt(INPUT_OPTION) .build()); options.addOption(Option.builder("i") + .required() .hasArg(true) .desc("Output file") .longOpt(OUTPUT_OPTION) .build()); options.addOption(Option.builder("f") + .required() .hasArg(true) .desc("Field to analyze") .longOpt(FIELD_OPTION) .build()); options.addOption(Option.builder("l") + .required() .hasArg(true) .desc("Language tag for output file") .longOpt(LANGUAGE_OPTION) .build()); options.addOption(Option.builder("d") + .required() .hasArg(true) .desc("Document type identifier") .longOpt(DOC_TYPE_OPTION) .build()); + options.addOption(Option.builder("zst") + .hasArg(true) + .desc("Use Zstandard compression") + .longOpt(ZST_COMPRESSION) + .build()); + return options; } @@ -93,6 +105,7 @@ public class CommandLineOptions { builder.setField(cl.getOptionValue(FIELD_OPTION)); builder.setLanguage(cl.getOptionValue(LANGUAGE_OPTION)); builder.setDocType(cl.getOptionValue(DOC_TYPE_OPTION)); + builder.setZstCompression(cl.hasOption(ZST_COMPRESSION) ? cl.getOptionValue(ZST_COMPRESSION) : "true"); return builder.build(); } catch (ParseException e) { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java index a60408b1f96..e368eebefa5 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/Main.java @@ -32,6 +32,7 @@ public class Main { if (params.help) { options.printHelp(); } else { + System.setProperty("vespa.replace_invalid_unicode", "true"); SignificanceModelGenerator significanceModelGenerator = createSignificanceModelGenerator(params); significanceModelGenerator.generate(); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java index e27158da3cb..d620820e14f 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespasignificance/SignificanceModelGenerator.java @@ -2,8 +2,6 @@ package com.yahoo.vespasignificance; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; @@ -27,6 +25,8 @@ import com.yahoo.language.process.Tokenizer; import com.yahoo.language.significance.impl.DocumentFrequencyFile; import com.yahoo.language.significance.impl.SignificanceModelFile; import com.yahoo.text.Utf8; +import io.airlift.compress.zstd.ZstdInputStream; +import io.airlift.compress.zstd.ZstdOutputStream; import java.io.IOException; import java.io.BufferedReader; @@ -34,11 +34,16 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.FileOutputStream; +import java.io.FileInputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; +import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -49,13 +54,15 @@ public class SignificanceModelGenerator { private final ClientParameters clientParameters; private final Tokenizer tokenizer; - private final HashMap<String, Long> documentFrequency = new HashMap<>(); + private final TreeMap<String, Long> documentFrequency = new TreeMap<>(); + private final Language language; private final ObjectMapper objectMapper; private final static JsonFactory parserFactory = new JsonFactory(); final DocumentTypeManager types = new DocumentTypeManager(); final DocumentType docType; + private final boolean useZstCompression; private final static String VERSION = "1.0"; private final static String ID = "1"; private final static String SIGNIFICANCE_DESCRIPTION = "Significance model for input file"; @@ -63,14 +70,24 @@ public class SignificanceModelGenerator { public SignificanceModelGenerator(ClientParameters clientParameters) { this.clientParameters = clientParameters; + + if (clientParameters.zstCompression && !clientParameters.outputFile.endsWith(".zst")) { + throw new IllegalArgumentException("Output file must have .zst extension when using zst compression"); + } + + language = Language.fromLanguageTag(clientParameters.language); + if (language == Language.UNKNOWN) { + throw new IllegalArgumentException("Unknown language: " + clientParameters.language); + } + OpenNlpLinguistics openNlpLinguistics = new OpenNlpLinguistics(); tokenizer = openNlpLinguistics.getTokenizer(); objectMapper = new ObjectMapper(); - language = Language.fromLanguageTag(clientParameters.language); - docType = new DocumentType(clientParameters.docType); docType.addField(new Field(clientParameters.field, DataType.STRING)); + useZstCompression = clientParameters.zstCompression; + types.registerDocumentType(docType); } @@ -100,8 +117,14 @@ public class SignificanceModelGenerator { long pageCount = i - 1; SignificanceModelFile modelFile; - if (Paths.get(clientParameters.outputFile).toFile().exists()) { - modelFile = objectMapper.readValue(new File(clientParameters.outputFile), SignificanceModelFile.class); + File outputFile = Paths.get(clientParameters.outputFile).toFile(); + if (outputFile.exists()) { + + InputStream in = outputFile.toString().endsWith(".zst") ? + new ZstdInputStream(new FileInputStream(outputFile)) : + new FileInputStream(outputFile); + + modelFile = objectMapper.readValue(in, SignificanceModelFile.class); modelFile.addLanguage(clientParameters.language, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); @@ -110,12 +133,16 @@ public class SignificanceModelGenerator { put(clientParameters.language, new DocumentFrequencyFile(DOC_FREQ_DESCRIPTION, pageCount, getFinalDocumentFrequency())); }}; - modelFile = new SignificanceModelFile(VERSION, ID, SIGNIFICANCE_DESCRIPTION, languages); + modelFile = new SignificanceModelFile(VERSION, ID, SIGNIFICANCE_DESCRIPTION + clientParameters.inputFile, languages); } try { - //objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); ObjectWriter writer = objectMapper.writerWithDefaultPrettyPrinter(); - writer.writeValue(new File(clientParameters.outputFile), modelFile); + + OutputStream out = useZstCompression ? + new ZstdOutputStream(new FileOutputStream(clientParameters.outputFile)) : + new FileOutputStream(clientParameters.outputFile); + + writer.writeValue(out, modelFile); } catch (IOException e) { throw new IllegalStateException("Failed to write model to output file", e); } @@ -139,9 +166,14 @@ public class SignificanceModelGenerator { } } - public HashMap<String, Long> getFinalDocumentFrequency() { + public Map<String, Long> getFinalDocumentFrequency() { return documentFrequency.entrySet().stream() .filter(k -> k.getValue() > 1) - .collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getValue()), HashMap::putAll); + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (e1, e2) -> e1, + TreeMap::new + )); } } diff --git a/vespaclient-java/src/main/sh/vespa-significance.sh b/vespaclient-java/src/main/sh/vespa-significance.sh index 0c0bac275e6..f26c8dfca46 100755 --- a/vespaclient-java/src/main/sh/vespa-significance.sh +++ b/vespaclient-java/src/main/sh/vespa-significance.sh @@ -79,6 +79,7 @@ export ROOT export MALLOC_ARENA_MAX=1 #Does not need fast allocation exec java \ +-D vespa.replace_invalid_unicode=true \ -server -enableassertions \ -XX:ThreadStackSize=512 \ -XX:MaxJavaStackTraceDepth=1000000 \ diff --git a/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java b/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java index 8e1e8fd5627..916fe05ef7b 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespasignificance/SignificanceModelGeneratorTest.java @@ -5,11 +5,14 @@ package com.yahoo.vespasignificance; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.language.significance.impl.DocumentFrequencyFile; import com.yahoo.language.significance.impl.SignificanceModelFile; +import io.airlift.compress.zstd.ZstdInputStream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.util.HashMap; @@ -25,7 +28,7 @@ public class SignificanceModelGeneratorTest { @TempDir private Path tempDir; - private ClientParameters.Builder createParameters(String inputPath, String outputPath, String field, String language, String docType) { + private ClientParameters.Builder createParameters(String inputPath, String outputPath, String field, String language, String docType, String zstCompression) { tempDir.toFile().mkdirs(); return new ClientParameters.Builder() @@ -33,7 +36,8 @@ public class SignificanceModelGeneratorTest { .setOutputFile(tempDir.resolve(outputPath).toString()) .setField(field) .setLanguage(language) - .setDocType(docType); + .setDocType(docType) + .setZstCompression(zstCompression); } private SignificanceModelGenerator createSignificanceModelGenerator(ClientParameters params) { @@ -44,7 +48,7 @@ public class SignificanceModelGeneratorTest { void testGenerateSimpleFile() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params = createParameters(inputPath, outputPath, "text", "NB", "nb").build(); + ClientParameters params = createParameters(inputPath, outputPath, "text", "NB", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params); generator.generate(); @@ -68,10 +72,47 @@ public class SignificanceModelGeneratorTest { } @Test + void testGenerateSimpleFileWithZST() throws IOException { + String inputPath = "no.jsonl"; + ClientParameters params1 = createParameters(inputPath, "output.json", "text", "NB", "nb", "true").build(); + + // Throws exception when outputfile does not have .zst extension when using zst compression + assertThrows(IllegalArgumentException.class, () -> createSignificanceModelGenerator(params1)); + + String outputPath = "output.json.zst"; + ClientParameters params = createParameters(inputPath, outputPath, "text", "NB", "nb", "true").build(); + + SignificanceModelGenerator generator = createSignificanceModelGenerator(params); + generator.generate(); + + + + File outputFile = new File(tempDir.resolve(outputPath ).toString()); + assertTrue(outputFile.exists()); + + InputStream in = new ZstdInputStream(new FileInputStream(outputFile)); + + SignificanceModelFile modelFile = objectMapper.readValue(in, SignificanceModelFile.class); + + HashMap<String, DocumentFrequencyFile> languages = modelFile.languages(); + assertEquals(1, languages.size()); + + assertTrue(languages.containsKey("NB")); + + DocumentFrequencyFile documentFrequencyFile = languages.get("NB"); + + assertEquals(3, documentFrequencyFile.frequencies().get("fra")); + assertEquals(3, documentFrequencyFile.frequencies().get("skriveform")); + assertEquals(3, documentFrequencyFile.frequencies().get("kategori")); + assertEquals(3, documentFrequencyFile.frequencies().get("eldr")); + + } + + @Test void testGenerateFileWithMultipleLanguages() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params1 = createParameters(inputPath, outputPath, "text", "NB", "nb").build(); + ClientParameters params1 = createParameters(inputPath, outputPath, "text", "NB", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params1); generator.generate(); @@ -79,7 +120,7 @@ public class SignificanceModelGeneratorTest { assertTrue(outputFile.exists()); String inputPath2 = "en.jsonl"; - ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "EN", "en").build(); + ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "EN", "en", "false").build(); generator = createSignificanceModelGenerator(params2); generator.generate(); @@ -113,7 +154,7 @@ public class SignificanceModelGeneratorTest { void testOverwriteExistingDocumentFrequencyLanguage() throws IOException { String inputPath = "no.jsonl"; String outputPath = "output.json"; - ClientParameters params1 = createParameters(inputPath, outputPath, "text", "NB", "nb").build(); + ClientParameters params1 = createParameters(inputPath, outputPath, "text", "NB", "nb", "false").build(); SignificanceModelGenerator generator = createSignificanceModelGenerator(params1); generator.generate(); @@ -134,7 +175,7 @@ public class SignificanceModelGeneratorTest { assertFalse(oldDf.frequencies().containsKey("nytt")); String inputPath2 = "no_2.jsonl"; - ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "NB", "nb").build(); + ClientParameters params2 = createParameters(inputPath2, outputPath, "text", "NB", "nb", "false").build(); SignificanceModelGenerator generator2 = createSignificanceModelGenerator(params2); generator2.generate(); diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index f421206a9bc..66d31307ab6 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -2,11 +2,11 @@ vespa_define_module( DEPENDS vespadefaults - configdefinitions - config_cloudconfig + vespa_configdefinitions + vespa_config vespalog - document - documentapi + vespa_document + vespa_documentapi vespalib LIBS diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 45e88ac2e94..06cf7d0d71a 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3456,7 +3456,8 @@ "public static java.util.Optional getChildValue(org.w3c.dom.Element, java.lang.String)", "public static java.lang.String getNodePath(org.w3c.dom.Node, java.lang.String)", "public static boolean isName(java.lang.CharSequence)", - "public static javax.xml.transform.TransformerFactory createTransformerFactory()" + "public static javax.xml.transform.TransformerFactory createTransformerFactory()", + "public static java.lang.String toString(org.w3c.dom.Node)" ], "fields" : [ ] }, diff --git a/vespajlib/src/main/java/ai/vespa/json/Json.java b/vespajlib/src/main/java/ai/vespa/json/Json.java index fef6e2e988e..01bd5c2dc19 100644 --- a/vespajlib/src/main/java/ai/vespa/json/Json.java +++ b/vespajlib/src/main/java/ai/vespa/json/Json.java @@ -16,14 +16,19 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; import java.util.function.Consumer; -import java.util.stream.Collectors; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; import java.util.stream.Stream; import java.util.stream.StreamSupport; import static com.yahoo.slime.Type.ARRAY; import static com.yahoo.slime.Type.STRING; +import static java.util.stream.Collectors.joining; /** * A {@link Slime} wrapper that throws {@link InvalidJsonException} on missing members or invalid types. @@ -161,7 +166,7 @@ public class Json implements Iterable<Json> { private void requirePresent() { if (isMissing()) throw createMissingMemberException(); } private InvalidJsonException createInvalidTypeException(Type... expected) { - var expectedTypesString = Arrays.stream(expected).map(this::toString).collect(Collectors.joining("' or '", "'", "'")); + var expectedTypesString = Arrays.stream(expected).map(this::toString).collect(joining("' or '", "'", "'")); var pathString = path.isEmpty() ? "JSON" : "JSON member '%s'".formatted(path); return new InvalidJsonException( "Expected %s to be a %s but got '%s'" @@ -222,6 +227,10 @@ public class Json implements Iterable<Json> { SlimeUtils.addValue(json.inspector, cursor); return this; } public Builder.Array add(Json.Builder builder) { return add(builder.build()); } + /** Add all values from {@code array} to this array. */ + public Builder.Array addAll(Json.Builder.Array array) { + SlimeUtils.copyArray(array.cursor, cursor); return this; + } /** Note: does not return {@code this}! */ public Builder.Array addArray() { return new Array(cursor.addArray()); } @@ -267,4 +276,18 @@ public class Json implements Iterable<Json> { public Cursor slimeCursor() { return cursor; } public Json build() { return Json.of(cursor); } } + + public static class Collectors { + private Collectors() {} + /** @param accumululator Specify one of the 'add' overloads from {@link Builder.Array} */ + public static <T> Collector<T, Json.Builder.Array, Json> toArray(BiConsumer<Builder.Array, T> accumululator) { + return new Collector<T, Builder.Array, Json>() { + @Override public Supplier<Builder.Array> supplier() { return Json.Builder.Array::newArray; } + @Override public BiConsumer<Builder.Array, T> accumulator() { return accumululator; } + @Override public BinaryOperator<Builder.Array> combiner() { return Builder.Array::addAll; } + @Override public Function<Builder.Array, Json> finisher() { return Builder.Array::build; } + @Override public Set<Characteristics> characteristics() { return Set.of(); } + }; + } + } } diff --git a/vespajlib/src/main/java/ai/vespa/net/CidrBlock.java b/vespajlib/src/main/java/ai/vespa/net/CidrBlock.java index 751f3ef8f32..a91db2795aa 100644 --- a/vespajlib/src/main/java/ai/vespa/net/CidrBlock.java +++ b/vespajlib/src/main/java/ai/vespa/net/CidrBlock.java @@ -205,7 +205,7 @@ public class CidrBlock { } public String asString() { - return InetAddresses.toAddrString(getInetAddress()) + "/" + prefixLength; + return InetAddressUtil.toString(getInetAddress()) + "/" + prefixLength; } public static CidrBlock fromString(String cidr) { diff --git a/vespajlib/src/main/java/ai/vespa/net/InetAddressUtil.java b/vespajlib/src/main/java/ai/vespa/net/InetAddressUtil.java new file mode 100644 index 00000000000..2176cd0a00d --- /dev/null +++ b/vespajlib/src/main/java/ai/vespa/net/InetAddressUtil.java @@ -0,0 +1,26 @@ +package ai.vespa.net; + +import com.google.common.net.InetAddresses; + +import java.net.Inet6Address; +import java.net.InetAddress; + +/** + * @author hakonhall + */ +public class InetAddressUtil { + private InetAddressUtil() {} + + /** Returns the lower case string representation of the IP address (w/o scope), with :: compression for IPv6. */ + public static String toString(InetAddress inetAddress) { + if (inetAddress instanceof Inet6Address) { + String address = InetAddresses.toAddrString(inetAddress); + // toAddrString() returns any interface/scope as a %-suffix, + // see https://github.com/google/guava/commit/3f61870ac6e5b18dbb74ce6f6cb2930ad8750a43 + int percentIndex = address.indexOf('%'); + return percentIndex < 0 ? address : address.substring(0, percentIndex); + } else { + return inetAddress.getHostAddress(); + } + } +} diff --git a/vespajlib/src/main/java/ai/vespa/utils/BytesQuantity.java b/vespajlib/src/main/java/ai/vespa/utils/BytesQuantity.java new file mode 100644 index 00000000000..f1e8a74a74a --- /dev/null +++ b/vespajlib/src/main/java/ai/vespa/utils/BytesQuantity.java @@ -0,0 +1,91 @@ +package ai.vespa.utils; + +import java.util.Locale; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Reprents a quantity of bytes with a human-readable string representation. + * Currently only supports binary units (e.g. 1 kB = 1024 bytes). + * + * @author bjorncs + */ +public class BytesQuantity { + public enum Unit { + BYTES, KB, MB, GB, TB; + + public long binarySize() { + return switch (this) { + case BYTES -> 1; + case KB -> 1 << 10; + case MB -> 1 << 20; + case GB -> 1 << 30; + case TB -> 1L << 40; + }; + } + + public String toUnitString() { + return switch (this) { + case BYTES -> "bytes"; + case KB -> "kB"; + case MB -> "MB"; + case GB -> "GB"; + case TB -> "TB"; + }; + } + + static Unit fromString(String s) { + return switch (s) { + case "", "B", "bytes", "byte" -> BYTES; + case "kB", "k", "K", "KB" -> KB; + case "MB", "m", "M" -> MB; + case "GB", "g", "G" -> GB; + case "TB", "t", "T" -> TB; + default -> throw new IllegalArgumentException("Invalid unit: " + s); + }; + } + } + + private final long bytes; + private BytesQuantity(long bytes) { this.bytes = bytes; } + + public long toBytes() { return bytes; } + + public static BytesQuantity ofBytes(long bytes) { return new BytesQuantity(bytes); } + public static BytesQuantity ofKB(long kb) { return BytesQuantity.of(kb, Unit.KB); } + public static BytesQuantity ofMB(long mb) { return BytesQuantity.of(mb, Unit.MB); } + public static BytesQuantity ofGB(long gb) { return BytesQuantity.of(gb, Unit.GB); } + public static BytesQuantity ofTB(long tb) { return BytesQuantity.of(tb, Unit.TB); } + public static BytesQuantity of(long value, Unit unit) { return new BytesQuantity(value * unit.binarySize()); } + + private static final Pattern PATTERN = Pattern.compile("^(?<digits>\\d+)\\s*(?<unit>[a-zA-Z]*)$"); + public static BytesQuantity fromString(String value) { + var matcher = PATTERN.matcher(value); + if (!matcher.matches()) + throw new IllegalArgumentException( + "Bytes quantity '%s' does not match pattern '%s'".formatted(value, PATTERN.pattern())); + var digits = Long.parseLong(matcher.group("digits")); + var unit = Unit.fromString(matcher.group("unit")); + return BytesQuantity.of(digits, unit); + } + + public String asPrettyString() { + if (bytes == 0) return "0 bytes"; + if (bytes == 1) return "1 byte"; + long remaining = bytes; + int unit = 0; + for (; remaining % 1024 == 0 && unit < Unit.values().length - 1; unit++) remaining /= 1024; + return String.format(Locale.ENGLISH, "%d %s", remaining, Unit.values()[unit].toUnitString()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BytesQuantity that = (BytesQuantity) o; + return bytes == that.bytes; + } + + @Override public int hashCode() { return Objects.hashCode(bytes); } + @Override public String toString() { return asPrettyString(); } +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java index f96fd65e15c..aef3b90af56 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java @@ -104,10 +104,12 @@ class TensorParser { if (type.isEmpty()) throw new IllegalArgumentException("The mixed tensor form requires an explicit tensor type " + "on the form 'tensor(dimensions):..."); - if (type.get().dimensions().stream().filter(d -> ! d.isIndexed()).count() > 1) + if (type.get().dimensions().stream().filter(d -> d.isMapped()).count() > 1) throw new IllegalArgumentException("The mixed tensor form requires a type with a single mapped dimension, " + "but got " + type.get()); - + if (! MixedValueParser.findMappedDimension(type.get()).isPresent()) + throw new IllegalArgumentException("No suitable dimension in " + type.get() + " for parsing a tensor on " + + "the mixed form: Should have one mapped dimension"); try { valueString = valueString.trim(); @@ -426,7 +428,7 @@ class TensorParser { } private void parse() { - TensorType.Dimension mappedDimension = findMappedDimension(); + TensorType.Dimension mappedDimension = findMappedDimension().get(); TensorType mappedSubtype = MixedTensor.createPartialType(builder.type().valueType(), List.of(mappedDimension)); if (dimensionOrder != null) dimensionOrder.remove(mappedDimension.name()); @@ -448,13 +450,16 @@ class TensorParser { } } - private TensorType.Dimension findMappedDimension() { - Optional<TensorType.Dimension> mappedDimension = builder.type().dimensions().stream().filter(TensorType.Dimension::isMapped).findAny(); - if (mappedDimension.isPresent()) return mappedDimension.get(); - if (builder.type().rank() == 1 && builder.type().dimensions().get(0).size().isEmpty()) - return builder.type().dimensions().get(0); - throw new IllegalStateException("No suitable dimension in " + builder.type() + - " for parsing as a mixed tensor. This is a bug."); + private Optional<TensorType.Dimension> findMappedDimension() { + return findMappedDimension(builder.type()); + } + + static Optional<TensorType.Dimension> findMappedDimension(TensorType type) { + Optional<TensorType.Dimension> mappedDimension = type.dimensions().stream().filter(TensorType.Dimension::isMapped).findAny(); + if (mappedDimension.isPresent()) return Optional.of(mappedDimension.get()); + if (type.rank() == 1 && type.dimensions().get(0).size().isEmpty()) + return Optional.of(type.dimensions().get(0)); + return Optional.empty(); } private void parseDenseSubspace(TensorAddress mappedAddress, List<String> denseDimensionOrder) { diff --git a/vespajlib/src/main/java/com/yahoo/text/XML.java b/vespajlib/src/main/java/com/yahoo/text/XML.java index 72a2dba54e1..a39e2f789be 100644 --- a/vespajlib/src/main/java/com/yahoo/text/XML.java +++ b/vespajlib/src/main/java/com/yahoo/text/XML.java @@ -13,7 +13,13 @@ import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.Reader; @@ -22,6 +28,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Static XML utility methods * @@ -459,6 +467,21 @@ public class XML { return transformerFactory; } + /** Returns the UTF-8 string representation of an XML root, tag or text node. */ + public static String toString(Node node) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + Transformer transformer = createTransformerFactory().newTransformer(); + transformer.setOutputProperty(OutputKeys.ENCODING, UTF_8.name()); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.transform(new DOMSource(node), new StreamResult(out)); + } + catch (TransformerException e) { + throw new IllegalArgumentException("invalid XML element", e); + } + return out.toString(UTF_8); + } + /** * The point of this weird class and the jumble of abstract methods is diff --git a/vespajlib/src/main/java/com/yahoo/yolean/concurrent/Memoized.java b/vespajlib/src/main/java/com/yahoo/yolean/concurrent/Memoized.java index 23f00e01a2a..1b8353807f2 100644 --- a/vespajlib/src/main/java/com/yahoo/yolean/concurrent/Memoized.java +++ b/vespajlib/src/main/java/com/yahoo/yolean/concurrent/Memoized.java @@ -58,7 +58,7 @@ public class Memoized<T, E extends Exception> implements Supplier<T>, AutoClosea @Override public T get() { - // Double-checked locking: try the variable, and if not initialized, try to initialize it. + // Double-checked locking: try the _volatile_ variable, and if not initialized, try to initialize it. if (wrapped == null) synchronized (monitor) { // Ensure the factory is called only once, by clearing it once successfully called. if (factory != null) wrapped = requireNonNull(factory.get()); diff --git a/vespajlib/src/test/java/ai/vespa/json/JsonTest.java b/vespajlib/src/test/java/ai/vespa/json/JsonTest.java index 51b64637fd8..f6ca4aa7199 100644 --- a/vespajlib/src/test/java/ai/vespa/json/JsonTest.java +++ b/vespajlib/src/test/java/ai/vespa/json/JsonTest.java @@ -2,6 +2,10 @@ package ai.vespa.json; import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -130,4 +134,39 @@ class JsonTest { .toJson(true); assertEquals(expected, json); } + + @Test + void add_all() { + var expected = + """ + [ + 1, + 2, + 3, + 4, + 5, + 6 + ] + """; + var json = Json.Builder.newArray() + .addAll(Json.Builder.Array.newArray().add(1).add(2).add(3)) + .add(4) + .addAll(Json.Builder.Array.newArray().add(5)) + .add(6) + .build() + .toJson(true); + assertEquals(expected, json); + } + + @Test + void collectors() { + var expected = Json.Builder.Array.newArray() + .add(1).add(2).add(3).add(4).add(5).add(6) + .build() + .toJson(false); + var actual = Stream.of(1, 2, 3, 4, 5, 6) + .collect(Json.Collectors.toArray(Json.Builder.Array::add)) + .toJson(false); + assertEquals(expected, actual); + } } diff --git a/vespajlib/src/test/java/ai/vespa/net/CidrBlockTest.java b/vespajlib/src/test/java/ai/vespa/net/CidrBlockTest.java index f8cf5463cd1..9b48c23a69d 100644 --- a/vespajlib/src/test/java/ai/vespa/net/CidrBlockTest.java +++ b/vespajlib/src/test/java/ai/vespa/net/CidrBlockTest.java @@ -2,7 +2,6 @@ package ai.vespa.net; import com.google.common.net.InetAddresses; -import ai.vespa.net.CidrBlock; import org.junit.Test; import java.net.InetAddress; @@ -99,7 +98,7 @@ public class CidrBlockTest { assertEquals(StreamSupport.stream(superBlock.iterableIps().spliterator(), false) .skip(24) - .map(ip -> InetAddresses.toAddrString(ip) + "/32") + .map(ip -> InetAddressUtil.toString(ip) + "/32") .collect(Collectors.toList()), StreamSupport.stream(CidrBlock.fromString("10.12.14.24/32").iterableCidrs().spliterator(), false) .takeWhile(superBlock::overlapsWith) @@ -111,12 +110,12 @@ public class CidrBlockTest { public void iterableIps() { assertEquals(List.of("10.12.14.24", "10.12.14.25", "10.12.14.26", "10.12.14.27", "10.12.14.28", "10.12.14.29", "10.12.14.30", "10.12.14.31"), StreamSupport.stream(CidrBlock.fromString("10.12.14.24/29").iterableIps().spliterator(), false) - .map(InetAddresses::toAddrString) + .map(InetAddressUtil::toString) .collect(Collectors.toList())); assertEquals(List.of("10.12.14.24"), StreamSupport.stream(CidrBlock.fromString("10.12.14.24/32").iterableIps().spliterator(), false) - .map(InetAddresses::toAddrString) + .map(InetAddressUtil::toString) .collect(Collectors.toList())); } diff --git a/vespajlib/src/test/java/ai/vespa/net/InetAddressUtilTest.java b/vespajlib/src/test/java/ai/vespa/net/InetAddressUtilTest.java new file mode 100644 index 00000000000..523d8a4fabf --- /dev/null +++ b/vespajlib/src/test/java/ai/vespa/net/InetAddressUtilTest.java @@ -0,0 +1,55 @@ +package ai.vespa.net; + +import com.google.common.net.InetAddresses; +import org.junit.jupiter.api.Test; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.UnknownHostException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +/** + * @author hakonhall + */ +class InetAddressUtilTest { + @Test + void testToString() { + assertEquals("1080::8:800:200c:417a", toString("1080:0:0:0:8:800:200C:417A")); + assertEquals("1080::8:0:0:417a", toString("1080:0:0:0:8:0:0:417A")); + assertEquals("::8190:3426", toString("::129.144.52.38")); + assertEquals("::", toString("::")); + + assertEquals("0.0.0.0", toString("0.0.0.0")); + assertEquals("222.173.190.239", toString("222.173.190.239")); + } + + @Test + void testToStringWithInterface() throws SocketException { + NetworkInterface.networkInterfaces() + .flatMap(NetworkInterface::inetAddresses) + .forEach(inetAddress -> { + String address = InetAddressUtil.toString(inetAddress); + assertEquals(-1, address.indexOf('%'), "No interface in " + address); + }); + } + + @Test + void testToStringWithInterface2() throws UnknownHostException { + byte[] bytes = new byte[] { 0x10,(byte)0x80, 0,0, 0,0, 0,0, 0,8, 8,0, 0x20,0x0c, 0x41,0x7a }; + Inet6Address address = Inet6Address.getByAddress(null, bytes, 1); + // Verify Guava's InetAddresses.toAddrString() includes the interface. + // If this assert fails, we can use InetAddresses.toAddrString() instead of InetAddressUtil.toString(). + assertNotEquals(-1, InetAddresses.toAddrString(address).indexOf('%')); + assertEquals("1080::8:800:200c:417a", InetAddressUtil.toString(address)); + } + + private static String toString(String ipAddress) { + InetAddress inetAddress = InetAddresses.forString(ipAddress); + return InetAddressUtil.toString(inetAddress); + } + +} diff --git a/vespajlib/src/test/java/ai/vespa/utils/BytesQuantityTest.java b/vespajlib/src/test/java/ai/vespa/utils/BytesQuantityTest.java new file mode 100644 index 00000000000..e8e5d6275a3 --- /dev/null +++ b/vespajlib/src/test/java/ai/vespa/utils/BytesQuantityTest.java @@ -0,0 +1,47 @@ +package ai.vespa.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author bjorncs + */ +class BytesQuantityTest { + + @Test + void from_string() { + assertEquals(0L, BytesQuantity.fromString("0 bytes").toBytes()); + assertEquals(1L, BytesQuantity.fromString("1 byte").toBytes()); + assertEquals(10L, BytesQuantity.fromString("10 bytes").toBytes()); + assertEquals(1L, BytesQuantity.fromString("1 B").toBytes()); + assertEquals(1L, BytesQuantity.fromString("1B").toBytes()); + assertEquals(1L, BytesQuantity.fromString("1").toBytes()); + assertEquals(1024L, BytesQuantity.fromString("1kB").toBytes()); + assertEquals(1024L, BytesQuantity.fromString("1k").toBytes()); + assertEquals(1024L, BytesQuantity.fromString("1K").toBytes()); + assertEquals(1024L, BytesQuantity.fromString("1KB").toBytes()); + assertEquals(1024L * 1024, BytesQuantity.fromString("1MB").toBytes()); + assertEquals(1024L * 1024, BytesQuantity.fromString("1M").toBytes()); + assertEquals(1024L * 1024 * 1024, BytesQuantity.fromString("1GB").toBytes()); + assertEquals(1024L * 1024 * 1024, BytesQuantity.fromString("1G").toBytes()); + assertEquals(1024L * 1024 * 1024 * 1024, BytesQuantity.fromString("1TB").toBytes()); + assertEquals(1024L * 1024 * 1024 * 1024, BytesQuantity.fromString("1T").toBytes()); + } + + @Test + void as_pretty_string() { + assertEquals("0 bytes", BytesQuantity.ofBytes(0).asPrettyString()); + assertEquals("1 byte", BytesQuantity.ofBytes(1).asPrettyString()); + assertEquals("10 bytes", BytesQuantity.ofBytes(10).asPrettyString()); + assertEquals("1 kB", BytesQuantity.ofBytes(1024).asPrettyString()); + assertEquals("2 kB", BytesQuantity.ofKB(2).asPrettyString()); + assertEquals("3 MB", BytesQuantity.ofMB(3).asPrettyString()); + assertEquals("4 GB", BytesQuantity.ofGB(4).asPrettyString()); + assertEquals("5 TB", BytesQuantity.ofTB(5).asPrettyString()); + + assertEquals("2560 bytes", BytesQuantity.ofBytes(2048 + 512).asPrettyString()); + assertEquals("3073 kB", BytesQuantity.ofBytes(3 * 1024 * 1024 + 1024).asPrettyString()); + assertEquals("5120 TB", BytesQuantity.ofBytes(1024L * 1024 * 1024 * 1024 * 1024 * 5).asPrettyString()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java index 3ed8a7237ec..4ab60ecb9b9 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java @@ -2,9 +2,14 @@ package com.yahoo.tensor; +import com.yahoo.tensor.functions.Reduce; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -159,4 +164,25 @@ public class MixedTensorTestCase { tensor.toString()); } + @Test + public void testSplitIntoDense() { + TensorType type = new TensorType.Builder().mapped("key").indexed("x", 3).build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("key", "key1").label("x", 0).value(1). + cell().label("key", "key1").label("x", 1).value(2). + cell().label("key", "key1").label("x", 2).value(3). + cell().label("key", "key2").label("x", 0).value(4). + cell().label("key", "key2").label("x", 1).value(5). + cell().label("key", "key2").label("x", 2).value(6). + build(); + + Map<String, Tensor> indexedTensors = new HashMap<>(); + tensor.sum("x").cellIterator() + .forEachRemaining(cell -> indexedTensors.put(cell.getKey().label(0), + tensor.multiply(Tensor.Builder.of(type.mappedSubtype()).cell(cell.getKey(), 1.0).build()).sum("key"))); + + assertEquals("tensor(x[3]):[1.0, 2.0, 3.0]", indexedTensors.get("key1").toString()); + assertEquals("tensor(x[3]):[4.0, 5.0, 6.0]", indexedTensors.get("key2").toString()); + } + } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java index 5a049eeca04..7bc0556987b 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java @@ -261,6 +261,8 @@ public class TensorParserTestCase { "tensor(x[3]):[1, 2]"); assertIllegal("At value position 8: Expected a ']' but got ','", "tensor(x[3]):[1, 2, 3, 4]"); + assertIllegal("No suitable dimension in tensor(x[3]) for parsing a tensor on the mixed form: Should have one mapped dimension", + "tensor(x[3]):{1:[1,2,3], 2:[2,3,4], 3:[3,4,5]}"); } private void assertIllegal(String message, String tensor) { diff --git a/vespajlib/src/test/java/com/yahoo/text/XMLTestCase.java b/vespajlib/src/test/java/com/yahoo/text/XMLTestCase.java index 1565d067a09..11b873968a7 100644 --- a/vespajlib/src/test/java/com/yahoo/text/XMLTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/XMLTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.text; import org.junit.Test; +import org.w3c.dom.Document; import java.io.StringReader; @@ -11,10 +12,9 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; - /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Bjorn Borud + * @author Steinar Knutsen */ public class XMLTestCase { @@ -30,13 +30,10 @@ public class XMLTestCase { assertEquals("this is a & test", XML.xmlEscape(s2, true)); // quotes are only escaped in attributes - // assertEquals("this is a " test", XML.xmlEscape(s3, true)); assertEquals("this is a \" test", XML.xmlEscape(s3, false)); - // quotes are only escaped in attributes. prevent bug - // no. 187006 from happening again! - // + // quotes are only escaped in attributes assertEquals("this is a <" test", XML.xmlEscape(s4, true)); assertEquals("this is a <\" test", XML.xmlEscape(s4, false)); @@ -112,4 +109,20 @@ public class XMLTestCase { assertTrue(e.getMessage().contains("error at line 2, column 5")); } } + + @Test + public void testParseAndWrite() { + String xml = """ + <foo> + <bar baz="quux"/> + <moo baah="boo">mux</moo> + <!-- zoink --> + </foo>"""; + Document document = XML.getDocument(xml); + assertEquals(xml, XML.toString(document)); + assertEquals(xml, XML.toString(document.getDocumentElement())); + assertEquals("<bar baz=\"quux\"/>", XML.toString(XML.getChild(document.getDocumentElement(), "bar"))); + assertEquals("mux", XML.toString(XML.getChild(document.getDocumentElement(), "moo").getFirstChild())); + } + } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 9b430bdd913..49e0438fec4 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -100,7 +100,7 @@ vespa_define_module( src/tests/growablebytebuffer src/tests/guard src/tests/host_name - src/tests/hwaccelrated + src/tests/hwaccelerated src/tests/invokeservice src/tests/io/fileutil src/tests/io/mapped_file_input @@ -233,7 +233,7 @@ vespa_define_module( src/vespa/vespalib/encoding src/vespa/vespalib/fuzzy src/vespa/vespalib/geo - src/vespa/vespalib/hwaccelrated + src/vespa/vespalib/hwaccelerated src/vespa/vespalib/io src/vespa/vespalib/locale src/vespa/vespalib/metrics diff --git a/vespalib/src/apps/vespa-detect-hostname/detect_hostname.cpp b/vespalib/src/apps/vespa-detect-hostname/detect_hostname.cpp index 6a01f616c11..978c6653352 100644 --- a/vespalib/src/apps/vespa-detect-hostname/detect_hostname.cpp +++ b/vespalib/src/apps/vespa-detect-hostname/detect_hostname.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdio.h> -#include <stdlib.h> +#include <cstdio> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/vespalib/src/apps/vespa-validate-hostname/validate_hostname.cpp b/vespalib/src/apps/vespa-validate-hostname/validate_hostname.cpp index dc677e4e4fa..37d9b8a349f 100644 --- a/vespalib/src/apps/vespa-validate-hostname/validate_hostname.cpp +++ b/vespalib/src/apps/vespa-validate-hostname/validate_hostname.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdio.h> -#include <stdlib.h> +#include <cstdio> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/stllike/string.h> #include <set> diff --git a/vespalib/src/tests/alloc/alloc_test.cpp b/vespalib/src/tests/alloc/alloc_test.cpp index 1436724267d..566ef8d9716 100644 --- a/vespalib/src/tests/alloc/alloc_test.cpp +++ b/vespalib/src/tests/alloc/alloc_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/config.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/memory_allocator.h> #include <vespa/vespalib/util/exceptions.h> diff --git a/vespalib/src/tests/array/array_test.cpp b/vespalib/src/tests/array/array_test.cpp index 48e82b56e7d..5446130d5d9 100644 --- a/vespalib/src/tests/array/array_test.cpp +++ b/vespalib/src/tests/array/array_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/util/round_up_to_page_size.h> diff --git a/vespalib/src/tests/barrier/barrier_test.cpp b/vespalib/src/tests/barrier/barrier_test.cpp index eba57f5381f..18032d92ce0 100644 --- a/vespalib/src/tests/barrier/barrier_test.cpp +++ b/vespalib/src/tests/barrier/barrier_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/barrier.h> +#include <vespa/vespalib/util/count_down_latch.h> using namespace vespalib; diff --git a/vespalib/src/tests/benchmark/benchmark.cpp b/vespalib/src/tests/benchmark/benchmark.cpp index d5805a04cbc..771ea8be129 100644 --- a/vespalib/src/tests/benchmark/benchmark.cpp +++ b/vespalib/src/tests/benchmark/benchmark.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include "testbase.h" #include <vespa/log/log.h> @@ -7,24 +7,16 @@ LOG_SETUP("benchmark_test"); using namespace vespalib; -TEST_SETUP(Test) - -int -Test::Main() -{ - TEST_INIT("benchmark_test"); - - if (_argc > 1) { +TEST_MAIN() { + if (argc > 1) { size_t concurrency(1); size_t numRuns(1000); - if (_argc > 2) { - numRuns = strtoul(_argv[2], NULL, 0); - if (_argc > 3) { - concurrency = strtoul(_argv[3], NULL, 0); + if (argc > 2) { + numRuns = strtoul(argv[2], NULL, 0); + if (argc > 3) { + concurrency = strtoul(argv[3], NULL, 0); } } - Benchmark::run(_argv[1], numRuns, concurrency); + Benchmark::run(argv[1], numRuns, concurrency); } - - TEST_DONE(); } diff --git a/vespalib/src/tests/component/component.cpp b/vespalib/src/tests/component/component.cpp index 01d006d8aa8..15b381f6ba2 100644 --- a/vespalib/src/tests/component/component.cpp +++ b/vespalib/src/tests/component/component.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("component_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/component/version.h> diff --git a/vespalib/src/tests/compression/compression_test.cpp b/vespalib/src/tests/compression/compression_test.cpp index 264f21aeefe..2257b57dc7e 100644 --- a/vespalib/src/tests/compression/compression_test.cpp +++ b/vespalib/src/tests/compression/compression_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> +#include <atomic> #include <vespa/log/log.h> LOG_SETUP("compression_test"); diff --git a/vespalib/src/tests/data/memory_input/memory_input_test.cpp b/vespalib/src/tests/data/memory_input/memory_input_test.cpp index b3ed505b6e5..418a595eadb 100644 --- a/vespalib/src/tests/data/memory_input/memory_input_test.cpp +++ b/vespalib/src/tests/data/memory_input/memory_input_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/memory_input.h> using namespace vespalib; diff --git a/vespalib/src/tests/data/output_writer/output_writer_test.cpp b/vespalib/src/tests/data/output_writer/output_writer_test.cpp index b3090624336..2fcc77c929f 100644 --- a/vespalib/src/tests/data/output_writer/output_writer_test.cpp +++ b/vespalib/src/tests/data/output_writer/output_writer_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/data/output_writer.h> diff --git a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp index 01233a20fb5..f64287d334f 100644 --- a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp +++ b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/datastore/entryref.h> #include <vespa/vespalib/datastore/array_store_config.h> #include <vespa/vespalib/util/size_literals.h> diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index b3af0c84b2d..d2dcf99081b 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -1,7 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/datastore/buffer_type.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <cassert> using namespace vespalib::datastore; diff --git a/vespalib/src/tests/dotproduct/CMakeLists.txt b/vespalib/src/tests/dotproduct/CMakeLists.txt index ff25f14e02c..41f8ba5c1cf 100644 --- a/vespalib/src/tests/dotproduct/CMakeLists.txt +++ b/vespalib/src/tests/dotproduct/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(vespalib_dotproductbenchmark_app dotproductbenchmark.cpp DEPENDS vespalib + vespa_hwaccelerated ) vespa_add_test(NAME vespalib_dotproductbenchmark_app_sparse-ordered COMMAND vespalib_dotproductbenchmark_app 10 10 1000 1000 BENCHMARK) vespa_add_test(NAME vespalib_dotproductbenchmark_app_sparse-unordered COMMAND vespalib_dotproductbenchmark_app 10 10 1000 1000 BENCHMARK) diff --git a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp index ce97683d1fc..ef88c8e94c3 100644 --- a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp +++ b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> #include <iostream> @@ -8,7 +8,7 @@ #include <functional> using namespace vespalib; -using vespalib::hwaccelrated::IAccelrated; +using vespalib::hwaccelerated::IAccelerated; class Benchmark { public: @@ -54,14 +54,14 @@ public: private: std::vector<T> _values; std::vector<T> _query; - const IAccelrated & _dp; + const IAccelerated & _dp; }; template <typename T> FullBenchmark<T>::FullBenchmark(size_t numDocs, size_t numValues) : _values(numDocs*numValues), _query(numValues), - _dp(IAccelrated::getAccelerator()) + _dp(IAccelerated::getAccelerator()) { for (size_t i(0); i < numDocs; i++) { for (size_t j(0); j < numValues; j++) { diff --git a/vespalib/src/tests/encoding/base64/base64_test.cpp b/vespalib/src/tests/encoding/base64/base64_test.cpp index 2b058b33b49..21ff9af9162 100644 --- a/vespalib/src/tests/encoding/base64/base64_test.cpp +++ b/vespalib/src/tests/encoding/base64/base64_test.cpp @@ -1,17 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/encoding/base64.h> #include <vector> using namespace vespalib; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("base64_test"); +TEST("base64_test") { // Basic test without padding std::string source = "No need to pad this string."; @@ -78,6 +73,6 @@ Test::Main() EXPECT_EQUAL(60, Base64::decode(encoded.c_str(), encoded.size(), &buffer[0], minSizeNeeded)); EXPECT_EQUAL(source, std::string(&buffer[0], 60)); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/executor/executor_test.cpp b/vespalib/src/tests/executor/executor_test.cpp index afe37710088..5fcc23fe618 100644 --- a/vespalib/src/tests/executor/executor_test.cpp +++ b/vespalib/src/tests/executor/executor_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/executor_stats.h> diff --git a/vespalib/src/tests/false/false.cpp b/vespalib/src/tests/false/false.cpp index e602bc9570e..1c86308a6ee 100644 --- a/vespalib/src/tests/false/false.cpp +++ b/vespalib/src/tests/false/false.cpp @@ -1,14 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("false_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> -TEST_SETUP(Test) - -int -Test::Main() -{ - TEST_INIT("false_test"); +TEST("false_test") { EXPECT_TRUE(false); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/gencnt/gencnt_test.cpp b/vespalib/src/tests/gencnt/gencnt_test.cpp index d689abb3b9c..1932c8ea1e6 100644 --- a/vespalib/src/tests/gencnt/gencnt_test.cpp +++ b/vespalib/src/tests/gencnt/gencnt_test.cpp @@ -1,17 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("gencnt_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/gencnt.h> using vespalib::GenCnt; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("gencnt_test"); +TEST("gencnt_test") { GenCnt first; @@ -80,6 +75,6 @@ Test::Main() EXPECT_TRUE(b.distance(c) == 10); EXPECT_TRUE(!first.inRangeInclusive(a, c)); EXPECT_TRUE(!first.inRangeInclusive(c, a)); - - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/guard/guard_test.cpp b/vespalib/src/tests/guard/guard_test.cpp index 2efe66201f9..c310f99ba3f 100644 --- a/vespalib/src/tests/guard/guard_test.cpp +++ b/vespalib/src/tests/guard/guard_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/guard.h> #include <fcntl.h> #include <unistd.h> diff --git a/vespalib/src/tests/hwaccelerated/.gitignore b/vespalib/src/tests/hwaccelerated/.gitignore new file mode 100644 index 00000000000..659184bbac3 --- /dev/null +++ b/vespalib/src/tests/hwaccelerated/.gitignore @@ -0,0 +1 @@ +vespalib_hwaccelerated_bench_app diff --git a/vespalib/src/tests/hwaccelerated/CMakeLists.txt b/vespalib/src/tests/hwaccelerated/CMakeLists.txt new file mode 100644 index 00000000000..044cfa2e9a3 --- /dev/null +++ b/vespalib/src/tests/hwaccelerated/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_hwaccelerated_test_app TEST + SOURCES + hwaccelerated_test.cpp + DEPENDS + vespalib + vespa_hwaccelerated +) +vespa_add_test(NAME vespalib_hwaccelerated_test_app COMMAND vespalib_hwaccelerated_test_app) + +vespa_add_executable(vespalib_hwaccelerated_bench_app + SOURCES + hwaccelerated_bench.cpp + DEPENDS + vespalib + vespa_hwaccelerated +) diff --git a/vespalib/src/tests/hwaccelrated/hwaccelrated_bench.cpp b/vespalib/src/tests/hwaccelerated/hwaccelerated_bench.cpp index 61c53a20cf5..cebdeeab28b 100644 --- a/vespalib/src/tests/hwaccelrated/hwaccelrated_bench.cpp +++ b/vespalib/src/tests/hwaccelerated/hwaccelerated_bench.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> -#include <vespa/vespalib/hwaccelrated/generic.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> +#include <vespa/vespalib/hwaccelerated/generic.h> #include <vespa/vespalib/util/time.h> #include <cinttypes> @@ -18,7 +18,7 @@ std::vector<T> createAndFill(size_t sz) { template<typename T> void -benchmarkEuclideanDistance(const hwaccelrated::IAccelrated & accel, size_t sz, size_t count) { +benchmarkEuclideanDistance(const hwaccelerated::IAccelerated & accel, size_t sz, size_t count) { srand(1); std::vector<T> a = createAndFill<T>(sz); std::vector<T> b = createAndFill<T>(sz); @@ -33,7 +33,7 @@ benchmarkEuclideanDistance(const hwaccelrated::IAccelrated & accel, size_t sz, s } void -benchMarkEuclidianDistance(const hwaccelrated::IAccelrated & accelrator, size_t sz, size_t count) { +benchMarkEuclidianDistance(const hwaccelerated::IAccelerated & accelrator, size_t sz, size_t count) { printf("double : "); benchmarkEuclideanDistance<double>(accelrator, sz, count); printf("float : "); @@ -53,8 +53,8 @@ int main(int argc, char *argv[]) { } printf("%s %d %d\n", argv[0], length, count); printf("Squared Euclidian Distance - Generic\n"); - benchMarkEuclidianDistance(hwaccelrated::GenericAccelrator(), length, count); + benchMarkEuclidianDistance(hwaccelerated::GenericAccelrator(), length, count); printf("Squared Euclidian Distance - Optimized for this cpu\n"); - benchMarkEuclidianDistance(hwaccelrated::IAccelrated::getAccelerator(), length, count); + benchMarkEuclidianDistance(hwaccelerated::IAccelerated::getAccelerator(), length, count); return 0; } diff --git a/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp b/vespalib/src/tests/hwaccelerated/hwaccelerated_test.cpp index e35efb20c1a..3305b911de3 100644 --- a/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp +++ b/vespalib/src/tests/hwaccelerated/hwaccelerated_test.cpp @@ -1,10 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> -#include <vespa/vespalib/hwaccelrated/generic.h> +#include <vespa/vespalib/hwaccelerated/iaccelerated.h> +#include <vespa/vespalib/hwaccelerated/generic.h> #include <vespa/log/log.h> -LOG_SETUP("hwaccelrated_test"); +LOG_SETUP("hwaccelerated_test"); using namespace vespalib; @@ -18,7 +18,7 @@ std::vector<T> createAndFill(size_t sz) { } template<typename T, typename P> -void verifyEuclideanDistance(const hwaccelrated::IAccelrated & accel, size_t testLength, double approxFactor) { +void verifyEuclideanDistance(const hwaccelerated::IAccelerated & accel, size_t testLength, double approxFactor) { srand(1); std::vector<T> a = createAndFill<T>(testLength); std::vector<T> b = createAndFill<T>(testLength); @@ -34,17 +34,17 @@ void verifyEuclideanDistance(const hwaccelrated::IAccelrated & accel, size_t tes } void -verifyEuclideanDistance(const hwaccelrated::IAccelrated & accelrator, size_t testLength) { +verifyEuclideanDistance(const hwaccelerated::IAccelerated & accelrator, size_t testLength) { verifyEuclideanDistance<int8_t, double>(accelrator, testLength, 0.0); verifyEuclideanDistance<float, double>(accelrator, testLength, 0.0001); // Small deviation requiring EXPECT_APPROX verifyEuclideanDistance<double, double>(accelrator, testLength, 0.0); } TEST("test euclidean distance") { - hwaccelrated::GenericAccelrator genericAccelrator; + hwaccelerated::GenericAccelrator genericAccelrator; constexpr size_t TEST_LENGTH = 140000; // must be longer than 64k - TEST_DO(verifyEuclideanDistance(hwaccelrated::GenericAccelrator(), TEST_LENGTH)); - TEST_DO(verifyEuclideanDistance(hwaccelrated::IAccelrated::getAccelerator(), TEST_LENGTH)); + TEST_DO(verifyEuclideanDistance(hwaccelerated::GenericAccelrator(), TEST_LENGTH)); + TEST_DO(verifyEuclideanDistance(hwaccelerated::IAccelerated::getAccelerator(), TEST_LENGTH)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/hwaccelrated/.gitignore b/vespalib/src/tests/hwaccelrated/.gitignore deleted file mode 100644 index 42f73a39d78..00000000000 --- a/vespalib/src/tests/hwaccelrated/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespalib_hwaccelrated_bench_app diff --git a/vespalib/src/tests/hwaccelrated/CMakeLists.txt b/vespalib/src/tests/hwaccelrated/CMakeLists.txt deleted file mode 100644 index b5e8f7daa2c..00000000000 --- a/vespalib/src/tests/hwaccelrated/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_hwaccelrated_test_app TEST - SOURCES - hwaccelrated_test.cpp - DEPENDS - vespalib -) -vespa_add_test(NAME vespalib_hwaccelrated_test_app COMMAND vespalib_hwaccelrated_test_app) - -vespa_add_executable(vespalib_hwaccelrated_bench_app - SOURCES - hwaccelrated_bench.cpp - DEPENDS - vespalib -) diff --git a/vespalib/src/tests/memory/memory_test.cpp b/vespalib/src/tests/memory/memory_test.cpp index 4b6815196c2..be413fd1a95 100644 --- a/vespalib/src/tests/memory/memory_test.cpp +++ b/vespalib/src/tests/memory/memory_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/memory.h> using namespace vespalib; diff --git a/vespalib/src/tests/memorydatastore/memorydatastore.cpp b/vespalib/src/tests/memorydatastore/memorydatastore.cpp index fbbc1e76dff..3efab9e0178 100644 --- a/vespalib/src/tests/memorydatastore/memorydatastore.cpp +++ b/vespalib/src/tests/memorydatastore/memorydatastore.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/memorydatastore.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/vespalib/src/tests/metrics/simple_metrics_test.cpp b/vespalib/src/tests/metrics/simple_metrics_test.cpp index 8d751bf3528..a5b22b6726d 100644 --- a/vespalib/src/tests/metrics/simple_metrics_test.cpp +++ b/vespalib/src/tests/metrics/simple_metrics_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/json_format.h> #include <vespa/vespalib/metrics/simple_metrics.h> diff --git a/vespalib/src/tests/metrics/stable_store_test.cpp b/vespalib/src/tests/metrics/stable_store_test.cpp index 026e6f5dfef..33bff5eec4a 100644 --- a/vespalib/src/tests/metrics/stable_store_test.cpp +++ b/vespalib/src/tests/metrics/stable_store_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/metrics/simple_metrics.h> #include <vespa/vespalib/metrics/simple_metrics_manager.h> #include <vespa/vespalib/metrics/stable_store.h> diff --git a/vespalib/src/tests/net/send_fd/CMakeLists.txt b/vespalib/src/tests/net/send_fd/CMakeLists.txt index 4c46a773f5c..b4baaeb5855 100644 --- a/vespalib/src/tests/net/send_fd/CMakeLists.txt +++ b/vespalib/src/tests/net/send_fd/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(vespalib_send_fd_test_app TEST send_fd_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_send_fd_test_app COMMAND vespalib_send_fd_test_app) diff --git a/vespalib/src/tests/net/send_fd/send_fd_test.cpp b/vespalib/src/tests/net/send_fd/send_fd_test.cpp index 59b9aacea07..8dc1235ff76 100644 --- a/vespalib/src/tests/net/send_fd/send_fd_test.cpp +++ b/vespalib/src/tests/net/send_fd/send_fd_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/selector.h> @@ -7,15 +7,19 @@ #include <vespa/vespalib/net/server_socket.h> #include <vespa/vespalib/net/socket_options.h> #include <vespa/vespalib/net/socket.h> +#include <vespa/vespalib/test/nexus.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/test/socket_options_verifier.h> -#include <thread> -#include <functional> #include <chrono> +#include <functional> +#include <latch> +#include <optional> +#include <thread> #include <unistd.h> #include <sys/stat.h> using namespace vespalib; +using vespalib::test::Nexus; vespalib::string read_bytes(SocketHandle &socket, size_t wanted_bytes) { char tmp[64]; @@ -23,7 +27,9 @@ vespalib::string read_bytes(SocketHandle &socket, size_t wanted_bytes) { while (result.size() < wanted_bytes) { size_t read_size = std::min(sizeof(tmp), wanted_bytes - result.size()); ssize_t read_result = socket.read(tmp, read_size); - ASSERT_GREATER(read_result, 0); + if (read_result <= 0) { + return result; + } result.append(tmp, read_result); } return result; @@ -34,14 +40,14 @@ void verify_socket_io(bool is_server, SocketHandle &socket) { vespalib::string client_message = "please pick up, I need to talk to you"; if(is_server) { ssize_t written = socket.write(server_message.data(), server_message.size()); - EXPECT_EQUAL(written, ssize_t(server_message.size())); + EXPECT_EQ(written, ssize_t(server_message.size())); vespalib::string read = read_bytes(socket, client_message.size()); - EXPECT_EQUAL(client_message, read); + EXPECT_EQ(client_message, read); } else { ssize_t written = socket.write(client_message.data(), client_message.size()); - EXPECT_EQUAL(written, ssize_t(client_message.size())); + EXPECT_EQ(written, ssize_t(client_message.size())); vespalib::string read = read_bytes(socket, server_message.size()); - EXPECT_EQUAL(server_message, read); + EXPECT_EQ(server_message, read); } } @@ -79,10 +85,10 @@ void send_fd(SocketHandle &socket, SocketHandle fd) { int *fd_dst = (int *) (void *) CMSG_DATA(hdr); fd_dst[0] = fd.get(); ssize_t res = sendmsg(socket.get(), &msg, 0); - ASSERT_EQUAL(res, 1); + ASSERT_EQ(res, 1); } -SocketHandle recv_fd(SocketHandle &socket) { +void recv_fd(SocketHandle &socket, std::optional<SocketHandle>& result) { struct msghdr msg = {}; char tag = '*'; struct iovec data; @@ -94,36 +100,62 @@ SocketHandle recv_fd(SocketHandle &socket) { msg.msg_control = buf; msg.msg_controllen = sizeof(buf); ssize_t res = recvmsg(socket.get(), &msg, 0); - ASSERT_EQUAL(res, 1); + ASSERT_EQ(res, 1); struct cmsghdr *hdr = CMSG_FIRSTHDR(&msg); bool type_ok = ((hdr->cmsg_level == SOL_SOCKET) && (hdr->cmsg_type == SCM_RIGHTS)); ASSERT_TRUE(type_ok); int *fd_src = (int *) (void *) CMSG_DATA(hdr); fprintf(stderr, "got fd: %d\n", fd_src[0]); - return SocketHandle(fd_src[0]); + result = SocketHandle(fd_src[0]); } //----------------------------------------------------------------------------- -TEST_MT_FFF("require that an open socket (handle) can be passed over a unix domain socket", 3, - ServerSocket("tcp/0"), ServerSocket("ipc/file:my_socket"), TimeBomb(60)) -{ - if (thread_id == 0) { // server - SocketHandle socket = accept(f1); - TEST_DO(verify_socket_io(true, socket)); // server side - TEST_BARRIER(); - } else if (thread_id == 1) { // proxy - SocketHandle server_socket = connect(f1); - SocketHandle client_socket = accept(f2); - send_fd(client_socket, std::move(server_socket)); - TEST_BARRIER(); - } else { // client - SocketHandle proxy_socket = connect(f2); - SocketHandle socket = recv_fd(proxy_socket); - TEST_DO(verify_socket_io(false, socket)); // client side - TEST_BARRIER(); +namespace { + +class WaitLatch { + std::latch& _latch; +public: + explicit WaitLatch(std::latch& latch) noexcept + : _latch(latch) + { } + ~WaitLatch() { _latch.arrive_and_wait(); } +}; + +} + +TEST(SendFdTest, require_that_an_open_socket_handle_can_be_passed_over_a_unix_domain_socket) +{ + constexpr size_t num_threads = 3; + ServerSocket f1("tcp/0"); + ServerSocket f2("ipc/file:my_socket"); + std::latch latch(num_threads); + TimeBomb f3(60); + auto task = [&f1,&f2,&latch](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + if (thread_id == 0) { // server + SocketHandle socket = accept(f1); + WaitLatch wait(latch); + SCOPED_TRACE("verify socket io server side"); + verify_socket_io(true, socket); // server side + } else if (thread_id == 1) { // proxy + SocketHandle server_socket = connect(f1); + SocketHandle client_socket = accept(f2); + WaitLatch wait(latch); + ASSERT_NO_FATAL_FAILURE(send_fd(client_socket, std::move(server_socket))); + } else { // client + SocketHandle proxy_socket = connect(f2); + std::optional<SocketHandle> socket; + WaitLatch wait(latch); + ASSERT_NO_FATAL_FAILURE(recv_fd(proxy_socket, socket)); + ASSERT_TRUE(socket.has_value()); + SCOPED_TRACE("verify socket io client side"); + verify_socket_io(false, socket.value()); // client side + } + }; + Nexus::run(num_threads, task); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/net/socket/CMakeLists.txt b/vespalib/src/tests/net/socket/CMakeLists.txt index d20720971d2..49c4ca20ba3 100644 --- a/vespalib/src/tests/net/socket/CMakeLists.txt +++ b/vespalib/src/tests/net/socket/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(vespalib_socket_test_app TEST socket_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_socket_test_app COMMAND vespalib_socket_test_app) vespa_add_executable(vespalib_socket_server_app diff --git a/vespalib/src/tests/net/socket/socket_test.cpp b/vespalib/src/tests/net/socket/socket_test.cpp index b0708388cd6..64f337c03f6 100644 --- a/vespalib/src/tests/net/socket/socket_test.cpp +++ b/vespalib/src/tests/net/socket/socket_test.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/selector.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/server_socket.h> #include <vespa/vespalib/net/socket_options.h> #include <vespa/vespalib/net/socket.h> +#include <vespa/vespalib/test/nexus.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/test/socket_options_verifier.h> #include <thread> @@ -14,19 +15,55 @@ #include <sys/stat.h> using namespace vespalib; +using vespalib::test::Nexus; -bool ipv4_enabled = false; -bool ipv6_enabled = false; +class SocketTest : public ::testing::Test { +protected: + static bool ipv4_enabled; + static bool ipv6_enabled; -int my_inet() { + SocketTest(); + ~SocketTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); + static int my_inet(); +}; + +bool SocketTest::ipv4_enabled = false; +bool SocketTest::ipv6_enabled = false; + +SocketTest::SocketTest() + : testing::Test() +{ +} + +SocketTest::~SocketTest() = default; + +void +SocketTest::SetUpTestSuite() +{ + auto list = SocketAddress::resolve(4080); + for (const auto &addr : list) { + (void) addr; + ipv4_enabled |= addr.is_ipv4(); + ipv6_enabled |= addr.is_ipv6(); + } + ASSERT_TRUE(ipv4_enabled || ipv6_enabled) << "tcp/ip support not detected"; +} + +void +SocketTest::TearDownTestSuite() +{ +} + +int +SocketTest::my_inet() +{ if (ipv6_enabled) { return AF_INET6; - } - if (ipv4_enabled) { + } else { return AF_INET; } - TEST_ERROR("tcp/ip support not detected"); - return AF_UNIX; } bool is_socket(const vespalib::string &path) { @@ -52,8 +89,8 @@ void remove_file(const vespalib::string &path) { void replace_file(const vespalib::string &path, const vespalib::string &data) { remove_file(path); int fd = creat(path.c_str(), 0600); - ASSERT_NOT_EQUAL(fd, -1); - ASSERT_EQUAL(write(fd, data.data(), data.size()), ssize_t(data.size())); + ASSERT_NE(fd, -1); + ASSERT_EQ(write(fd, data.data(), data.size()), ssize_t(data.size())); close(fd); } @@ -96,14 +133,14 @@ void verify_socket_io(bool is_server, SocketHandle &socket) { vespalib::string client_message = "please pick up, I need to talk to you"; if(is_server) { ssize_t written = socket.write(server_message.data(), server_message.size()); - EXPECT_EQUAL(written, ssize_t(server_message.size())); + EXPECT_EQ(written, ssize_t(server_message.size())); vespalib::string read = read_bytes(socket, client_message.size()); - EXPECT_EQUAL(client_message, read); + EXPECT_EQ(client_message, read); } else { ssize_t written = socket.write(client_message.data(), client_message.size()); - EXPECT_EQUAL(written, ssize_t(client_message.size())); + EXPECT_EQ(written, ssize_t(client_message.size())); vespalib::string read = read_bytes(socket, server_message.size()); - EXPECT_EQUAL(server_message, read); + EXPECT_EQ(server_message, read); } } @@ -122,22 +159,22 @@ SocketHandle connect_sockets(bool is_server, ServerSocket &server_socket) { //----------------------------------------------------------------------------- -TEST("my local address") { +TEST_F(SocketTest, my_local_address) +{ auto list = SocketAddress::resolve(4080); fprintf(stderr, "resolve(4080):\n"); for (const auto &addr: list) { EXPECT_TRUE(addr.is_wildcard()); EXPECT_TRUE(addr.is_ipv4() || addr.is_ipv6()); - ipv4_enabled |= addr.is_ipv4(); - ipv6_enabled |= addr.is_ipv6(); EXPECT_TRUE(!addr.is_ipc()); EXPECT_TRUE(!addr.is_abstract()); - EXPECT_EQUAL(addr.port(), 4080); + EXPECT_EQ(addr.port(), 4080); fprintf(stderr, " %s (%s)\n", addr.spec().c_str(), get_meta(addr).c_str()); } } -TEST("yahoo.com address") { +TEST_F(SocketTest, yahoo_com_address) +{ auto list = SocketAddress::resolve(80, "yahoo.com"); fprintf(stderr, "resolve(80, 'yahoo.com'):\n"); for (const auto &addr: list) { @@ -145,80 +182,102 @@ TEST("yahoo.com address") { EXPECT_TRUE(addr.is_ipv4() || addr.is_ipv6()); EXPECT_TRUE(!addr.is_ipc()); EXPECT_TRUE(!addr.is_abstract()); - EXPECT_EQUAL(addr.port(), 80); + EXPECT_EQ(addr.port(), 80); fprintf(stderr, " %s (%s)\n", addr.spec().c_str(), get_meta(addr).c_str()); } } -TEST("ipc address (path)") { +TEST_F(SocketTest, ipc_address_with_path) +{ auto addr = SocketAddress::from_path("my_socket"); EXPECT_TRUE(!addr.is_ipv4()); EXPECT_TRUE(!addr.is_ipv6()); EXPECT_TRUE(addr.is_ipc()); EXPECT_TRUE(!addr.is_abstract()); EXPECT_TRUE(!addr.is_wildcard()); - EXPECT_EQUAL(addr.port(), -1); - EXPECT_EQUAL(vespalib::string("my_socket"), addr.path()); + EXPECT_EQ(addr.port(), -1); + EXPECT_EQ(vespalib::string("my_socket"), addr.path()); EXPECT_TRUE(addr.name().empty()); fprintf(stderr, "from_path(my_socket)\n"); fprintf(stderr, " %s (%s)\n", addr.spec().c_str(), get_meta(addr).c_str()); } -TEST("ipc address (name)") { +TEST_F(SocketTest, ipc_address_with_name) +{ auto addr = SocketAddress::from_name("my_socket"); EXPECT_TRUE(!addr.is_ipv4()); EXPECT_TRUE(!addr.is_ipv6()); EXPECT_TRUE(addr.is_ipc()); EXPECT_TRUE(addr.is_abstract()); EXPECT_TRUE(!addr.is_wildcard()); - EXPECT_EQUAL(addr.port(), -1); + EXPECT_EQ(addr.port(), -1); EXPECT_TRUE(addr.path().empty()); - EXPECT_EQUAL(vespalib::string("my_socket"), addr.name()); + EXPECT_EQ(vespalib::string("my_socket"), addr.name()); fprintf(stderr, "from_path(my_socket)\n"); fprintf(stderr, " %s (%s)\n", addr.spec().c_str(), get_meta(addr).c_str()); } -TEST("local client/server addresses") { +TEST_F(SocketTest, local_client_and_server_addresses) { auto spec = SocketSpec("tcp/123"); auto client = spec.client_address(); auto server = spec.server_address(); EXPECT_TRUE(!client.is_wildcard()); - EXPECT_EQUAL(client.port(), 123); + EXPECT_EQ(client.port(), 123); EXPECT_TRUE(server.is_wildcard()); - EXPECT_EQUAL(server.port(), 123); + EXPECT_EQ(server.port(), 123); fprintf(stderr, "client(tcp/123): %s (%s)\n", client.spec().c_str(), get_meta(client).c_str()); fprintf(stderr, "server(tcp/123): %s (%s)\n", server.spec().c_str(), get_meta(server).c_str()); } -TEST_MT_FF("require that basic socket io works", 2, ServerSocket("tcp/0"), TimeBomb(60)) { - bool is_server = (thread_id == 0); - SocketHandle socket = connect_sockets(is_server, f1); - TEST_DO(verify_socket_io(is_server, socket)); -} - -TEST_MT_FF("require that basic unix domain socket io works (path)", 2, - ServerSocket("ipc/file:my_socket"), TimeBomb(60)) +TEST_F(SocketTest, require_that_basic_socket_io_works) +{ + constexpr size_t num_threads = 2; + ServerSocket f1("tcp/0"); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + bool is_server = (ctx.thread_id() == 0); + SocketHandle socket = connect_sockets(is_server, f1); + verify_socket_io(is_server, socket); + }; + Nexus::run(num_threads, task); +} + +TEST_F(SocketTest, require_that_basic_unix_domain_socket_io_works_with_path) +{ + constexpr size_t num_threads = 2; + ServerSocket f1("ipc/file:my_socket"); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + bool is_server = (ctx.thread_id() == 0); + SocketHandle socket = connect_sockets(is_server, f1); + verify_socket_io(is_server, socket); + }; + Nexus::run(num_threads, task); +} + +TEST_F(SocketTest, require_that_server_accept_can_be_interrupted) +{ + constexpr size_t num_threads = 2; + ServerSocket f1("tcp/0"); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + bool is_server = (ctx.thread_id() == 0); + if (is_server) { + fprintf(stderr, "--> calling accept\n"); + SocketHandle socket = f1.accept(); + fprintf(stderr, "<-- accept returned\n"); + EXPECT_TRUE(!socket.valid()); + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + fprintf(stderr, "--- closing server socket\n"); + f1.shutdown(); + } + }; + Nexus::run(num_threads, task); +} + +TEST_F(SocketTest, require_that_socket_file_is_removed_by_server_socket_when_destructed) { - bool is_server = (thread_id == 0); - SocketHandle socket = connect_sockets(is_server, f1); - TEST_DO(verify_socket_io(is_server, socket)); -} - -TEST_MT_FF("require that server accept can be interrupted", 2, ServerSocket("tcp/0"), TimeBomb(60)) { - bool is_server = (thread_id == 0); - if (is_server) { - fprintf(stderr, "--> calling accept\n"); - SocketHandle socket = f1.accept(); - fprintf(stderr, "<-- accept returned\n"); - EXPECT_TRUE(!socket.valid()); - } else { - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - fprintf(stderr, "--- closing server socket\n"); - f1.shutdown(); - } -} - -TEST("require that socket file is removed by server socket when destructed") { remove_file("my_socket"); ServerSocket server("ipc/file:my_socket"); EXPECT_TRUE(server.valid()); @@ -227,7 +286,8 @@ TEST("require that socket file is removed by server socket when destructed") { EXPECT_TRUE(!is_socket("my_socket")); } -TEST("require that socket file is only removed on destruction if it is a socket") { +TEST_F(SocketTest, require_that_socket_file_is_only_removed_on_destruction_if_it_is_a_socket) +{ remove_file("my_socket"); ServerSocket server("ipc/file:my_socket"); EXPECT_TRUE(server.valid()); @@ -238,7 +298,8 @@ TEST("require that socket file is only removed on destruction if it is a socket" remove_file("my_socket"); } -TEST("require that a server socket will fail to listen to a path that is already a regular file") { +TEST_F(SocketTest, require_that_a_server_socket_will_fail_to_listen_to_a_path_that_is_already_a_regular_file) +{ replace_file("my_socket", "hello\n"); ServerSocket server("ipc/file:my_socket"); EXPECT_TRUE(!server.valid()); @@ -247,7 +308,8 @@ TEST("require that a server socket will fail to listen to a path that is already remove_file("my_socket"); } -TEST("require that a server socket will fail to listen to a path that is already taken by another server") { +TEST_F(SocketTest, require_that_a_server_socket_will_fail_to_listen_to_a_path_that_is_already_taken_by_another_server) +{ remove_file("my_socket"); ServerSocket server1("ipc/file:my_socket"); ServerSocket server2("ipc/file:my_socket"); @@ -258,7 +320,8 @@ TEST("require that a server socket will fail to listen to a path that is already EXPECT_TRUE(!is_socket("my_socket")); } -TEST("require that a server socket will remove an old socket file if it cannot be connected to") { +TEST_F(SocketTest, require_that_a_server_socket_will_remove_an_old_socket_file_if_it_cannot_be_connected_to) +{ remove_file("my_socket"); { SocketHandle server_handle = SocketAddress::from_path("my_socket").listen(); @@ -272,15 +335,21 @@ TEST("require that a server socket will remove an old socket file if it cannot b } #ifdef __linux__ -TEST_MT_FF("require that basic unix domain socket io works (name)", 2, - ServerSocket(make_string("ipc/name:my_socket-%d", int(getpid()))), TimeBomb(60)) +TEST_F(SocketTest, require_that_basic_unix_domain_socket_io_works_with_name) +{ + constexpr size_t num_threads = 2; + ServerSocket f1(make_string("ipc/name:my_socket-%d", int(getpid()))); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + bool is_server = (ctx.thread_id() == 0); + SocketHandle socket = connect_sockets(is_server, f1); + verify_socket_io(is_server, socket); + }; + Nexus::run(num_threads, task); +} + +TEST_F(SocketTest, require_that_two_server_sockets_cannot_have_the_same_abstract_unix_domain_socket_name) { - bool is_server = (thread_id == 0); - SocketHandle socket = connect_sockets(is_server, f1); - TEST_DO(verify_socket_io(is_server, socket)); -} - -TEST("require that two server sockets cannot have the same abstract unix domain socket name") { vespalib::string spec = make_string("ipc/name:my_socket-%d", int(getpid())); ServerSocket server1(spec); ServerSocket server2(spec); @@ -288,7 +357,8 @@ TEST("require that two server sockets cannot have the same abstract unix domain EXPECT_TRUE(!server2.valid()); } -TEST("require that abstract socket names are freed when the server socket is destructed") { +TEST_F(SocketTest, require_that_abstract_socket_names_are_freed_when_the_server_socket_is_destructed) +{ vespalib::string spec = make_string("ipc/name:my_socket-%d", int(getpid())); ServerSocket server1(spec); EXPECT_TRUE(server1.valid()); @@ -297,100 +367,162 @@ TEST("require that abstract socket names are freed when the server socket is des EXPECT_TRUE(server2.valid()); } -TEST("require that abstract sockets do not have socket files") { +TEST_F(SocketTest, require_that_abstract_sockets_do_not_have_socket_files) +{ vespalib::string name = make_string("my_socket-%d", int(getpid())); ServerSocket server(SocketSpec::from_name(name)); EXPECT_TRUE(server.valid()); EXPECT_TRUE(!is_socket(name)); - EXPECT_TRUE(!is_file(name)); + EXPECT_TRUE(!is_file(name)); } -TEST_MT_FFF("require that abstract and file-based unix domain sockets are not in conflict", 4, - ServerSocket(make_string("ipc/file:my_socket-%d", int(getpid()))), - ServerSocket(make_string("ipc/name:my_socket-%d", int(getpid()))), TimeBomb(60)) +TEST_F(SocketTest, require_that_abstract_and_file_based_unix_domain_sockets_are_not_in_conflict) { - bool is_server = ((thread_id % 2) == 0); - ServerSocket &server_socket = ((thread_id / 2) == 0) ? f1 : f2; - SocketHandle socket = connect_sockets(is_server, server_socket); - TEST_DO(verify_socket_io(is_server, socket)); + constexpr size_t num_threads = 4; + ServerSocket f1(make_string("ipc/file:my_socket-%d", int(getpid()))); + ServerSocket f2(make_string("ipc/name:my_socket-%d", int(getpid()))); + TimeBomb f3(60); + auto task = [&f1,&f2](Nexus& ctx) { + auto thread_id = ctx.thread_id(); + bool is_server = ((thread_id % 2) == 0); + ServerSocket &server_socket = ((thread_id / 2) == 0) ? f1 : f2; + SocketHandle socket = connect_sockets(is_server, server_socket); + verify_socket_io(is_server, socket); + }; + Nexus::run(num_threads, task); } #endif -TEST("require that sockets can be set blocking and non-blocking") { +TEST_F(SocketTest, require_that_sockets_can_be_set_blocking_and_non_blocking) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_blocking(-1, true)); EXPECT_TRUE(handle.set_blocking(true)); - TEST_DO(verifier.verify_blocking(true)); + { + SCOPED_TRACE("verify blocking true"); + verifier.verify_blocking(true); + } EXPECT_TRUE(handle.set_blocking(false)); - TEST_DO(verifier.verify_blocking(false)); + { + SCOPED_TRACE("verify blocking false"); + verifier.verify_blocking(false); + } } -TEST("require that server sockets use non-blocking underlying socket") { +TEST_F(SocketTest, require_that_server_sockets_use_non_blocking_underlying_socket) +{ ServerSocket tcp_server("tcp/0"); ServerSocket ipc_server("ipc/file:my_socket"); test::SocketOptionsVerifier tcp_verifier(tcp_server.get_fd()); test::SocketOptionsVerifier ipc_verifier(ipc_server.get_fd()); - TEST_DO(tcp_verifier.verify_blocking(false)); - TEST_DO(ipc_verifier.verify_blocking(false)); + { + SCOPED_TRACE("verify tcp nonblocking"); + tcp_verifier.verify_blocking(false); + } + { + SCOPED_TRACE("verify ipc nonblocking"); + ipc_verifier.verify_blocking(false); + } } -TEST("require that tcp nodelay can be enabled and disabled") { +TEST_F(SocketTest, require_that_tcp_nodelay_can_be_enabled_and_disabled) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_nodelay(-1, true)); EXPECT_TRUE(handle.set_nodelay(true)); - TEST_DO(verifier.verify_nodelay(true)); + { + SCOPED_TRACE("verify nodelay true"); + verifier.verify_nodelay(true); + } EXPECT_TRUE(handle.set_nodelay(false)); - TEST_DO(verifier.verify_nodelay(false)); + { + SCOPED_TRACE("verify nodelay false"); + verifier.verify_nodelay(false); + } } -TEST("require that reuse addr can be set and cleared") { +TEST_F(SocketTest, require_that_reuse_addr_can_be_set_and_cleared) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_reuse_addr(-1, true)); EXPECT_TRUE(handle.set_reuse_addr(true)); - TEST_DO(verifier.verify_reuse_addr(true)); + { + SCOPED_TRACE("verify reuse addr true"); + verifier.verify_reuse_addr(true); + } EXPECT_TRUE(handle.set_reuse_addr(false)); - TEST_DO(verifier.verify_reuse_addr(false)); + { + SCOPED_TRACE("verify reuse addr false"); + verifier.verify_reuse_addr(false); + } } -TEST("require that ipv6_only can be set and cleared") { +TEST_F(SocketTest, require_that_ipv6_only_can_be_set_and_cleared) +{ if (ipv6_enabled) { SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_ipv6_only(-1, true)); EXPECT_TRUE(handle.set_ipv6_only(true)); - TEST_DO(verifier.verify_ipv6_only(true)); + { + SCOPED_TRACE("verify ipv6 only true"); + verifier.verify_ipv6_only(true); + } EXPECT_TRUE(handle.set_ipv6_only(false)); - TEST_DO(verifier.verify_ipv6_only(false)); + { + SCOPED_TRACE("verify ipv6 only false"); + verifier.verify_ipv6_only(false); + } } else { fprintf(stderr, "WARNING: skipping ipv6_only test since ipv6 is disabled"); } } -TEST("require that tcp keepalive can be set and cleared") { +TEST_F(SocketTest, require_that_tcp_keepalive_can_be_set_and_cleared) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_keepalive(-1, true)); EXPECT_TRUE(handle.set_keepalive(true)); - TEST_DO(verifier.verify_keepalive(true)); + { + SCOPED_TRACE("verify keepalive true"); + verifier.verify_keepalive(true); + } EXPECT_TRUE(handle.set_keepalive(false)); - TEST_DO(verifier.verify_keepalive(false)); + { + SCOPED_TRACE("verify keepalive false"); + verifier.verify_keepalive(false); + } } -TEST("require that tcp lingering can be adjusted") { +TEST_F(SocketTest, require_that_tcp_lingering_can_be_adjusted) +{ SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); EXPECT_TRUE(!SocketOptions::set_linger(-1, true, 0)); EXPECT_TRUE(handle.set_linger(true, 0)); - TEST_DO(verifier.verify_linger(true, 0)); + { + SCOPED_TRACE("verify linger true 0"); + verifier.verify_linger(true, 0); + } EXPECT_TRUE(handle.set_linger(true, 10)); - TEST_DO(verifier.verify_linger(true, 10)); + { + SCOPED_TRACE("verify linger true 10"); + verifier.verify_linger(true, 10); + } EXPECT_TRUE(handle.set_linger(false, 0)); - TEST_DO(verifier.verify_linger(false, 0)); + { + SCOPED_TRACE("verify linger false 0"); + verifier.verify_linger(false, 0); + } EXPECT_TRUE(handle.set_linger(false, 10)); - TEST_DO(verifier.verify_linger(false, 0)); + { + SCOPED_TRACE("verify linger false 0 (overridden)"); + verifier.verify_linger(false, 0); + } } SocketHandle connect_async(const SocketAddress &addr) { @@ -411,7 +543,10 @@ SocketHandle connect_async(const SocketAddress &addr) { ctx.handle = addr.connect_async(); EXPECT_TRUE(ctx.handle.valid()); test::SocketOptionsVerifier verifier(ctx.handle.get()); - TEST_DO(verifier.verify_blocking(false)); + { + SCOPED_TRACE("verify blocking false"); + verifier.verify_blocking(false); + } if (ctx.handle.valid()) { selector.add(ctx.handle.get(), ctx, true, true); while (!ctx.connect_done) { @@ -420,23 +555,30 @@ SocketHandle connect_async(const SocketAddress &addr) { } selector.remove(ctx.handle.get()); } - EXPECT_EQUAL(ctx.error, 0); + EXPECT_EQ(ctx.error, 0); return std::move(ctx.handle); } -TEST_MT_FF("require that async connect pattern works", 2, ServerSocket("tcp/0"), TimeBomb(60)) { - if (thread_id == 0) { - SocketHandle socket = f1.accept(); - EXPECT_TRUE(socket.valid()); - TEST_DO(verify_socket_io(true, socket)); - } else { - SocketAddress addr = SocketSpec::from_port(f1.address().port()).client_address(); - SocketHandle socket = connect_async(addr); - socket.set_blocking(true); - TEST_DO(verify_socket_io(false, socket)); - // TEST_DO(connect_async(SocketAddress::select_remote(80, "www.yahoo.com"))); - // TEST_DO(connect_async(SocketAddress::select_remote(85, "myinternalhost"))); - } -} - -TEST_MAIN() { TEST_RUN_ALL(); } +TEST_F(SocketTest, require_that_async_connect_pattern_works) +{ + constexpr size_t num_threads = 2; + ServerSocket f1("tcp/0"); + TimeBomb f2(60); + auto task = [&f1](Nexus& ctx) { + if (ctx.thread_id() == 0) { + SocketHandle socket = f1.accept(); + EXPECT_TRUE(socket.valid()); + SCOPED_TRACE("verify socket io true"); + verify_socket_io(true, socket); + } else { + SocketAddress addr = SocketSpec::from_port(f1.address().port()).client_address(); + SocketHandle socket = connect_async(addr); + socket.set_blocking(true); + SCOPED_TRACE("verify socket io false"); + verify_socket_io(false, socket); + } + }; + Nexus::run(num_threads, task); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/CMakeLists.txt b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/CMakeLists.txt index 6bbf0189862..87e13f14875 100644 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/CMakeLists.txt +++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(vespalib_net_tls_auto_reloading_tls_crypto_engine_test_app auto_reloading_tls_crypto_engine_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_net_tls_auto_reloading_tls_crypto_engine_test_app COMMAND vespalib_net_tls_auto_reloading_tls_crypto_engine_test_app) diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp index b6efb66c8bb..ed20dd6bcf4 100644 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp +++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp @@ -1,15 +1,17 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h> #include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/transport_security_options_reading.h> #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/testkit/time_bomb.h> #include <openssl/ssl.h> #include <filesystem> +#include <fstream> using namespace vespalib; using namespace vespalib::net::tls; @@ -89,6 +91,42 @@ void write_file(vespalib::stringref path, vespalib::stringref data) { f.write(data.data(), data.size(), 0); } +class AutoReloadingTlsCryptoEngineTest : public ::testing::Test +{ +protected: + AutoReloadingTlsCryptoEngineTest(); + ~AutoReloadingTlsCryptoEngineTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +AutoReloadingTlsCryptoEngineTest::AutoReloadingTlsCryptoEngineTest() + : testing::Test() +{ +} + +AutoReloadingTlsCryptoEngineTest::~AutoReloadingTlsCryptoEngineTest() = default; + +void +AutoReloadingTlsCryptoEngineTest::SetUpTestSuite() +{ + std::ofstream test_config("test_config.json"); + test_config << "{\n" << + " \"files\":{\n" << + " \"private-key\": \"" + TEST_PATH("test_key.pem") << "\",\n" << + " \"ca-certificates\": \"" + TEST_PATH("test_ca.pem") << "\",\n" << + " \"certificates\": \"test_cert.pem\"\n" << + " }\n" << + "}" << std::endl; + test_config.close(); +} + +void +AutoReloadingTlsCryptoEngineTest::TearDownTestSuite() +{ + std::filesystem::remove("test_config.json"); +} + struct Fixture { std::unique_ptr<AutoReloadingTlsCryptoEngine> engine; explicit Fixture(AutoReloadingTlsCryptoEngine::TimeInterval reload_interval, @@ -114,9 +152,13 @@ struct Fixture { } }; -TEST_FF("Config reloading transitively loads updated files", Fixture(50ms), TimeBomb(60)) { +TEST_F(AutoReloadingTlsCryptoEngineTest, config_reloading_transitively_loads_updated_files) +{ + Fixture f1(50ms); + TimeBomb f2(60); + auto current_certs = f1.current_cert_chain(); - ASSERT_EQUAL(cert1_pem, current_certs); + ASSERT_EQ(cert1_pem, current_certs); write_file("test_cert.pem.tmp", cert2_pem); std::filesystem::rename(std::filesystem::path("test_cert.pem.tmp"), std::filesystem::path("test_cert.pem")); // We expect this to be an atomic rename under the hood @@ -129,15 +171,25 @@ TEST_FF("Config reloading transitively loads updated files", Fixture(50ms), Time // If the config is never reloaded, test will go boom. } -TEST_FF("Shutting down auto-reloading engine immediately stops background thread", Fixture(600s), TimeBomb(60)) { +TEST_F(AutoReloadingTlsCryptoEngineTest, shutting_down_auto_reloading_engine_immediately_stops_background_thread) +{ + Fixture f1(600s); + TimeBomb f2(60); // This passes just from not having the TimeBomb blow up. } -TEST_FF("Authorization mode is propagated to engine", Fixture(50ms, AuthorizationMode::LogOnly), TimeBomb(60)) { - EXPECT_EQUAL(AuthorizationMode::LogOnly, f1.current_authorization_mode()); +TEST_F(AutoReloadingTlsCryptoEngineTest, authorization_mode_is_propagated_to_engine) +{ + Fixture f1(50ms, AuthorizationMode::LogOnly); + TimeBomb f2(60); + EXPECT_EQ(AuthorizationMode::LogOnly, f1.current_authorization_mode()); } -TEST_FF("Config reload failure increments failure statistic", Fixture(50ms), TimeBomb(60)) { +TEST_F(AutoReloadingTlsCryptoEngineTest, config_reload_failure_increments_failure_statistic) +{ + Fixture f1(50ms); + TimeBomb f2(60); + auto before = ConfigStatistics::get().snapshot(); write_file("test_cert.pem.tmp", "Broken file oh no :("); @@ -148,4 +200,4 @@ TEST_FF("Config reload failure increments failure statistic", Fixture(50ms), Tim } } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/test_config.json b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/test_config.json deleted file mode 100644 index 2b2322d928f..00000000000 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/test_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "files":{ - "private-key": "test_key.pem", - "ca-certificates": "test_ca.pem", - "certificates": "test_cert.pem" - } -} diff --git a/vespalib/src/tests/net/tls/transport_options/CMakeLists.txt b/vespalib/src/tests/net/tls/transport_options/CMakeLists.txt index 3623912bb42..2013879569f 100644 --- a/vespalib/src/tests/net/tls/transport_options/CMakeLists.txt +++ b/vespalib/src/tests/net/tls/transport_options/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(vespalib_net_tls_transport_options_test_app TEST transport_options_reading_test.cpp DEPENDS vespalib + GTest::gtest ) vespa_add_test(NAME vespalib_net_tls_transport_options_test_app COMMAND vespalib_net_tls_transport_options_test_app) diff --git a/vespalib/src/tests/net/tls/transport_options/ok_config.json b/vespalib/src/tests/net/tls/transport_options/ok_config.json deleted file mode 100644 index dd2591661dc..00000000000 --- a/vespalib/src/tests/net/tls/transport_options/ok_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "files":{ - "private-key": "dummy_privkey.txt", - "ca-certificates": "dummy_ca_certs.txt", - "certificates": "dummy_certs.txt" - } -} diff --git a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp index ef0fdac0495..ea499607685 100644 --- a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp +++ b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp @@ -1,113 +1,245 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/net/tls/transport_security_options.h> #include <vespa/vespalib/net/tls/transport_security_options_reading.h> #include <vespa/vespalib/test/peer_policy_utils.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/util/exceptions.h> +#include <gmock/gmock.h> +#include <filesystem> +#include <fstream> +#include <optional> +#include <sstream> using namespace vespalib; using namespace vespalib::net::tls; -TEST("can load TLS credentials via config file") { +namespace { + +class ConfigWriter { + std::optional<std::string> _private_key; + std::optional<std::string> _ca_certificates; + std::optional<std::string> _certificates; + std::optional<std::string> _accepted_ciphers; + std::optional<std::string> _authorized_peers; + std::optional<std::string> _disable_hostname_validation; + std::optional<std::string> _flipper_the_dolphin; +public: + ConfigWriter(); + ConfigWriter(ConfigWriter&&); + ~ConfigWriter(); + ConfigWriter private_key(std::optional<std::string> value) && { _private_key = value; return std::move(*this); } + ConfigWriter ca_certificates(std::optional<std::string> value) && { _ca_certificates = value; return std::move(*this); } + ConfigWriter certificates(std::optional<std::string> value) && { _certificates = value; return std::move(*this); } + ConfigWriter accepted_ciphers(std::optional<std::string> value) && { _accepted_ciphers = value; return std::move(*this); } + ConfigWriter authorized_peers(std::optional<std::string> value) && { _authorized_peers = value; return std::move(*this); } + ConfigWriter disable_hostname_validation(std::optional<std::string> value) && { _disable_hostname_validation = value; return std::move(*this); } + ConfigWriter flipper_the_dolphin(std::optional<std::string> value) && { _flipper_the_dolphin = value; return std::move(*this); } + void write(std::ostream& os); + std::string write(); +}; + +ConfigWriter::ConfigWriter() + : _private_key(TEST_PATH("dummy_privkey.txt")), + _ca_certificates(TEST_PATH("dummy_ca_certs.txt")), + _certificates(TEST_PATH("dummy_certs.txt")), + _accepted_ciphers(), + _authorized_peers(), + _disable_hostname_validation(), + _flipper_the_dolphin() +{ +} + +ConfigWriter::ConfigWriter(ConfigWriter&&) = default; + +ConfigWriter::~ConfigWriter() = default; + +void +ConfigWriter::write(std::ostream& os) +{ + os << "{\n" << + R"( "files": {)"; + bool had_files_entry = false; + if (_private_key.has_value()) { + os << "\n" << R"( "private-key": ")" << _private_key.value() << R"(")"; + had_files_entry = true; + } + if (_ca_certificates.has_value()) { + if (had_files_entry) { + os << ","; + } + os << "\n" << R"( "ca-certificates": ")" << _ca_certificates.value() << R"(")"; + had_files_entry = true; + } + if (_certificates.has_value()) { + if (had_files_entry) { + os << ","; + } + os << "\n" << R"( "certificates": ")" << _certificates.value() << R"(")"; + had_files_entry = true; + } + os << "\n }"; + if (_accepted_ciphers.has_value()) { + os << ",\n" << R"( "accepted-ciphers" : )" << _accepted_ciphers.value(); + } + if (_authorized_peers.has_value()) { + os << ",\n" << R"( "authorized-peers": )" << _authorized_peers.value(); + } + if (_disable_hostname_validation.has_value()) { + os << ",\n" << R"( "disable-hostname-validation": )" << _disable_hostname_validation.value(); + } + if (_flipper_the_dolphin.has_value()) { + os << ",\n" << R"( "flipper-the-dolphin": )" << _flipper_the_dolphin.value(); + } + os << "\n}" << std::endl; +} + +std::string +ConfigWriter::write() +{ + std::ostringstream os; + write(os); + return os.str(); +} + +} + +class TransportSecurityOptionsTest : public ::testing::Test { +protected: + TransportSecurityOptionsTest(); + ~TransportSecurityOptionsTest() override; + static void SetUpTestSuite(); + static void TearDownTestSuite(); +}; + +TransportSecurityOptionsTest::TransportSecurityOptionsTest() + : ::testing::Test() +{ +} + +TransportSecurityOptionsTest::~TransportSecurityOptionsTest() = default; + +void +TransportSecurityOptionsTest::SetUpTestSuite() +{ + std::ofstream ok_config("ok_config.json"); + ConfigWriter().write(ok_config); + ok_config.close(); +} + +void +TransportSecurityOptionsTest::TearDownTestSuite() +{ + std::filesystem::remove("ok_config.json"); +} + +TEST_F(TransportSecurityOptionsTest, can_load_tls_credentials_via_config_file) +{ auto opts = read_options_from_json_file("ok_config.json"); ASSERT_TRUE(opts.get() != nullptr); // Obviously we'd need to change this to actual PEM data if config reading started // actually verifying the _content_ of files, not just reading them. - EXPECT_EQUAL("My private key\n", opts->private_key_pem()); - EXPECT_EQUAL("My CA certificates\n", opts->ca_certs_pem()); - EXPECT_EQUAL("My certificate chain\n", opts->cert_chain_pem()); + EXPECT_EQ("My private key\n", opts->private_key_pem()); + EXPECT_EQ("My CA certificates\n", opts->ca_certs_pem()); + EXPECT_EQ("My certificate chain\n", opts->cert_chain_pem()); } -TEST("copying options without private key does, in fact, not include private key") { +TEST_F(TransportSecurityOptionsTest, copying_options_without_private_key_does_in_fact_not_include_private_key) +{ auto opts = read_options_from_json_file("ok_config.json"); auto cloned = opts->copy_without_private_key(); - EXPECT_EQUAL("", cloned.private_key_pem()); - EXPECT_EQUAL("My CA certificates\n", cloned.ca_certs_pem()); - EXPECT_EQUAL("My certificate chain\n", cloned.cert_chain_pem()); + EXPECT_EQ("", cloned.private_key_pem()); + EXPECT_EQ("My CA certificates\n", cloned.ca_certs_pem()); + EXPECT_EQ("My certificate chain\n", cloned.cert_chain_pem()); } -TEST("missing JSON file throws exception") { - EXPECT_EXCEPTION(read_options_from_json_file("missing_config.json"), IllegalArgumentException, - "TLS config file 'missing_config.json' could not be read"); +TEST_F(TransportSecurityOptionsTest, missing_json_file_throws_exception) +{ + EXPECT_THAT([]() { read_options_from_json_file("missing_config.json"); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config file 'missing_config.json' could not be read"))); } -TEST("bad JSON content throws exception") { +TEST_F(TransportSecurityOptionsTest, bad_json_content_throws_exception) +{ const char* bad_json = "hello world :D"; - EXPECT_EXCEPTION(read_options_from_json_string(bad_json), IllegalArgumentException, - "Provided TLS config file is not valid JSON"); + EXPECT_THAT([bad_json]() { read_options_from_json_string(bad_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("Provided TLS config file is not valid JSON"))); } -TEST("missing 'files' field throws exception") { +TEST_F(TransportSecurityOptionsTest, missing_files_field_throws_exception) +{ const char* incomplete_json = R"({})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "TLS config root field 'files' is missing or empty"); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config root field 'files' is missing or empty"))); } -TEST("missing 'private-key' field throws exception") { - const char* incomplete_json = R"({"files":{"certificates":"dummy_certs.txt","ca-certificates":"dummy_ca_certs.txt"}})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "TLS config field 'private-key' has not been set"); +TEST_F(TransportSecurityOptionsTest, missing_private_key_field_throws_exception) +{ + auto incomplete_json = ConfigWriter().private_key(std::nullopt).write(); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config field 'private-key' has not been set"))); } -TEST("missing 'certificates' field throws exception") { - const char* incomplete_json = R"({"files":{"private-key":"dummy_privkey.txt","ca-certificates":"dummy_ca_certs.txt"}})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "TLS config field 'certificates' has not been set"); +TEST_F(TransportSecurityOptionsTest, missing_certificates_field_throws_exception) +{ + auto incomplete_json = ConfigWriter().certificates(std::nullopt).write(); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config field 'certificates' has not been set"))); } -TEST("missing 'ca-certificates' field throws exception") { - const char* incomplete_json = R"({"files":{"private-key":"dummy_privkey.txt","certificates":"dummy_certs.txt"}})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "TLS config field 'ca-certificates' has not been set"); +TEST_F(TransportSecurityOptionsTest, missing_ca_certificates_field_throws_exception) +{ + auto incomplete_json = ConfigWriter().ca_certificates(std::nullopt).write(); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("TLS config field 'ca-certificates' has not been set"))); } -TEST("missing file referenced by field throws exception") { - const char* incomplete_json = R"({"files":{"private-key":"missing_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}})"; - EXPECT_EXCEPTION(read_options_from_json_string(incomplete_json), IllegalArgumentException, - "File 'missing_privkey.txt' referenced by TLS config does not exist"); +TEST_F(TransportSecurityOptionsTest, missing_file_referenced_by_field_throws_exception) +{ + auto incomplete_json = ConfigWriter().private_key("missing_privkey.txt").write(); + EXPECT_THAT([incomplete_json]() { read_options_from_json_string(incomplete_json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("File 'missing_privkey.txt' referenced by TLS config does not exist"))); } vespalib::string json_with_policies(const vespalib::string& policies) { - const char* fmt = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "authorized-peers":[%s]})"; - return vespalib::make_string(fmt, policies.c_str()); + return ConfigWriter().authorized_peers(std::string("[") + policies + "]").write(); } -TransportSecurityOptions parse_policies(const vespalib::string& policies) { +TransportSecurityOptions parse_policies(const vespalib::string& policies) +{ return *read_options_from_json_string(json_with_policies(policies)); } -TEST("config file without authorized-peers accepts all pre-verified certificates") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}})"; +TEST_F(TransportSecurityOptionsTest, config_file_without_authorized_peers_accepts_all_pre_verified_certificates) +{ + auto json = ConfigWriter().write(); EXPECT_TRUE(read_options_from_json_string(json)->authorized_peers().allows_all_authenticated()); } // Instead of contemplating what the semantics of an empty allow list should be, // we do the easy way out and just say it's not allowed in the first place. -TEST("empty policy array throws exception") { - EXPECT_EXCEPTION(parse_policies(""), vespalib::IllegalArgumentException, - "\"authorized-peers\" must either be not present (allows " - "all peers with valid certificates) or a non-empty array"); +TEST_F(TransportSecurityOptionsTest, empty_policy_array_throws_exception) +{ + EXPECT_THAT([]() { parse_policies(""); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("\"authorized-peers\" must either be not present (allows " + "all peers with valid certificates) or a non-empty array"))); } -TEST("can parse single peer policy with single requirement") { +TEST_F(TransportSecurityOptionsTest, can_parse_single_peer_policy_with_single_requirement) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "hello.world"} ] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("hello.world")})}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("hello.world")})}), + parse_policies(json).authorized_peers()); } -TEST("can parse single peer policy with multiple requirements") { +TEST_F(TransportSecurityOptionsTest, can_parse_single_peer_policy_with_multiple_requirements) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "hello.world"}, @@ -115,57 +247,56 @@ TEST("can parse single peer policy with multiple requirements") { {"field": "CN", "must-match": "goodbye.moon"} ] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("hello.world"), - required_san_uri("foo://bar/baz"), - required_cn("goodbye.moon")})}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("hello.world"), + required_san_uri("foo://bar/baz"), + required_cn("goodbye.moon")})}), + parse_policies(json).authorized_peers()); } -TEST("unknown field type throws exception") { +TEST_F(TransportSecurityOptionsTest, unknown_field_type_throws_exception) +{ const char* json = R"({ "required-credentials":[ {"field": "winnie the pooh", "must-match": "piglet"} ] })"; - EXPECT_EXCEPTION(parse_policies(json), vespalib::IllegalArgumentException, - "Unsupported credential field type: 'winnie the pooh'. Supported are: CN, SAN_DNS"); + EXPECT_THAT([json]() { parse_policies(json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("Unsupported credential field type: 'winnie the pooh'. Supported are: CN, SAN_DNS"))); } -TEST("empty required-credentials array throws exception") { +TEST_F(TransportSecurityOptionsTest, empty_required_credentials_array_throws_exception) +{ const char* json = R"({ "required-credentials":[] })"; - EXPECT_EXCEPTION(parse_policies(json), vespalib::IllegalArgumentException, - "\"required-credentials\" array can't be empty (would allow all peers)"); + EXPECT_THAT([json]() { parse_policies(json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("\"required-credentials\" array can't be empty (would allow all peers)"))); } -TEST("accepted cipher list is empty if not specified") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}})"; +TEST_F(TransportSecurityOptionsTest, accepted_cipher_list_is_empty_if_not_specified) +{ + auto json = ConfigWriter().write(); EXPECT_TRUE(read_options_from_json_string(json)->accepted_ciphers().empty()); } -TEST("accepted cipher list is populated if specified") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "accepted-ciphers":["foo", "bar"]})"; +TEST_F(TransportSecurityOptionsTest, accepted_cipher_list_is_populated_if_specified) +{ + auto json = ConfigWriter().accepted_ciphers(R"(["foo", "bar"])").write(); auto ciphers = read_options_from_json_string(json)->accepted_ciphers(); - ASSERT_EQUAL(2u, ciphers.size()); - EXPECT_EQUAL("foo", ciphers[0]); - EXPECT_EQUAL("bar", ciphers[1]); + ASSERT_EQ(2u, ciphers.size()); + EXPECT_EQ("foo", ciphers[0]); + EXPECT_EQ("bar", ciphers[1]); } // FIXME this is temporary until we know enabling it by default won't break the world! -TEST("hostname validation is DISABLED by default when creating options from config file") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}})"; +TEST_F(TransportSecurityOptionsTest, hostname_validation_is_DISABLED_by_default_when_creating_options_from_config_file) +{ + auto json = ConfigWriter().write(); EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation()); } -TEST("TransportSecurityOptions builder does not disable hostname validation by default") { +TEST_F(TransportSecurityOptionsTest, transport_security_options_builder_does_not_disable_hostname_validation_by_default) +{ auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params(). ca_certs_pem("foo"). cert_chain_pem("bar"). @@ -174,67 +305,65 @@ TEST("TransportSecurityOptions builder does not disable hostname validation by d EXPECT_FALSE(ts_opts.disable_hostname_validation()); } -TEST("hostname validation can be explicitly disabled") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "disable-hostname-validation": true})"; +TEST_F(TransportSecurityOptionsTest, hostname_validation_can_be_explicitly_disabled) +{ + auto json = ConfigWriter().disable_hostname_validation("true").write(); EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation()); } -TEST("hostname validation can be explicitly enabled") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "disable-hostname-validation": false})"; +TEST_F(TransportSecurityOptionsTest, hostname_validation_can_be_explicitly_enabled) +{ + auto json = ConfigWriter().disable_hostname_validation("false").write(); EXPECT_FALSE(read_options_from_json_string(json)->disable_hostname_validation()); } -TEST("unknown fields are ignored at parse-time") { - const char* json = R"({"files":{"private-key":"dummy_privkey.txt", - "certificates":"dummy_certs.txt", - "ca-certificates":"dummy_ca_certs.txt"}, - "flipper-the-dolphin": "*weird dolphin noises*"})"; +TEST_F(TransportSecurityOptionsTest, unknown_fields_are_ignored_at_parse_time) +{ + auto json = ConfigWriter().flipper_the_dolphin(R"("*weird dolphin noises*")").write(); EXPECT_TRUE(read_options_from_json_string(json).get() != nullptr); // And no exception thrown. } -TEST("policy without explicit capabilities implicitly get all capabilities") { +TEST_F(TransportSecurityOptionsTest, policy_without_explicit_capabilities_implicitly_get_all_capabilities) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "hello.world"} ] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("hello.world")}, - CapabilitySet::make_with_all_capabilities())}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("hello.world")}, + CapabilitySet::make_with_all_capabilities())}), + parse_policies(json).authorized_peers()); } -TEST("specifying a capability set adds all its underlying capabilities") { +TEST_F(TransportSecurityOptionsTest, specifying_a_capability_set_adds_all_its_underlying_capabilities) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } ], "capabilities": ["vespa.content_node"] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, - CapabilitySet::content_node())}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, + CapabilitySet::content_node())}), + parse_policies(json).authorized_peers()); } -TEST("can specify single leaf capabilities") { +TEST_F(TransportSecurityOptionsTest, can_specify_single_leaf_capabilities) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } ], "capabilities": ["vespa.content.metrics_api", "vespa.slobrok.api"] })"; - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, - CapabilitySet::of({Capability::content_metrics_api(), - Capability::slobrok_api()}))}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, + CapabilitySet::of({Capability::content_metrics_api(), + Capability::slobrok_api()}))}), + parse_policies(json).authorized_peers()); } -TEST("specifying multiple capability sets adds union of underlying capabilities") { +TEST_F(TransportSecurityOptionsTest, specifying_multiple_capability_sets_adds_union_of_underlying_capabilities) +{ const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } @@ -244,23 +373,22 @@ TEST("specifying multiple capability sets adds union of underlying capabilities" CapabilitySet caps; caps.add_all(CapabilitySet::content_node()); caps.add_all(CapabilitySet::container_node()); - EXPECT_EQUAL(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, caps)}), - parse_policies(json).authorized_peers()); + EXPECT_EQ(authorized_peers({policy_with({required_san_dns("*.cool-content-clusters.example")}, caps)}), + parse_policies(json).authorized_peers()); } -TEST("empty capabilities array is not allowed") { +TEST_F(TransportSecurityOptionsTest, empty_capabilities_array_is_not_allowed) { const char* json = R"({ "required-credentials":[ {"field": "SAN_DNS", "must-match": "*.cool-content-clusters.example" } ], "capabilities": [] })"; - EXPECT_EXCEPTION(parse_policies(json), vespalib::IllegalArgumentException, - "\"capabilities\" array must either be not present (implies " - "all capabilities) or contain at least one capability name"); + EXPECT_THAT([json]() { parse_policies(json); }, + testing::ThrowsMessage<IllegalArgumentException>(testing::HasSubstr("\"capabilities\" array must either be not present (implies " + "all capabilities) or contain at least one capability name"))); } // TODO test parsing of multiple policies -TEST_MAIN() { TEST_RUN_ALL(); } - +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/objects/objectdump/objectdump.cpp b/vespalib/src/tests/objects/objectdump/objectdump.cpp index a54f14a0185..f2f4920c37b 100644 --- a/vespalib/src/tests/objects/objectdump/objectdump.cpp +++ b/vespalib/src/tests/objects/objectdump/objectdump.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/objects/visit.hpp> @@ -103,13 +103,9 @@ Foo::visitMembers(ObjectVisitor &v) const { IMPLEMENT_IDENTIFIABLE(Foo, Base); -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("objectdump_test"); +TEST("objectdump_test") { Foo foo; fprintf(stderr, "%s", foo.asString().c_str()); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/objects/objectselection/objectselection.cpp b/vespalib/src/tests/objects/objectselection/objectselection.cpp index e1de71a7329..f6706463ae5 100644 --- a/vespalib/src/tests/objects/objectselection/objectselection.cpp +++ b/vespalib/src/tests/objects/objectselection/objectselection.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/objects/identifiable.hpp> #include <vespa/vespalib/objects/objectpredicate.h> #include <vespa/vespalib/objects/objectoperation.h> @@ -56,12 +56,7 @@ struct ObjectCollect : public ObjectOperation ObjectCollect::~ObjectCollect() = default; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("objectselection_test"); +TEST("objectselection_test") { { Foo f1; Foo f2; @@ -90,5 +85,6 @@ Test::Main() ASSERT_TRUE(((Bar*)operation.nodes[2])->value == 3); ASSERT_TRUE(((Bar*)operation.nodes[3])->value == 4); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp index 212ea417524..62b7dc5f179 100644 --- a/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp +++ b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/polymorphicarrays.h> +#include <cassert> using namespace vespalib; diff --git a/vespalib/src/tests/priority_queue/priority_queue_test.cpp b/vespalib/src/tests/priority_queue/priority_queue_test.cpp index ae85dcfa47a..0f422cacb6d 100644 --- a/vespalib/src/tests/priority_queue/priority_queue_test.cpp +++ b/vespalib/src/tests/priority_queue/priority_queue_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("priority_queue_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/priority_queue.h> using namespace vespalib; diff --git a/vespalib/src/tests/rendezvous/rendezvous_test.cpp b/vespalib/src/tests/rendezvous/rendezvous_test.cpp index d2e2ac2fbab..13c4f968db1 100644 --- a/vespalib/src/tests/rendezvous/rendezvous_test.cpp +++ b/vespalib/src/tests/rendezvous/rendezvous_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/rendezvous.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/count_down_latch.h> #include <utility> #include <thread> diff --git a/vespalib/src/tests/rusage/rusage_test.cpp b/vespalib/src/tests/rusage/rusage_test.cpp index 5c08c99de43..28d3db72099 100644 --- a/vespalib/src/tests/rusage/rusage_test.cpp +++ b/vespalib/src/tests/rusage/rusage_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/rusage.h> using namespace vespalib; diff --git a/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp index 66f155f679b..bcd8ddb24f5 100644 --- a/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/adaptive_sequenced_executor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> #include <condition_variable> diff --git a/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp index eb71709ae43..d899b8b4a2c 100644 --- a/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/foregroundtaskexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <condition_variable> #include <unistd.h> diff --git a/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 96cc23ef70e..474df9bdf3b 100644 --- a/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -4,7 +4,7 @@ #include <vespa/vespalib/util/adaptive_sequenced_executor.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/singleexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> #include <condition_variable> diff --git a/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp b/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp index 0f1c6d3a083..18613c0bc79 100644 --- a/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp +++ b/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/barrier.h> #include <thread> +#include <cassert> using vespalib::steady_clock; diff --git a/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp b/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp index 3e0935e85e0..6376386096e 100644 --- a/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp +++ b/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/shutdownguard.h> #include <vespa/vespalib/util/malloc_mmap_guard.h> #include <thread> diff --git a/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp index 3ae0b709e31..224a4a95afe 100644 --- a/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp +++ b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/singleexecutor.h> #include <vespa/vespalib/util/lambdatask.h> diff --git a/vespalib/src/tests/slime/json_slime_benchmark.cpp b/vespalib/src/tests/slime/json_slime_benchmark.cpp index 78000a5a25d..df9492ba46c 100644 --- a/vespalib/src/tests/slime/json_slime_benchmark.cpp +++ b/vespalib/src/tests/slime/json_slime_benchmark.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <iostream> #include <fstream> +#include <cassert> using namespace vespalib::slime::convenience; diff --git a/vespalib/src/tests/slime/slime_test.cpp b/vespalib/src/tests/slime/slime_test.cpp index 591b3bd6465..d2d1e368715 100644 --- a/vespalib/src/tests/slime/slime_test.cpp +++ b/vespalib/src/tests/slime/slime_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/object_value.h> #include <vespa/vespalib/data/slime/array_value.h> diff --git a/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp b/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp index 87ecb42efa8..065984e5ed9 100644 --- a/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp +++ b/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> +#include <cassert> using namespace vespalib; using namespace vespalib::slime::convenience; diff --git a/vespalib/src/tests/stllike/cache_test.cpp b/vespalib/src/tests/stllike/cache_test.cpp index 9171f923ecf..79f8515162d 100644 --- a/vespalib/src/tests/stllike/cache_test.cpp +++ b/vespalib/src/tests/stllike/cache_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/cache.hpp> #include <map> diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index 493ea700fb2..592fc72edeb 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_map_equal.hpp> diff --git a/vespalib/src/tests/stllike/hashtable_test.cpp b/vespalib/src/tests/stllike/hashtable_test.cpp index c7890c79164..4880c7a872d 100644 --- a/vespalib/src/tests/stllike/hashtable_test.cpp +++ b/vespalib/src/tests/stllike/hashtable_test.cpp @@ -4,7 +4,7 @@ #include <vespa/vespalib/stllike/hashtable.hpp> #include <vespa/vespalib/stllike/hash_fun.h> #include <vespa/vespalib/stllike/identity.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <memory> #include <vector> diff --git a/vespalib/src/tests/stllike/lrucache.cpp b/vespalib/src/tests/stllike/lrucache.cpp index 3722845919d..9f81b29394e 100644 --- a/vespalib/src/tests/stllike/lrucache.cpp +++ b/vespalib/src/tests/stllike/lrucache.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/lrucache_map.hpp> diff --git a/vespalib/src/tests/stllike/vector_map_test.cpp b/vespalib/src/tests/stllike/vector_map_test.cpp index 00794bcc1e3..18b573f323b 100644 --- a/vespalib/src/tests/stllike/vector_map_test.cpp +++ b/vespalib/src/tests/stllike/vector_map_test.cpp @@ -5,7 +5,7 @@ LOG_SETUP("vector_map_test"); #include <vespa/vespalib/stllike/vector_map.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using vespalib::vector_map; diff --git a/vespalib/src/tests/sync/sync_test.cpp b/vespalib/src/tests/sync/sync_test.cpp index df781dea423..dd15e2f0057 100644 --- a/vespalib/src/tests/sync/sync_test.cpp +++ b/vespalib/src/tests/sync/sync_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/gate.h> using namespace vespalib; diff --git a/vespalib/src/tests/text/lowercase/lowercase_test.cpp b/vespalib/src/tests/text/lowercase/lowercase_test.cpp index 40803b8f295..75512a733a5 100644 --- a/vespalib/src/tests/text/lowercase/lowercase_test.cpp +++ b/vespalib/src/tests/text/lowercase/lowercase_test.cpp @@ -2,7 +2,6 @@ #include <vespa/log/log.h> LOG_SETUP("lowercase_test"); #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/text/lowercase.h> #include <iostream> #include <fstream> diff --git a/vespalib/src/tests/text/stringtokenizer/stringtokenizer_test.cpp b/vespalib/src/tests/text/stringtokenizer/stringtokenizer_test.cpp index aabe9ba1980..75415f7093b 100644 --- a/vespalib/src/tests/text/stringtokenizer/stringtokenizer_test.cpp +++ b/vespalib/src/tests/text/stringtokenizer/stringtokenizer_test.cpp @@ -1,18 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("stringtokenizer_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <set> using namespace vespalib; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("stringtokenizer_test"); +TEST("stringtokenizer_test") { { string s("This,is ,a,,list ,\tof,,sepa rated\n, \rtokens,"); StringTokenizer tokenizer(s); @@ -67,5 +62,6 @@ Test::Main() StringTokenizer tokenizer(s); EXPECT_EQUAL(0u, tokenizer.size()); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/text/utf8/utf8_test.cpp b/vespalib/src/tests/text/utf8/utf8_test.cpp index 7f48537f179..ffe1e44c079 100644 --- a/vespalib/src/tests/text/utf8/utf8_test.cpp +++ b/vespalib/src/tests/text/utf8/utf8_test.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/text/utf8.h> #include <fcntl.h> #include <unistd.h> @@ -15,12 +14,7 @@ LOG_SETUP("utf8_test"); using namespace vespalib; -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("utf8_test"); +TEST("utf8_test") { for (uint32_t h = 0; h < 0x1100; h++) { vespalib::string data; @@ -80,5 +74,6 @@ Test::Main() } EXPECT_TRUE(! r.hasMore()); } - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/trace/trace.cpp b/vespalib/src/tests/trace/trace.cpp index 2b35740deb1..5dce0e5cc6b 100644 --- a/vespalib/src/tests/trace/trace.cpp +++ b/vespalib/src/tests/trace/trace.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/trace/trace.h> #include <vespa/vespalib/trace/tracevisitor.h> diff --git a/vespalib/src/tests/true/true.cpp b/vespalib/src/tests/true/true.cpp index fee248200ad..8d2ed2aeca8 100644 --- a/vespalib/src/tests/true/true.cpp +++ b/vespalib/src/tests/true/true.cpp @@ -1,14 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("true_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> -TEST_SETUP(Test) - -int -Test::Main() -{ - TEST_INIT("true_test"); +TEST("true_test") { EXPECT_TRUE(true); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp index 77c109bb6eb..38e8467ff35 100644 --- a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/testkit/test_path.h> #include <vespa/vespalib/util/cgroup_resource_limits.h> #include <vespa/vespalib/util/size_literals.h> @@ -20,7 +21,8 @@ CGroupResourceLimitsTest::~CGroupResourceLimitsTest() = default; void CGroupResourceLimitsTest::check_limits(const std::string &base, const std::optional<uint64_t>& memory_limit, const std::optional<uint32_t>& cpu_limit) { - CGroupResourceLimits cg_limits(base + "/cgroup", base + "/self"); + auto src_base = TEST_PATH(base); + CGroupResourceLimits cg_limits(src_base + "/cgroup", src_base + "/self"); EXPECT_EQ(memory_limit, cg_limits.get_memory_limit()); EXPECT_EQ(cpu_limit, cg_limits.get_cpu_limit()); } diff --git a/vespalib/src/tests/valgrind/valgrind_test.cpp b/vespalib/src/tests/valgrind/valgrind_test.cpp index e9fd265cfa3..afecff17eb9 100644 --- a/vespalib/src/tests/valgrind/valgrind_test.cpp +++ b/vespalib/src/tests/valgrind/valgrind_test.cpp @@ -1,20 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/valgrind.h> using namespace vespalib; -class Test : public TestApp -{ - int Main() override; - void testUninitializedUser(); - void testUninitializedSystemCall(); - void testInitializedUser(); - void testInitializedSystemCall(); -}; - -void Test::testUninitializedUser() +void testUninitializedUser() { char buf[7]; buf[0] = 7; @@ -22,45 +13,37 @@ void Test::testUninitializedUser() Valgrind::testUninitialized(buf, sizeof(buf)); } -void Test::testUninitializedSystemCall() +void testUninitializedSystemCall() { char buf[7]; buf[0] = 7; buf[5] = 7; Valgrind::testSystemCall(buf, sizeof(buf)); } -void Test::testInitializedUser() +void testInitializedUser() { char buf[7]; memset(buf, 0, sizeof(buf)); Valgrind::testUninitialized(buf, sizeof(buf)); } -void Test::testInitializedSystemCall() +void testInitializedSystemCall() { char buf[7]; memset(buf, 0, sizeof(buf)); Valgrind::testSystemCall(buf, sizeof(buf)); } -int -Test::Main() -{ - TEST_INIT("valgrind_test"); - - if (strcmp(_argv[1], "testInitializedUser") == 0) { +TEST_MAIN() { + if (strcmp(argv[1], "testInitializedUser") == 0) { testInitializedUser(); - } else if (strcmp(_argv[1], "testInitializedSystemCall") == 0) { + } else if (strcmp(argv[1], "testInitializedSystemCall") == 0) { testInitializedSystemCall(); - } else if (strcmp(_argv[1], "testUninitializedUser") == 0) { + } else if (strcmp(argv[1], "testUninitializedUser") == 0) { testUninitializedUser(); - } else if (strcmp(_argv[1], "testUninitializedSystemCall") == 0) { + } else if (strcmp(argv[1], "testUninitializedSystemCall") == 0) { testUninitializedSystemCall(); } else { testInitializedUser(); } - - TEST_DONE(); } - -TEST_APPHOOK(Test) diff --git a/vespalib/src/tests/zcurve/zcurve_test.cpp b/vespalib/src/tests/zcurve/zcurve_test.cpp index 8ed92304783..453bf6ddeba 100644 --- a/vespalib/src/tests/zcurve/zcurve_test.cpp +++ b/vespalib/src/tests/zcurve/zcurve_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/geo/zcurve.h> #include <algorithm> #include <limits> @@ -10,45 +10,14 @@ #include <vespa/log/log.h> LOG_SETUP("zcurve_test"); -namespace vespalib { +using namespace vespalib; using geo::ZCurve; -class ZCurveTest : public vespalib::TestApp -{ -public: - ZCurveTest(void) - : vespalib::TestApp() - { - } - - void testEncoding(); - - void testDecoding(); - - double ftime(); - - static inline int64_t encodexy3(int32_t x, int32_t y); - #define BMLIMIT 0x1000000 - template <bool decode> - int64_t bm(); - - template <bool decode> - int64_t bm2(); - - template <bool decode> - int64_t bm3(); - - int64_t bmcheck(); - - int Main() override; -}; - - void -ZCurveTest::testEncoding(void) +testEncoding(void) { int32_t x = 0; int32_t y = 0; @@ -83,7 +52,7 @@ ZCurveTest::testEncoding(void) void -ZCurveTest::testDecoding(void) +testDecoding(void) { int32_t x = 0; int32_t y = 0; @@ -134,7 +103,7 @@ ZCurveTest::testDecoding(void) double -ZCurveTest::ftime() +ftime() { struct timeval tv; gettimeofday(&tv, NULL); @@ -142,7 +111,7 @@ ZCurveTest::ftime() } int64_t -ZCurveTest::encodexy3(int32_t x, int32_t y) +encodexy3(int32_t x, int32_t y) { uint32_t resxl; uint32_t resxh; @@ -177,7 +146,7 @@ ZCurveTest::encodexy3(int32_t x, int32_t y) template <bool decode> int64_t -ZCurveTest::bm() +bm() { int64_t res = 0; double before = ftime(); @@ -218,7 +187,7 @@ ZCurveTest::bm() template <bool decode> int64_t -ZCurveTest::bm2(void) +bm2(void) { int64_t res = 0; double before = ftime(); @@ -259,7 +228,7 @@ ZCurveTest::bm2(void) template <bool decode> int64_t -ZCurveTest::bm3() +bm3() { int64_t res = 0; double before = ftime(); @@ -299,7 +268,7 @@ ZCurveTest::bm3() int64_t -ZCurveTest::bmcheck() +bmcheck() { int64_t res = 0; double before = ftime(); @@ -340,12 +309,7 @@ ZCurveTest::bmcheck() return res; } - -int -ZCurveTest::Main() -{ - TEST_INIT("zcurve_test"); - +TEST_MAIN() { for (int32_t x = 0; x < 4; x++) { for (int32_t y = 0; y < 4; y++) { int64_t enc = 0; @@ -368,7 +332,7 @@ ZCurveTest::Main() } testEncoding(); testDecoding(); - if (_argc >= 2) { + if (argc >= 2) { int64_t enc1 = bm<true>(); int64_t enc1b = bm<false>(); int64_t enc2 = bm2<true>(); @@ -383,10 +347,4 @@ ZCurveTest::Main() ASSERT_TRUE(enc1 == enc3b); ASSERT_TRUE(enc1 == enc4); } - - TEST_DONE(); } - -} - -TEST_APPHOOK(vespalib::ZCurveTest); diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 3206baa703b..ac5459c10e9 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -11,7 +11,6 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_encoding> $<TARGET_OBJECTS:vespalib_vespalib_fuzzy> $<TARGET_OBJECTS:vespalib_vespalib_geo> - $<TARGET_OBJECTS:vespalib_vespalib_hwaccelrated> $<TARGET_OBJECTS:vespalib_vespalib_io> $<TARGET_OBJECTS:vespalib_vespalib_locale> $<TARGET_OBJECTS:vespalib_vespalib_metrics> @@ -38,8 +37,6 @@ vespa_add_library(vespalib ${VESPA_GCC_LIB} ) -set(BLA_VENDOR OpenBLAS) -vespa_add_target_package_dependency(vespalib BLAS) vespa_add_target_package_dependency(vespalib OpenSSL) vespa_add_target_package_dependency(vespalib RE2) diff --git a/vespalib/src/vespa/vespalib/data/slime/type.h b/vespalib/src/vespa/vespalib/data/slime/type.h index d3d15526f12..90e79acccd0 100644 --- a/vespalib/src/vespa/vespalib/data/slime/type.h +++ b/vespalib/src/vespa/vespalib/data/slime/type.h @@ -19,6 +19,7 @@ protected: public: uint32_t getId() const noexcept { return _id; } + bool operator==(const Type &rhs) const noexcept { return _id == rhs._id; } }; /** diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt b/vespalib/src/vespa/vespalib/hwaccelerated/CMakeLists.txt index 33545fdb2fe..99e442446b6 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/hwaccelerated/CMakeLists.txt @@ -6,12 +6,15 @@ else() unset(ACCEL_FILES) endif() -vespa_add_library(vespalib_vespalib_hwaccelrated OBJECT +vespa_add_library(vespa_hwaccelerated SOURCES - iaccelrated.cpp + iaccelerated.cpp generic.cpp ${ACCEL_FILES} DEPENDS + INSTALL lib64 ) set_source_files_properties(avx2.cpp PROPERTIES COMPILE_FLAGS "-O3 -march=haswell") set_source_files_properties(avx512.cpp PROPERTIES COMPILE_FLAGS "-O3 -march=skylake-avx512 -mprefer-vector-width=512") +set(BLA_VENDOR OpenBLAS) +vespa_add_target_package_dependency(vespa_hwaccelerated BLAS) diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp b/vespalib/src/vespa/vespalib/hwaccelerated/avx2.cpp index 3b63a904b2f..9dbc8d92034 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelerated/avx2.cpp @@ -3,7 +3,7 @@ #include "avx2.h" #include "avxprivate.hpp" -namespace vespalib::hwaccelrated { +namespace vespalib::hwaccelerated { size_t Avx2Accelrator::populationCount(const uint64_t *a, size_t sz) const noexcept { diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h b/vespalib/src/vespa/vespalib/hwaccelerated/avx2.h index 279483fcec3..b970e1529e8 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h +++ b/vespalib/src/vespa/vespalib/hwaccelerated/avx2.h @@ -4,7 +4,7 @@ #include "generic.h" -namespace vespalib::hwaccelrated { +namespace vespalib::hwaccelerated { /** * Avx-512 implementation. diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp b/vespalib/src/vespa/vespalib/hwaccelerated/avx512.cpp index aec81c718c7..dd2924856c0 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelerated/avx512.cpp @@ -3,7 +3,7 @@ #include "avx512.h" #include "avxprivate.hpp" -namespace vespalib:: hwaccelrated { +namespace vespalib:: hwaccelerated { float Avx512Accelrator::dotProduct(const float * af, const float * bf, size_t sz) const noexcept { diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h b/vespalib/src/vespa/vespalib/hwaccelerated/avx512.h index 49d2dc63f77..f226b7f2225 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h +++ b/vespalib/src/vespa/vespalib/hwaccelerated/avx512.h @@ -4,7 +4,7 @@ #include "avx2.h" -namespace vespalib::hwaccelrated { +namespace vespalib::hwaccelerated { /** * Avx-512 implementation. diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelerated/avxprivate.hpp index 602faeb4dc0..7e5bb012099 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelerated/avxprivate.hpp @@ -4,7 +4,7 @@ #include "private_helpers.hpp" -namespace vespalib::hwaccelrated::avx { +namespace vespalib::hwaccelerated::avx { namespace { diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp b/vespalib/src/vespa/vespalib/hwaccelerated/generic.cpp index 4ac99d90a7e..bef2cb947a6 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelerated/generic.cpp @@ -4,7 +4,7 @@ #include "private_helpers.hpp" #include <cblas.h> -namespace vespalib::hwaccelrated { +namespace vespalib::hwaccelerated { namespace { diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h b/vespalib/src/vespa/vespalib/hwaccelerated/generic.h index fee1fec6165..5a7a543aff7 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h +++ b/vespalib/src/vespa/vespalib/hwaccelerated/generic.h @@ -2,14 +2,14 @@ #pragma once -#include "iaccelrated.h" +#include "iaccelerated.h" -namespace vespalib::hwaccelrated { +namespace vespalib::hwaccelerated { /** * Generic cpu agnostic implementation. */ -class GenericAccelrator : public IAccelrated +class GenericAccelrator : public IAccelerated { public: float dotProduct(const float * a, const float * b, size_t sz) const noexcept override; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp b/vespalib/src/vespa/vespalib/hwaccelerated/iaccelerated.cpp index a02e9545765..10b4661894e 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelerated/iaccelerated.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "iaccelrated.h" +#include "iaccelerated.h" #include "generic.h" #ifdef __x86_64__ #include "avx2.h" @@ -11,13 +11,13 @@ #include <vector> #include <vespa/log/log.h> -LOG_SETUP(".vespalib.hwaccelrated"); +LOG_SETUP(".vespalib.hwaccelerated"); -namespace vespalib::hwaccelrated { +namespace vespalib::hwaccelerated { namespace { -IAccelrated::UP create_accelerator() { +IAccelerated::UP create_accelerator() { #ifdef __x86_64__ __builtin_cpu_init(); if (__builtin_cpu_supports("avx512f")) { @@ -41,7 +41,7 @@ std::vector<T> createAndFill(size_t sz) { template<typename T> void -verifyDotproduct(const IAccelrated & accel) +verifyDotproduct(const IAccelerated & accel) { const size_t testLength(255); srand(1); @@ -62,7 +62,7 @@ verifyDotproduct(const IAccelrated & accel) template<typename T> void -verifyEuclideanDistance(const IAccelrated & accel) { +verifyEuclideanDistance(const IAccelerated & accel) { const size_t testLength(255); srand(1); std::vector<T> a = createAndFill<T>(testLength); @@ -81,7 +81,7 @@ verifyEuclideanDistance(const IAccelrated & accel) { } void -verifyPopulationCount(const IAccelrated & accel) +verifyPopulationCount(const IAccelerated & accel) { const uint64_t words[7] = {0x123456789abcdef0L, // 32 0x0000000000000000L, // 0 @@ -124,15 +124,15 @@ std::vector<uint64_t> simpleInvert(const std::vector<uint64_t> & src) { std::vector<uint64_t> inverted; inverted.reserve(src.size()); - for (size_t i(0); i < src.size(); i++) { - inverted.push_back(~src[i]); + for (unsigned long i : src) { + inverted.push_back(~i); } return inverted; } std::vector<uint64_t> optionallyInvert(bool invert, std::vector<uint64_t> v) { - return invert ? simpleInvert(std::move(v)) : std::move(v); + return invert ? simpleInvert(v) : std::move(v); } bool shouldInvert(bool invertSome) { @@ -140,7 +140,7 @@ bool shouldInvert(bool invertSome) { } void -verifyOr64(const IAccelrated & accel, const std::vector<std::vector<uint64_t>> & vectors, +verifyOr64(const IAccelerated & accel, const std::vector<std::vector<uint64_t>> & vectors, size_t offset, size_t num_vectors, bool invertSome) { std::vector<std::pair<const void *, bool>> vRefs; @@ -162,7 +162,7 @@ verifyOr64(const IAccelrated & accel, const std::vector<std::vector<uint64_t>> & } void -verifyAnd64(const IAccelrated & accel, const std::vector<std::vector<uint64_t>> & vectors, +verifyAnd64(const IAccelerated & accel, const std::vector<std::vector<uint64_t>> & vectors, size_t offset, size_t num_vectors, bool invertSome) { std::vector<std::pair<const void *, bool>> vRefs; @@ -183,7 +183,7 @@ verifyAnd64(const IAccelrated & accel, const std::vector<std::vector<uint64_t>> } void -verifyOr64(const IAccelrated & accel) { +verifyOr64(const IAccelerated & accel) { std::vector<std::vector<uint64_t>> vectors(3) ; for (auto & v : vectors) { fill(v, 32); @@ -197,7 +197,7 @@ verifyOr64(const IAccelrated & accel) { } void -verifyAnd64(const IAccelrated & accel) { +verifyAnd64(const IAccelerated & accel) { std::vector<std::vector<uint64_t>> vectors(3); for (auto & v : vectors) { fill(v, 32); @@ -215,36 +215,32 @@ class RuntimeVerificator public: RuntimeVerificator(); private: - void verify(const IAccelrated & accelrated) { - verifyDotproduct<float>(accelrated); - verifyDotproduct<double>(accelrated); - verifyDotproduct<int32_t>(accelrated); - verifyDotproduct<int64_t>(accelrated); - verifyEuclideanDistance<float>(accelrated); - verifyEuclideanDistance<double>(accelrated); - verifyPopulationCount(accelrated); - verifyAnd64(accelrated); - verifyOr64(accelrated); + static void verify(const IAccelerated & accelerated) { + verifyDotproduct<float>(accelerated); + verifyDotproduct<double>(accelerated); + verifyDotproduct<int32_t>(accelerated); + verifyDotproduct<int64_t>(accelerated); + verifyEuclideanDistance<float>(accelerated); + verifyEuclideanDistance<double>(accelerated); + verifyPopulationCount(accelerated); + verifyAnd64(accelerated); + verifyOr64(accelerated); } }; RuntimeVerificator::RuntimeVerificator() { - GenericAccelrator generic; - verify(generic); - - const IAccelrated & thisCpu(IAccelrated::getAccelerator()); - verify(thisCpu); + verify(GenericAccelrator()); + verify(*create_accelerator()); } } -RuntimeVerificator _G_verifyAccelrator; - -const IAccelrated & -IAccelrated::getAccelerator() +const IAccelerated & +IAccelerated::getAccelerator() { - static IAccelrated::UP accelrator = create_accelerator(); + static RuntimeVerificator verifyAccelrator_once; + static IAccelerated::UP accelrator = create_accelerator(); return *accelrator; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h b/vespalib/src/vespa/vespalib/hwaccelerated/iaccelerated.h index 337dc3b4ab1..400c73772b2 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h +++ b/vespalib/src/vespa/vespalib/hwaccelerated/iaccelerated.h @@ -6,17 +6,17 @@ #include <cstdint> #include <vector> -namespace vespalib::hwaccelrated { +namespace vespalib::hwaccelerated { /** * This contains an interface to all primitives that has different cpu supported accelrations. * The actual implementation you get by calling the the static getAccelrator method. */ -class IAccelrated +class IAccelerated { public: - virtual ~IAccelrated() = default; - using UP = std::unique_ptr<IAccelrated>; + virtual ~IAccelerated() = default; + using UP = std::unique_ptr<IAccelerated>; virtual float dotProduct(const float * a, const float * b, size_t sz) const noexcept = 0; virtual double dotProduct(const double * a, const double * b, size_t sz) const noexcept = 0; virtual int64_t dotProduct(const int8_t * a, const int8_t * b, size_t sz) const noexcept = 0; @@ -37,7 +37,7 @@ public: // OR 128 bytes from multiple, optionally inverted sources virtual void or128(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const noexcept = 0; - static const IAccelrated & getAccelerator() __attribute__((noinline)); + static const IAccelerated & getAccelerator() __attribute__((noinline)); }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp b/vespalib/src/vespa/vespalib/hwaccelerated/private_helpers.hpp index 236d2a135a5..245916e08d3 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelerated/private_helpers.hpp @@ -6,7 +6,7 @@ #include <vespa/vespalib/util/optimized.h> #include <cstring> -namespace vespalib::hwaccelrated::helper { +namespace vespalib::hwaccelerated::helper { namespace { inline size_t diff --git a/vespalib/src/vespa/vespalib/test/CMakeLists.txt b/vespalib/src/vespa/vespalib/test/CMakeLists.txt index 4a43f9a92dd..b3b872f14c7 100644 --- a/vespalib/src/vespa/vespalib/test/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/test/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(vespalib_vespalib_test OBJECT memory_allocator_observer.cpp nexus.cpp peer_policy_utils.cpp + test_data_base.cpp thread_meets.cpp time_tracer.cpp DEPENDS diff --git a/vespalib/src/vespa/vespalib/test/socket_options_verifier.h b/vespalib/src/vespa/vespalib/test/socket_options_verifier.h index 04b4dea414e..3831c03d629 100644 --- a/vespalib/src/vespa/vespalib/test/socket_options_verifier.h +++ b/vespalib/src/vespa/vespalib/test/socket_options_verifier.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <fcntl.h> #include <unistd.h> #include <netinet/tcp.h> @@ -16,9 +16,9 @@ namespace { void verify_bool_opt(int fd, int level, int name, bool expect) { int data = 0; socklen_t size = sizeof(data); - EXPECT_EQUAL(getsockopt(fd, level, name, &data, &size), 0); - EXPECT_EQUAL(size, sizeof(data)); - EXPECT_EQUAL(data != 0, expect); + EXPECT_EQ(getsockopt(fd, level, name, &data, &size), 0); + EXPECT_EQ(size, sizeof(data)); + EXPECT_EQ(data != 0, expect); } } // namespace vespalib::test::<unnamed> @@ -31,31 +31,35 @@ struct SocketOptionsVerifier { SocketOptionsVerifier(int fd_in) : fd(fd_in) {} void verify_blocking(bool value) { int flags = fcntl(fd, F_GETFL, NULL); - EXPECT_NOT_EQUAL(flags, -1); - EXPECT_EQUAL(((flags & O_NONBLOCK) == 0), value); + EXPECT_NE(flags, -1); + EXPECT_EQ(((flags & O_NONBLOCK) == 0), value); } void verify_nodelay(bool value) { - TEST_DO(verify_bool_opt(fd, IPPROTO_TCP, TCP_NODELAY, value)); + SCOPED_TRACE("verify nodelay"); + verify_bool_opt(fd, IPPROTO_TCP, TCP_NODELAY, value); } void verify_reuse_addr(bool value) { - TEST_DO(verify_bool_opt(fd, SOL_SOCKET, SO_REUSEADDR, value)); + SCOPED_TRACE("verify reuse addr"); + verify_bool_opt(fd, SOL_SOCKET, SO_REUSEADDR, value); } void verify_ipv6_only(bool value) { - TEST_DO(verify_bool_opt(fd, IPPROTO_IPV6, IPV6_V6ONLY, value)); + SCOPED_TRACE("verify ipv6 only"); + verify_bool_opt(fd, IPPROTO_IPV6, IPV6_V6ONLY, value); } void verify_keepalive(bool value) { - TEST_DO(verify_bool_opt(fd, SOL_SOCKET, SO_KEEPALIVE, value)); + SCOPED_TRACE("verify keepalive"); + verify_bool_opt(fd, SOL_SOCKET, SO_KEEPALIVE, value); } void verify_linger(bool enable, int value) { struct linger data; socklen_t size = sizeof(data); memset(&data, 0, sizeof(data)); - EXPECT_EQUAL(getsockopt(fd, SOL_SOCKET, SO_LINGER, &data, &size), 0); - EXPECT_EQUAL(size, sizeof(data)); - EXPECT_EQUAL(enable, data.l_onoff); + EXPECT_EQ(getsockopt(fd, SOL_SOCKET, SO_LINGER, &data, &size), 0); + EXPECT_EQ(size, sizeof(data)); + EXPECT_EQ(enable, data.l_onoff); if (enable) { - EXPECT_EQUAL(value, data.l_linger); + EXPECT_EQ(value, data.l_linger); } } }; diff --git a/vespalib/src/vespa/vespalib/test/test_data.h b/vespalib/src/vespa/vespalib/test/test_data.h new file mode 100644 index 00000000000..1a5c24616b0 --- /dev/null +++ b/vespalib/src/vespa/vespalib/test/test_data.h @@ -0,0 +1,60 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "test_data_base.h" +#include <vespa/vespalib/gtest/gtest.h> +#include <filesystem> + +namespace vespalib::test { + +// Utility class for accessing test data used by unit tests. +template <class Derived> +class TestData : public TestDataBase { +protected: + static std::string _source_testdata; + static std::string _build_testdata; +public: + static void setup_test_data(const std::string& source_testdata_in, const std::string& build_testdata_in); + static void tear_down_test_data(); + static const std::string& source_testdata() noexcept { return _source_testdata; } + static const std::string& build_testdata() noexcept { return _build_testdata; } + void remove_unchanged_build_testdata_file_or_fail(const nbostream& buf, const std::string& file_name); +}; + +template <class Derived> +std::string TestData<Derived>::_source_testdata; + +template <class Derived> +std::string TestData<Derived>::_build_testdata; + +template <class Derived> +void +TestData<Derived>::setup_test_data(const std::string& source_testdata_in, const std::string& build_testdata_in) +{ + _source_testdata = source_testdata_in; + _build_testdata = build_testdata_in; + std::filesystem::create_directory(build_testdata()); +} + +template <class Derived> +void +TestData<Derived>::tear_down_test_data() +{ + std::filesystem::remove(build_testdata()); +} + +template <class Derived> +void +TestData<Derived>::remove_unchanged_build_testdata_file_or_fail(const nbostream& buf, const std::string& file_name) +{ + auto act_path = build_testdata() + "/" + file_name; + auto exp_path = source_testdata() + "/" + file_name; + ASSERT_TRUE(std::filesystem::exists(exp_path)) << "Missing expected contents file " << exp_path; + auto exp_buf = read_buffer_from_file(exp_path); + ASSERT_TRUE(equiv_buffers(exp_buf, buf)) << "Files " << exp_path << " and " << act_path << + " have diferent contents"; + std::filesystem::remove(act_path); +} + +} diff --git a/vespalib/src/vespa/vespalib/test/test_data_base.cpp b/vespalib/src/vespa/vespalib/test/test_data_base.cpp new file mode 100644 index 00000000000..12bfca50d5e --- /dev/null +++ b/vespalib/src/vespa/vespalib/test/test_data_base.cpp @@ -0,0 +1,45 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "test_data.h" +#include <vespa/vespalib/objects/nbostream.h> +#include <cassert> +#include <cstring> +#include <fstream> + +namespace vespalib::test { + +bool +TestDataBase::equiv_buffers(const nbostream& lhs, const nbostream& rhs) +{ + return lhs.size() == rhs.size() && memcmp(lhs.data(), rhs.data(), lhs.size()) == 0; +} + +nbostream +TestDataBase::read_buffer_from_file(const std::string& path) +{ + auto file = std::ifstream(path, std::ios::in | std::ios::binary | std::ios::ate); + auto size = file.tellg(); + file.seekg(0); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(size); + file.read(static_cast<char *>(buf.get()), size); + assert(file.good()); + file.close(); + return nbostream(std::move(buf), size); +} + +void +TestDataBase::write_buffer_to_file(const nbostream& buf, const std::string& path) +{ + write_buffer_to_file(std::string_view{buf.data(), buf.size()}, path); +} + +void +TestDataBase::write_buffer_to_file(std::string_view buf, const std::string& path) +{ + auto file = std::ofstream(path, std::ios::out | std::ios::binary | std::ios::trunc); + file.write(buf.data(), buf.size()); + assert(file.good()); + file.close(); +} + +} diff --git a/vespalib/src/vespa/vespalib/test/test_data_base.h b/vespalib/src/vespa/vespalib/test/test_data_base.h new file mode 100644 index 00000000000..65ea4533f81 --- /dev/null +++ b/vespalib/src/vespa/vespalib/test/test_data_base.h @@ -0,0 +1,20 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <string_view> + +namespace vespalib { class nbostream; } + +namespace vespalib::test { + +// Utility base class for accessing test data used by unit tests. +class TestDataBase { +public: + static bool equiv_buffers(const nbostream& lhs, const nbostream& rhs); + static nbostream read_buffer_from_file(const std::string& path); + static void write_buffer_to_file(const nbostream& buf, const std::string& path); + static void write_buffer_to_file(std::string_view buf, const std::string& path); +}; + +} diff --git a/vespalib/src/vespa/vespalib/testkit/CMakeLists.txt b/vespalib/src/vespa/vespalib/testkit/CMakeLists.txt index 0ec3754b66d..41bb1345f2c 100644 --- a/vespalib/src/vespa/vespalib/testkit/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/testkit/CMakeLists.txt @@ -6,7 +6,6 @@ vespa_add_library(vespalib_vespalib_testkit OBJECT test_master.cpp test_path.cpp test_state_guard.cpp - testapp.cpp time_bomb.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/testkit/test_hook.cpp b/vespalib/src/vespa/vespalib/testkit/test_hook.cpp index 4f7c7cf53bc..b14b575ae93 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_hook.cpp +++ b/vespalib/src/vespa/vespalib/testkit/test_hook.cpp @@ -1,8 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "test_hook.h" +#include <vespa/vespalib/util/count_down_latch.h> +#include <vespa/vespalib/util/barrier.h> +#include <vespa/vespalib/util/thread.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/size_literals.h> +#include <cassert> #include <regex> namespace vespalib { @@ -29,7 +32,7 @@ TestThreadWrapper::threadEntry() master.check(false, __FILE__, __LINE__, "test threw stuff", false); } _barrier.destroy(); - master.setThreadBarrier(0); + master.setThreadBarrier(nullptr); bool fail = (master.getThreadFailCnt() > preThreadFailCnt); master.setThreadUnwind(false); master.setThreadIgnore(false); @@ -39,30 +42,64 @@ TestThreadWrapper::threadEntry() master.setThreadName(oldThreadName.c_str()); } -TestHook *TestHook::_head = 0; -TestHook *TestHook::_tail = 0; +TestHook *TestHook::_head = nullptr; +TestHook *TestHook::_tail = nullptr; + +TestHook::~TestHook() = default; TestHook::TestHook(const std::string &file, const std::string &name, bool ignore) - : _next(0), + : _next(nullptr), _name(name), _tag(make_string("%s:%s", file.c_str(), name.c_str())), _ignore(ignore) { - if (_head == 0) { - assert(_tail == 0); + if (_head == nullptr) { + assert(_tail == nullptr); _head = this; _tail = this; } else { - assert(_tail != 0); - assert(_tail->_next == 0); + assert(_tail != nullptr); + assert(_tail->_next == nullptr); _tail->_next = this; _tail = this; } } +bool TestHook::runMyTest(const FixtureFactory & fixture_factory, size_t num_threads) { + assert(num_threads > 0); + using ThreadUP = std::unique_ptr<TestThreadWrapper>; + using FixtureUP = std::unique_ptr<TestFixtureWrapper>; + std::vector<TestMaster::TraceItem> traceStack = TestMaster::master.getThreadTraceStack(); + CountDownLatch latch(num_threads); + Barrier barrier(num_threads); + std::vector<FixtureUP> fixtures; + std::vector<ThreadUP> threads; + ThreadPool pool; + threads.reserve(num_threads); + fixtures.reserve(num_threads); + for (size_t i = 0; i < num_threads; ++i) { + FixtureUP fixture_up = fixture_factory(); + fixture_up->thread_id = i; + fixture_up->num_threads = num_threads; + threads.emplace_back(new TestThreadWrapper(_ignore, latch, barrier, traceStack, *fixture_up)); + fixtures.push_back(std::move(fixture_up)); + } + for (size_t i = 1; i < num_threads; ++i) { + pool.start([&target = *threads[i]](){ target.threadEntry(); }); + } + threads[0]->threadEntry(); + latch.await(); + pool.join(); + bool result = true; + for (size_t i = 0; i < num_threads; ++i) { + result = result && threads[i]->getResult(); + } + return result; +} + const char *lookup_subset_pattern(const std::string &name) { const char *pattern = getenv("TEST_SUBSET"); - if (pattern != 0) { + if (pattern != nullptr) { fprintf(stderr, "%s: info: only running tests matching '%s'\n", name.c_str(), pattern); } else { @@ -80,7 +117,7 @@ TestHook::runAll() size_t testsFailed = 0; size_t testsIgnored = 0; size_t testsSkipped = 0; - for (TestHook *test = _head; test != 0; test = test->_next) { + for (TestHook *test = _head; test != nullptr; test = test->_next) { if (std::regex_search(test->_tag, pattern)) { bool ignored = test->_ignore; bool failed = !test->run(); diff --git a/vespalib/src/vespa/vespalib/testkit/test_hook.h b/vespalib/src/vespa/vespalib/testkit/test_hook.h index fd9d743ff19..6887ae157d1 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_hook.h +++ b/vespalib/src/vespa/vespalib/testkit/test_hook.h @@ -2,28 +2,24 @@ #pragma once -#include <vespa/vespalib/util/count_down_latch.h> -#include <vespa/vespalib/util/barrier.h> -#include <vespa/vespalib/util/thread.h> -#include <thread> -#include <string> -#include <vector> -#include <cassert> #include "test_master.h" +#include <functional> namespace vespalib { +class CountDownLatch; + struct TestThreadEntry { virtual void threadEntry() = 0; - virtual ~TestThreadEntry() {} + virtual ~TestThreadEntry() = default; }; struct TestFixtureWrapper { size_t thread_id; size_t num_threads; - TestFixtureWrapper() : thread_id(0), num_threads(1) {} + TestFixtureWrapper() noexcept: thread_id(0), num_threads(1) {} virtual void test_entry_point() = 0; - virtual ~TestFixtureWrapper() {} + virtual ~TestFixtureWrapper() = default; }; class TestThreadWrapper : public TestThreadEntry @@ -39,13 +35,13 @@ private: public: TestThreadWrapper(bool ignore, CountDownLatch &l, Barrier &b, const std::vector<TestMaster::TraceItem> &traceStack, - TestFixtureWrapper &fixture) + TestFixtureWrapper &fixture) noexcept : _result(false), _ignore(ignore), _latch(l), _barrier(b), _traceStack(traceStack), _fixture(fixture) {} void threadEntry() override; - bool getResult() const { + bool getResult() const noexcept { return _result; } }; @@ -61,48 +57,21 @@ private: std::string _tag; bool _ignore; - TestHook(const TestHook &); - TestHook &operator=(const TestHook &); - + using FixtureFactory = std::function<std::unique_ptr<TestFixtureWrapper>()>; + bool runMyTest(const FixtureFactory &fixture_factory, size_t num_threads); protected: TestHook(const std::string &file, const std::string &name, bool ignore); - virtual ~TestHook() {} + virtual ~TestHook(); template <typename T> bool runTest(const T &fixture, size_t num_threads) { - assert(num_threads > 0); - using ThreadUP = std::unique_ptr<TestThreadWrapper>; - using FixtureUP = std::unique_ptr<T>; - std::vector<TestMaster::TraceItem> traceStack = TestMaster::master.getThreadTraceStack(); - CountDownLatch latch(num_threads); - Barrier barrier(num_threads); - std::vector<FixtureUP> fixtures; - std::vector<ThreadUP> threads; - ThreadPool pool; - threads.reserve(num_threads); - fixtures.reserve(num_threads); - for (size_t i = 0; i < num_threads; ++i) { - FixtureUP fixture_up(new T(fixture)); - fixture_up->thread_id = i; - fixture_up->num_threads = num_threads; - threads.emplace_back(new TestThreadWrapper(_ignore, latch, barrier, traceStack, *fixture_up)); - fixtures.push_back(std::move(fixture_up)); - } - for (size_t i = 1; i < num_threads; ++i) { - pool.start([&target = *threads[i]](){ target.threadEntry(); }); - } - threads[0]->threadEntry(); - latch.await(); - pool.join(); - bool result = true; - for (size_t i = 0; i < num_threads; ++i) { - result = result && threads[i]->getResult(); - } - return result; + return runMyTest([&fixture]() { return std::make_unique<T>(fixture); }, num_threads); } virtual bool run() = 0; public: + TestHook(const TestHook &) = delete; + TestHook &operator=(const TestHook &) = delete; static void runAll(); }; #endif diff --git a/vespalib/src/vespa/vespalib/testkit/test_macros.h b/vespalib/src/vespa/vespalib/testkit/test_macros.h index 257ca34cae2..cb9b4c580a1 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_macros.h +++ b/vespalib/src/vespa/vespalib/testkit/test_macros.h @@ -11,21 +11,22 @@ #define TEST_CAT(a, b) TEST_CAT_IMPL(a, b) #define TEST_MASTER vespalib::TestMaster::master #define TEST_DEBUG(lhsFile, rhsFile) TEST_MASTER.openDebugFiles(lhsFile, rhsFile) +#define TEST_DEBUG_STOP() TEST_MASTER.close_debug_files() #define TEST_STATE(msg) vespalib::TestStateGuard TEST_CAT(testStateGuard, __LINE__) (__FILE__, __LINE__, msg) #define TEST_DO(doit) do { TEST_STATE(TEST_STR(doit)); doit; } while(false) #define TEST_FLUSH() TEST_MASTER.flush(__FILE__, __LINE__) #define TEST_TRACE() TEST_MASTER.trace(__FILE__, __LINE__) #define TEST_THREAD(name) TEST_MASTER.setThreadName(name) #define TEST_BARRIER() TEST_MASTER.awaitThreadBarrier(__FILE__, __LINE__) -#define TEST_MAIN() \ - void test_kit_main(); \ - int main(int, char **) \ - { \ - TEST_MASTER.init(__FILE__); \ - test_kit_main(); \ - return (TEST_MASTER.fini() ? 0 : 1); \ - } \ - void test_kit_main() +#define TEST_MAIN() \ + void test_kit_main(int argc, char **argv); \ + int main(int argc, char **argv) \ + { \ + TEST_MASTER.init(__FILE__); \ + test_kit_main(argc, argv); \ + return (TEST_MASTER.fini() ? 0 : 1); \ + } \ + void test_kit_main([[maybe_unused]] int argc, [[maybe_unused]] char **argv) //----------------------------------------------------------------------------- #include "generated_fixture_macros.h" diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.cpp b/vespalib/src/vespa/vespalib/testkit/test_master.cpp index 20fa5a7e860..4698d40ecc5 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.cpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.cpp @@ -38,6 +38,12 @@ TestMaster::TraceItem::TraceItem(const TraceItem &) = default; TestMaster::TraceItem & TestMaster::TraceItem::operator=(const TraceItem &) = default; TestMaster::TraceItem::~TraceItem() = default; +TestMaster::ThreadState::~ThreadState() = default; +TestMaster::ThreadState::ThreadState(const std::string &n) + : name(n), passCnt(0), failCnt(0), preIgnoreFailCnt(0), + ignore(false), unwind(false), traceStack(), barrier(nullptr) +{} + TestMaster::ThreadState & TestMaster::threadState(const lock_guard &) { @@ -171,6 +177,8 @@ TestMaster::TestMaster() setThreadName("master"); } +TestMaster::~TestMaster() = default; + //----------------------------------------------------------------------------- void @@ -291,6 +299,13 @@ TestMaster::openDebugFiles(const std::string &lhsFile, } void +TestMaster::close_debug_files() +{ + lock_guard guard(_lock); + closeDebugFiles(guard); +} + +void TestMaster::pushState(const char *file, uint32_t line, const char *msg) { threadState().traceStack.emplace_back(skip_path(file), line, msg); diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.h b/vespalib/src/vespa/vespalib/testkit/test_master.h index cd73103c49f..232ef8009c9 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.h +++ b/vespalib/src/vespa/vespalib/testkit/test_master.h @@ -17,10 +17,6 @@ class Barrier; **/ class TestMaster { -private: - TestMaster(const TestMaster &); - TestMaster &operator=(const TestMaster &); - public: struct Progress { size_t passCnt; @@ -47,17 +43,17 @@ public: private: struct ThreadState { std::string name; - bool unwind; size_t passCnt; size_t failCnt; - bool ignore; size_t preIgnoreFailCnt; + bool ignore; + bool unwind; std::vector<TraceItem> traceStack; Barrier *barrier; - ThreadState(const std::string &n) - : name(n), unwind(false), passCnt(0), - failCnt(0), ignore(false), preIgnoreFailCnt(0), traceStack(), - barrier(0) {} + ~ThreadState(); + ThreadState(const std::string &n); + ThreadState(ThreadState &&) noexcept = default; + ThreadState & operator=(ThreadState &&) noexcept = default; }; static __thread ThreadState *_threadState; @@ -66,18 +62,20 @@ private: size_t failCnt; FILE *lhsFile; FILE *rhsFile; - SharedState() : passCnt(0), failCnt(0), - lhsFile(0), rhsFile(0) {} + SharedState() noexcept + : passCnt(0), failCnt(0), + lhsFile(nullptr), rhsFile(nullptr) + {} }; -private: - std::mutex _lock; - std::string _name; - SharedState _state; + std::mutex _lock; + std::string _name; + SharedState _state; std::vector<std::unique_ptr<ThreadState> > _threadStorage; using lock_guard = std::lock_guard<std::mutex>; private: + TestMaster(); ThreadState &threadState(const lock_guard &); ThreadState &threadState(); void checkFailed(const lock_guard &, @@ -90,10 +88,11 @@ private: void importThreads(const lock_guard &); bool reportConclusion(const lock_guard &); -private: - TestMaster(); public: + ~TestMaster(); + TestMaster(const TestMaster &) = delete; + TestMaster &operator=(const TestMaster &) = delete; void init(const char *name); std::string getName(); void setThreadName(const char *name); @@ -107,6 +106,7 @@ public: size_t getThreadFailCnt(); Progress getProgress(); void openDebugFiles(const std::string &lhsFile, const std::string &rhsFile); + void close_debug_files(); void pushState(const char *file, uint32_t line, const char *msg); void popState(); bool check(bool rc, const char *file, uint32_t line, diff --git a/vespalib/src/vespa/vespalib/testkit/testapp.cpp b/vespalib/src/vespa/vespalib/testkit/testapp.cpp deleted file mode 100644 index 6998a6dad17..00000000000 --- a/vespalib/src/vespa/vespalib/testkit/testapp.cpp +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "testapp.h" - -namespace vespalib { - -int -TestApp::Entry(int argc, char **argv) -{ - _argc = argc; - _argv = argv; - return Main(); -} - -TestApp::~TestApp() = default; - -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/testkit/testapp.h b/vespalib/src/vespa/vespalib/testkit/testapp.h deleted file mode 100644 index f960a1b1e37..00000000000 --- a/vespalib/src/vespa/vespalib/testkit/testapp.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "test_kit.h" - -#define TEST_INIT(name) do { TEST_MASTER.init(name); } while(false) -#define TEST_DONE() do { return TEST_MASTER.fini() ? 0 : 1; } while(false) - -#define TEST_APPHOOK(app) \ - int main(int argc, char **argv) \ - { \ - app myapp; \ - return myapp.Entry(argc, argv); \ - } -#define TEST_SETUP(test) \ - class test : public vespalib::TestApp \ - { \ - public: int Main() override; \ - }; \ - TEST_APPHOOK(test) - -namespace vespalib { - -class TestApp -{ -public: - int _argc = 0; - char **_argv = nullptr; - - virtual int Main() = 0; - int Entry(int argc, char **argv); - virtual ~TestApp(); -}; - -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 79000a89f5f..406ea68a08a 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -49,6 +49,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT jsonwriter.cpp latch.cpp left_right_heap.cpp + limited_thread_bundle_wrapper.cpp lz4compressor.cpp malloc_mmap_guard.cpp md5.c diff --git a/vespalib/src/vespa/vespalib/util/limited_thread_bundle_wrapper.cpp b/vespalib/src/vespa/vespalib/util/limited_thread_bundle_wrapper.cpp new file mode 100644 index 00000000000..15a5bf101f0 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/limited_thread_bundle_wrapper.cpp @@ -0,0 +1,31 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "limited_thread_bundle_wrapper.h" +#include "exceptions.h" + +namespace vespalib { + +LimitedThreadBundleWrapper::LimitedThreadBundleWrapper(ThreadBundle& thread_bundle, uint32_t max_threads) + : _thread_bundle(thread_bundle), + _max_threads(std::min(max_threads, static_cast<uint32_t>(thread_bundle.size()))) +{ +} + +LimitedThreadBundleWrapper::~LimitedThreadBundleWrapper() = default; + +size_t +LimitedThreadBundleWrapper::size() const +{ + return _max_threads; +} + +void +LimitedThreadBundleWrapper::run(Runnable* const* targets, size_t cnt) +{ + if (cnt > size()) { + throw IllegalArgumentException("too many targets"); + } + _thread_bundle.run(targets, cnt); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/limited_thread_bundle_wrapper.h b/vespalib/src/vespa/vespalib/util/limited_thread_bundle_wrapper.h new file mode 100644 index 00000000000..de3c58bf74d --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/limited_thread_bundle_wrapper.h @@ -0,0 +1,24 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "thread_bundle.h" + +namespace vespalib { + +/* + * A ThreadBundle implementation that limits the number of available threads + * from the backing thread bundle. + */ +class LimitedThreadBundleWrapper : public ThreadBundle +{ + ThreadBundle& _thread_bundle; + const uint32_t _max_threads; +public: + LimitedThreadBundleWrapper(ThreadBundle& thread_bundle, uint32_t max_threadss); + ~LimitedThreadBundleWrapper() override; + size_t size() const override; + void run(Runnable* const* targets, size_t cnt) override; +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/ref_counted.h b/vespalib/src/vespa/vespalib/util/ref_counted.h index 1992bdaa5a8..3daa8bbff99 100644 --- a/vespalib/src/vespa/vespalib/util/ref_counted.h +++ b/vespalib/src/vespa/vespalib/util/ref_counted.h @@ -96,6 +96,7 @@ public: } T *operator->() const noexcept { return _ptr; } T &operator*() const noexcept { return *_ptr; } + T* get() const noexcept { return _ptr; } operator bool() const noexcept { return (_ptr != nullptr); } void reset() noexcept { replace_with(nullptr); } ~ref_counted() noexcept { maybe_subref(); } diff --git a/vespamalloc/src/tests/allocfree/allocfree.cpp b/vespamalloc/src/tests/allocfree/allocfree.cpp index 57362301eec..2d622e08f70 100644 --- a/vespamalloc/src/tests/allocfree/allocfree.cpp +++ b/vespamalloc/src/tests/allocfree/allocfree.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "producerconsumer.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <map> #include <vespa/log/log.h> @@ -10,8 +10,6 @@ using vespalib::Consumer; using vespalib::Producer; using vespalib::ProducerConsumer; -TEST_SETUP(Test); - //----------------------------------------------------------------------------- class FreeWorker : public Consumer { @@ -57,21 +55,19 @@ MallocFreeWorker::~MallocFreeWorker() = default; //----------------------------------------------------------------------------- -int Test::Main() { +TEST_MAIN() { int duration = 10; int numCrossThreadAlloc(2); int numSameThreadAlloc(2); - if (_argc > 1) { - duration = atoi(_argv[1]); + if (argc > 1) { + duration = atoi(argv[1]); } - if (_argc > 2) { - numCrossThreadAlloc = atoi(_argv[2]); + if (argc > 2) { + numCrossThreadAlloc = atoi(argv[2]); } - if (_argc > 3) { - numSameThreadAlloc = atoi(_argv[3]); + if (argc > 3) { + numSameThreadAlloc = atoi(argv[3]); } - TEST_INIT("allocfree_test"); - vespalib::ThreadPool pool; std::map<int, std::shared_ptr<FreeWorker> > freeWorkers; @@ -120,6 +116,4 @@ int Test::Main() { fprintf(stderr, "Did %lu Cross thread malloc/free operations\n", numCrossThreadMallocFreeOperations); fprintf(stderr, "Did %lu Same thread malloc/free operations\n", numSameThreadMallocFreeOperations); fprintf(stderr, "Did %lu Total operations\n", numCrossThreadMallocFreeOperations + numSameThreadMallocFreeOperations); - - TEST_DONE(); } diff --git a/vespamalloc/src/tests/allocfree/creatingmanythreads.cpp b/vespamalloc/src/tests/allocfree/creatingmanythreads.cpp index 2c7384c52bc..be847189c32 100644 --- a/vespamalloc/src/tests/allocfree/creatingmanythreads.cpp +++ b/vespamalloc/src/tests/allocfree/creatingmanythreads.cpp @@ -1,11 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> LOG_SETUP("creatingmanythreads_test"); -TEST_SETUP(Test); - void * thread_alloc(void * arg) { char * v = new char [*static_cast<int *>(arg)]; @@ -13,16 +11,15 @@ void * thread_alloc(void * arg) return NULL; } -int Test::Main() { +TEST_MAIN() { int numThreads(10000); int allocSize(256); - if (_argc > 1) { - numThreads = atoi(_argv[1]); + if (argc > 1) { + numThreads = atoi(argv[1]); } - if (_argc > 2) { - allocSize = atoi(_argv[2]); + if (argc > 2) { + allocSize = atoi(argv[2]); } - TEST_INIT("creatingmanythreads_test"); LOG(info, "Will create and run %d threads each allocating a single block of memory of %d size\n", numThreads, allocSize); for (int i(0); i < numThreads; ) { @@ -33,6 +30,4 @@ int Test::Main() { } LOG(info, "Completed %d tests", i); } - - TEST_DONE(); } diff --git a/vespamalloc/src/tests/allocfree/linklist.cpp b/vespamalloc/src/tests/allocfree/linklist.cpp index 6fd192fd1c6..ef1b2bc893a 100644 --- a/vespamalloc/src/tests/allocfree/linklist.cpp +++ b/vespamalloc/src/tests/allocfree/linklist.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "producerconsumer.h" -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespamalloc/malloc/allocchunk.h> #include <vespamalloc/util/callstack.h> #include <thread> @@ -12,8 +12,6 @@ using vespalib::Consumer; using vespalib::Producer; using vespalib::ProducerConsumer; -TEST_SETUP(Test); - //----------------------------------------------------------------------------- template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC> @@ -115,12 +113,11 @@ private: //----------------------------------------------------------------------------- -int Test::Main() { +TEST_MAIN() { int duration = 10; - if (_argc > 1) { - duration = atoi(_argv[1]); + if (argc > 1) { + duration = atoi(argv[1]); } - TEST_INIT("allocfree_test"); ASSERT_EQUAL(1024ul, sizeof(List)); @@ -185,5 +182,4 @@ int Test::Main() { } n = List::linkOut(sharedList); ASSERT_TRUE(n == nullptr); - TEST_DONE(); } diff --git a/vespamalloc/src/tests/doubledelete/expectsignal.cpp b/vespamalloc/src/tests/doubledelete/expectsignal.cpp index d26c0dae6ae..e3ed1079522 100644 --- a/vespamalloc/src/tests/doubledelete/expectsignal.cpp +++ b/vespamalloc/src/tests/doubledelete/expectsignal.cpp @@ -1,28 +1,19 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/process/process.h> #include <sys/wait.h> using namespace vespalib; -class Test : public TestApp -{ -public: - int Main() override; -}; +TEST_MAIN() { -int Test::Main() -{ - TEST_INIT("expectsignal_test"); + ASSERT_EQUAL(argc, 3); - EXPECT_EQUAL(_argc, 3); - ASSERT_TRUE(_argc == 3); + int retval = strtol(argv[1], NULL, 0); - int retval = strtol(_argv[1], NULL, 0); + fprintf(stderr, "argc=%d : Running '%s' expecting signal %d\n", argc, argv[2], retval); - fprintf(stderr, "argc=%d : Running '%s' expecting signal %d\n", _argc, _argv[2], retval); - - Process cmd(_argv[2]); + Process cmd(argv[2]); for (vespalib::string line = cmd.read_line(); !(line.empty() && cmd.eof()); line = cmd.read_line()) { fprintf(stdout, "%s\n", line.c_str()); } @@ -42,8 +33,4 @@ int Test::Main() } EXPECT_EQUAL(exitCode & 0x7f, retval); - - TEST_DONE(); } - -TEST_APPHOOK(Test) diff --git a/vespamalloc/src/tests/overwrite/expectsignal.cpp b/vespamalloc/src/tests/overwrite/expectsignal.cpp index d26c0dae6ae..2c725ac0784 100644 --- a/vespamalloc/src/tests/overwrite/expectsignal.cpp +++ b/vespamalloc/src/tests/overwrite/expectsignal.cpp @@ -1,28 +1,18 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/process/process.h> #include <sys/wait.h> using namespace vespalib; -class Test : public TestApp -{ -public: - int Main() override; -}; +TEST_MAIN() { + ASSERT_EQUAL(argc, 3); -int Test::Main() -{ - TEST_INIT("expectsignal_test"); + int retval = strtol(argv[1], NULL, 0); - EXPECT_EQUAL(_argc, 3); - ASSERT_TRUE(_argc == 3); + fprintf(stderr, "argc=%d : Running '%s' expecting signal %d\n", argc, argv[2], retval); - int retval = strtol(_argv[1], NULL, 0); - - fprintf(stderr, "argc=%d : Running '%s' expecting signal %d\n", _argc, _argv[2], retval); - - Process cmd(_argv[2]); + Process cmd(argv[2]); for (vespalib::string line = cmd.read_line(); !(line.empty() && cmd.eof()); line = cmd.read_line()) { fprintf(stdout, "%s\n", line.c_str()); } @@ -42,8 +32,4 @@ int Test::Main() } EXPECT_EQUAL(exitCode & 0x7f, retval); - - TEST_DONE(); } - -TEST_APPHOOK(Test) diff --git a/vespamalloc/src/tests/overwrite/overwrite.cpp b/vespamalloc/src/tests/overwrite/overwrite.cpp index c09824c2dad..a7d60fa0de8 100644 --- a/vespamalloc/src/tests/overwrite/overwrite.cpp +++ b/vespamalloc/src/tests/overwrite/overwrite.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> using namespace vespalib; @@ -31,21 +31,7 @@ void delete_vec_real(char *ptr) void (*delete_vec)(char *ptr) = delete_vec_real; -class Test : public TestApp -{ -public: - int Main() override; - ~Test(); -private: - void testFillValue(char *a); - void verifyPreWriteDetection(); // Should abort - void verifyPostWriteDetection(); // Should abort - void verifyWriteAfterFreeDetection(); // Should abort -}; - -Test::~Test() = default; - -void Test::testFillValue(char *a) +void testFillValue(char *a) { // Verify fillvalue EXPECT_EQUAL((int)a[0], 0x66); @@ -79,21 +65,21 @@ void Test::testFillValue(char *a) } } -void Test::verifyPreWriteDetection() +void verifyPreWriteDetection() { char * a = new_vec(8); overwrite_memory(a, -1); delete_vec(a); } -void Test::verifyPostWriteDetection() +void verifyPostWriteDetection() { char * a = new_vec(8); overwrite_memory(a, 8); delete_vec(a); } -void Test::verifyWriteAfterFreeDetection() +void verifyWriteAfterFreeDetection() { // Make sure that enough blocks of memory is allocated and freed. char * a = new_vec(256); @@ -120,10 +106,7 @@ void Test::verifyWriteAfterFreeDetection() } } -int Test::Main() -{ - TEST_INIT("overwrite_test"); - +TEST_MAIN() { char * a = new_vec(256); memset(a, 0x77, 256); a[0] = 0; @@ -136,17 +119,14 @@ int Test::Main() delete_vec(a); EXPECT_EQUAL(a, b); - if (_argc > 1) { + if (argc > 1) { testFillValue(a); - if (strcmp(_argv[1], "prewrite") == 0) { + if (strcmp(argv[1], "prewrite") == 0) { verifyPreWriteDetection(); - return 0; - } else if (strcmp(_argv[1], "postwrite") == 0) { + } else if (strcmp(argv[1], "postwrite") == 0) { verifyPostWriteDetection(); - return 0; - } else if (strcmp(_argv[1], "writeafterfree") == 0) { + } else if (strcmp(argv[1], "writeafterfree") == 0) { verifyWriteAfterFreeDetection(); - return 0; } } else { @@ -155,9 +135,4 @@ int Test::Main() EXPECT_EQUAL((int)a[1], 0x77); EXPECT_EQUAL((int)a[255], 0x77); } - - TEST_DONE(); - return 42; } - -TEST_APPHOOK(Test) diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp index 8e5c6b6cf3c..74a45f9a9d0 100644 --- a/vespamalloc/src/tests/test1/new_test.cpp +++ b/vespamalloc/src/tests/test1/new_test.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/optimized.h> #include <vespa/log/log.h> #include <malloc.h> #include <dlfcn.h> #include <functional> +#include <cassert> LOG_SETUP("new_test"); diff --git a/vespamalloc/src/tests/test1/testatomic.cpp b/vespamalloc/src/tests/test1/testatomic.cpp index 4630d58b735..79347bd1ab8 100644 --- a/vespamalloc/src/tests/test1/testatomic.cpp +++ b/vespamalloc/src/tests/test1/testatomic.cpp @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <vespamalloc/malloc/allocchunk.h> #include <vespamalloc/malloc/mmappool.h> #include <unistd.h> diff --git a/vespamalloc/src/tests/thread/racemanythreads.cpp b/vespamalloc/src/tests/thread/racemanythreads.cpp index bcceb96a24e..d5ef2475e9a 100644 --- a/vespamalloc/src/tests/thread/racemanythreads.cpp +++ b/vespamalloc/src/tests/thread/racemanythreads.cpp @@ -1,28 +1,18 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <unistd.h> +#include <cassert> using namespace vespalib; -class Test : public TestApp -{ -public: - ~Test(); - int Main() override; -}; - -Test::~Test() -{ -} - void * hammer(void * arg) { usleep(4000000); long seconds = * static_cast<const long *>(arg); - long stopTime(time(NULL) + seconds); + long stopTime(time(nullptr) + seconds); pthread_t id = pthread_self(); - while (time(NULL) < stopTime) { + while (time(nullptr) < stopTime) { std::vector<pthread_t *> allocations; for (size_t i(0); i < 2000; i++) { pthread_t *t = new pthread_t[20]; @@ -32,25 +22,23 @@ void * hammer(void * arg) } } - for (size_t i(0); i < allocations.size(); i++) { + for (auto & allocation : allocations) { for (size_t j(0); j < 20; j++) { - assert(allocations[i][j] == id); + assert(allocation[j] == id); } - delete [] allocations[i]; + delete [] allocation; } } return arg; } -int Test::Main() -{ - TEST_INIT("racemanythreads_test"); +TEST_MAIN() { size_t threadCount(1024); long seconds(10); - if (_argc >= 2) { - threadCount = strtoul(_argv[1], NULL, 0); - if (_argc >= 3) { - seconds = strtoul(_argv[2], NULL, 0); + if (argc >= 2) { + threadCount = strtoul(argv[1], nullptr, 0); + if (argc >= 3) { + seconds = strtoul(argv[2], nullptr, 0); } } @@ -65,8 +53,4 @@ int Test::Main() void *retval; EXPECT_EQUAL(pthread_join(threads[i], &retval), 0); } - - TEST_DONE(); } - -TEST_APPHOOK(Test); diff --git a/vespamalloc/src/tests/thread/thread.cpp b/vespamalloc/src/tests/thread/thread.cpp index b0d89f852d0..0498f828cc4 100644 --- a/vespamalloc/src/tests/thread/thread.cpp +++ b/vespamalloc/src/tests/thread/thread.cpp @@ -1,22 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/testkit/test_kit.h> #include <atomic> #include <unistd.h> using namespace vespalib; -class Test : public TestApp -{ -public: - ~Test(); - int Main() override; -}; - -Test::~Test() -{ -} - void * just_return(void * arg) { return arg; @@ -61,15 +50,13 @@ void * just_wait(void * arg) return arg; } -int Test::Main() -{ - TEST_INIT("thread_test"); +TEST_MAIN() { size_t threadCount(102400); - if (_argc >= 3) { - threadCount = strtoul(_argv[2], NULL, 0); + if (argc >= 3) { + threadCount = strtoul(argv[2], NULL, 0); } - const char * testType = _argv[1]; + const char * testType = argv[1]; for (size_t i(0); i < threadCount; i++) { pthread_t th; @@ -114,7 +101,4 @@ int Test::Main() } EXPECT_EQUAL(pthread_attr_destroy(&attr), 0); EXPECT_EQUAL(info._count, 0ul); - TEST_DONE(); } - -TEST_APPHOOK(Test); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 169aee416e5..ba597d177c4 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -228,7 +228,9 @@ public class Curator extends AbstractComponent implements AutoCloseable { /** @see #create(Path, Duration) */ - public boolean create(Path path) { return create(path, null); } + public boolean create(Path path) { + return create(path, null); + } /** * Creates an empty node at a path, creating any parents as necessary. @@ -238,6 +240,7 @@ public class Curator extends AbstractComponent implements AutoCloseable { public boolean create(Path path, Duration ttl) { return create(path, ttl, null); } + private boolean create(Path path, Duration ttl, Stat stat) { if (exists(path)) return false; @@ -261,7 +264,9 @@ public class Curator extends AbstractComponent implements AutoCloseable { } /** - * Creates all the given paths in a single transaction. Any paths which already exists are ignored. + * Creates all the given paths in a single transaction. Any paths which already exists are ignored.<br> + * <em>No, this is <strong>NOT</strong> atomic, when viewed from other ZK clients!</em> Maybe it once was, but it is not anymore, + * unless ZK is configured to run commits and read in a single thread, which is not the default. */ public void createAtomically(Path... paths) { try { @@ -271,7 +276,7 @@ public class Curator extends AbstractComponent implements AutoCloseable { transaction = transaction.create().forPath(path.getAbsolute(), new byte[0]).and(); } } - ((CuratorTransactionFinal)transaction).commit(); + ((CuratorTransactionFinal) transaction).commit(); } catch (Exception e) { throw new RuntimeException("Could not create " + Arrays.toString(paths), e); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index aad71d1f7fa..dbd9357ac26 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -47,12 +47,6 @@ public class MockCurator extends Curator { return (MockCuratorFramework) super.framework(); } - /** - * Lists the entire content of this curator instance as a multiline string. - * Useful for debugging. - */ - public String dumpState() { return mockFramework().fileSystem().dumpState(); } - /** Returns an atomic counter in this, or empty if no such counter is created */ public Optional<DistributedAtomicLong> counter(String path) { return Optional.ofNullable(mockFramework().atomicCounters().get(path)); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java index 4732016e428..4350ed4ebd3 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java @@ -82,12 +82,15 @@ import org.apache.curator.framework.state.ConnectionStateErrorPolicy; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.retry.RetryForever; import org.apache.curator.utils.EnsurePath; +import org.apache.curator.utils.ZookeeperCompatibility; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; + +import java.nio.file.FileSystem; import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; @@ -134,11 +137,11 @@ public class MockCuratorFramework implements CuratorFramework { this.shouldTimeoutOnEnter = shouldTimeoutOnEnter; } - public Map<String, MockAtomicCounter> atomicCounters() { + public Map<String, DistributedAtomicLong> atomicCounters() { return Collections.unmodifiableMap(atomicCounters); } - public MemoryFileSystem fileSystem() { + public FileSystem fileSystem() { return fileSystem; } @@ -239,6 +242,11 @@ public class MockCuratorFramework implements CuratorFramework { @Override public void removeWatchers() {} + + @Override + public ZookeeperCompatibility getZookeeperCompatibility() { + return ZookeeperCompatibility.LATEST; + } } return new MockWatcherRemoveCuratorFramework(true, false); } @@ -305,6 +313,11 @@ public class MockCuratorFramework implements CuratorFramework { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + @Override + public ZookeeperCompatibility getZookeeperCompatibility() { + return ZookeeperCompatibility.LATEST; + } + @Deprecated @Override public EnsurePath newNamespaceAwareEnsurePath(String path) { @@ -337,7 +350,7 @@ public class MockCuratorFramework implements CuratorFramework { return new MockLock(path); } - public MockAtomicCounter createAtomicCounter(String path) { + public DistributedAtomicLong createAtomicCounter(String path) { return atomicCounters.computeIfAbsent(path, (k) -> new MockAtomicCounter(path)); } @@ -438,17 +451,17 @@ public class MockCuratorFramework implements CuratorFramework { } private String nodeName(String baseName, CreateMode createMode) { - switch (createMode) { - case PERSISTENT: case EPHEMERAL: case PERSISTENT_WITH_TTL: return baseName; - case PERSISTENT_SEQUENTIAL: case EPHEMERAL_SEQUENTIAL: return baseName + monotonicallyIncreasingNumber++; - default: throw new UnsupportedOperationException(createMode + " support not implemented in MockCurator"); - } + return switch (createMode) { + case PERSISTENT, EPHEMERAL, PERSISTENT_WITH_TTL -> baseName; + case PERSISTENT_SEQUENTIAL, EPHEMERAL_SEQUENTIAL -> baseName + monotonicallyIncreasingNumber++; + default -> throw new UnsupportedOperationException(createMode + " support not implemented in MockCurator"); + }; } /** Validates a path using the same rules as ZooKeeper */ public static String validatePath(String path) throws IllegalArgumentException { if (path == null) throw new IllegalArgumentException("Path cannot be null"); - if (path.length() == 0) throw new IllegalArgumentException("Path length must be > 0"); + if (path.isEmpty()) throw new IllegalArgumentException("Path length must be > 0"); if (path.charAt(0) != '/') throw new IllegalArgumentException("Path must start with / character"); if (path.length() == 1) return path; // done checking - it's the root if (path.charAt(path.length() - 1) == '/') @@ -675,10 +688,6 @@ public class MockCuratorFramework implements CuratorFramework { return true; } - public void setValue(long value) { - this.value.set(value); - } - @Override public Long preValue() { return value.get(); @@ -1269,7 +1278,7 @@ public class MockCuratorFramework implements CuratorFramework { private class MockCuratorTransactionFinal implements CuratorTransactionFinal { /** The new directory root in which the transactional changes are made */ - private MemoryFileSystem.Node newRoot; + private final MemoryFileSystem.Node newRoot; private boolean committed = false; |