aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHarald Musum <musum@yahooinc.com>2023-04-14 09:18:05 +0200
committerHarald Musum <musum@yahooinc.com>2023-04-14 09:18:05 +0200
commit16176985401e8fef7bd3d95e081c333cd652fe30 (patch)
treef8b2b3663c09b77f349d364f4fe3854b5ac5e39f
parent9a71eb340d65db6b8fb8a815e3a913708d57e2c1 (diff)
parent2279c18214465fd88ce2cc8aa22d8856342e2d85 (diff)
Merge branch 'master' into hmusum/allow-groups-to-be-down
-rw-r--r--client/go/internal/admin/prog/valgrind.go3
-rw-r--r--client/go/internal/admin/prog/valgrind_test.go5
-rw-r--r--client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go3
-rw-r--r--client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go5
-rw-r--r--client/go/internal/cli/auth/zts/zts.go2
-rw-r--r--client/go/internal/cli/cmd/config.go10
-rw-r--r--client/go/internal/cli/cmd/config_test.go1
-rw-r--r--client/go/internal/cli/cmd/feed.go59
-rw-r--r--client/go/internal/cli/cmd/feed_test.go15
-rw-r--r--client/go/internal/cli/cmd/prod.go14
-rw-r--r--client/go/internal/cli/cmd/root.go4
-rw-r--r--client/go/internal/cli/cmd/test.go2
-rw-r--r--client/go/internal/cli/cmd/visit_test.go2
-rw-r--r--client/go/internal/mock/http.go4
-rw-r--r--client/go/internal/util/http.go71
-rw-r--r--client/go/internal/vespa/document/circuit_breaker.go77
-rw-r--r--client/go/internal/vespa/document/circuit_breaker_test.go49
-rw-r--r--client/go/internal/vespa/document/dispatcher.go225
-rw-r--r--client/go/internal/vespa/document/dispatcher_test.go54
-rw-r--r--client/go/internal/vespa/document/document.go21
-rw-r--r--client/go/internal/vespa/document/feeder.go32
-rw-r--r--client/go/internal/vespa/document/feeder_test.go2
-rw-r--r--client/go/internal/vespa/document/http.go123
-rw-r--r--client/go/internal/vespa/document/http_test.go72
-rw-r--r--client/go/internal/vespa/document/throttler.go120
-rw-r--r--client/go/internal/vespa/document/throttler_test.go21
-rw-r--r--client/go/internal/vespa/target.go16
-rw-r--r--client/go/internal/vespa/target_cloud.go4
-rw-r--r--clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java2
-rw-r--r--config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java3
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java6
-rw-r--r--config-model/src/main/java/com/yahoo/schema/OnnxModel.java7
-rw-r--r--config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java14
-rw-r--r--config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java2
-rw-r--r--config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java61
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java11
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java50
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java41
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java36
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java25
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java12
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java24
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java11
-rw-r--r--config-model/src/main/javacc/SchemaParser.jj304
-rw-r--r--config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg2
-rw-r--r--config-model/src/test/derived/globalphase_onnx_inside/test.sd1
-rw-r--r--config-model/src/test/derived/indexswitches/ilscripts.cfg2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java98
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java11
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java24
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java6
-rw-r--r--configdefinitions/src/vespa/dispatch.def26
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java12
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java3
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java2
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java39
-rw-r--r--container-core/src/main/java/com/yahoo/processing/Request.java4
-rw-r--r--container-core/src/main/java/com/yahoo/processing/Response.java2
-rw-r--r--container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java9
-rw-r--r--container-core/src/main/java/com/yahoo/processing/request/CompoundName.java56
-rw-r--r--container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java112
-rw-r--r--container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java85
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/IndexFacts.java98
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java42
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java58
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java3
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java14
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java2
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java2
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java2
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java62
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java8
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java2
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java22
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java23
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java6
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/Query.java39
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java12
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java12
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java31
-rw-r--r--container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java15
-rw-r--r--container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java39
-rw-r--r--container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java13
-rw-r--r--container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java8
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/Model.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/Ranking.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java8
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java6
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java3
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java6
-rw-r--r--container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java12
-rw-r--r--container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java8
-rw-r--r--container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java6
-rw-r--r--container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java2
-rwxr-xr-xcontainer-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java10
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java6
-rw-r--r--container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java7
-rw-r--r--container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java6
-rw-r--r--container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java14
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java15
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java8
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java32
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java4
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java4
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java4
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java13
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java6
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java2
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java (renamed from container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java)5
-rw-r--r--container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java6
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java8
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java4
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java27
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java20
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java15
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java22
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java228
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java70
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java1
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java6
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java17
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java16
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java18
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java14
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java12
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java47
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java15
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java28
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java6
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java20
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java12
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java16
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json8
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json12
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json46
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java7
-rw-r--r--default_build_settings.cmake18
-rw-r--r--document/src/vespa/document/select/parse_utils.cpp2
-rw-r--r--flags/src/main/java/com/yahoo/vespa/flags/Flags.java23
-rw-r--r--flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java6
-rw-r--r--functions.cmake8
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java116
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java2
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java7
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java4
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java10
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java4
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java10
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java9
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java10
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java12
-rw-r--r--indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java35
-rw-r--r--indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java26
-rw-r--r--model-integration/pom.xml5
-rw-r--r--model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java19
-rw-r--r--model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java70
-rw-r--r--model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java53
-rw-r--r--model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java95
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java51
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java7
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java9
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java88
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java7
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java2
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java7
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java23
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java6
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java3
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java6
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java29
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java8
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java3
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java4
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java9
-rw-r--r--parent/pom.xml2
-rwxr-xr-xscrewdriver/release-container-image.sh3
-rw-r--r--searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt1
-rw-r--r--searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp218
-rw-r--r--searchcore/src/tests/proton/docsummary/docsummary_test.cpp5
-rw-r--r--searchlib/src/tests/attribute/attribute_test.cpp44
-rw-r--r--searchlib/src/tests/attribute/extendattributes/extendattribute.cpp176
-rw-r--r--searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp24
-rw-r--r--searchlib/src/tests/memoryindex/field_index/field_index_test.cpp36
-rw-r--r--searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp4
-rw-r--r--searchlib/src/tests/predicate/document_features_store_test.cpp14
-rw-r--r--searchlib/src/vespa/searchlib/attribute/attributevector.h3
-rw-r--r--searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h14
-rw-r--r--searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp36
-rw-r--r--searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h6
-rw-r--r--searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp2
-rw-r--r--searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp2
-rw-r--r--searchlib/src/vespa/searchlib/attribute/postinglistattribute.h2
-rw-r--r--searchlib/src/vespa/searchlib/attribute/postingstore.cpp12
-rw-r--r--searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp2
-rw-r--r--searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp2
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/feature_store.h3
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/field_index.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/memoryindex/word_store.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/predicate/document_features_store.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/predicate/simple_index.hpp6
-rw-r--r--searchlib/src/vespa/searchlib/tensor/CMakeLists.txt1
-rw-r--r--searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp10
-rw-r--r--searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h2
-rw-r--r--searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp6
-rw-r--r--searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h4
-rw-r--r--searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp22
-rw-r--r--searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h8
-rw-r--r--searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp30
-rw-r--r--searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h8
-rw-r--r--searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp181
-rw-r--r--searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h54
-rw-r--r--searchlib/src/vespa/searchlib/test/attribute_builder.cpp8
-rw-r--r--searchlib/src/vespa/searchlib/test/attribute_builder.h4
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp12
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h4
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h5
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp5
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java22
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java8
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java8
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java4
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java4
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java9
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java4
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java9
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java15
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java8
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java9
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java5
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java5
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java3
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java20
-rw-r--r--storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp2
-rw-r--r--storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp4
-rw-r--r--streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp26
-rw-r--r--streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp9
-rw-r--r--streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h3
-rw-r--r--vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java16
-rw-r--r--vdslib/src/tests/distribution/distributiontest.cpp6
-rw-r--r--vespa-dependencies-enforcer/allowed-maven-dependencies.txt2
-rw-r--r--vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java37
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java2
-rw-r--r--vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java26
-rw-r--r--vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java95
-rw-r--r--vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java88
-rw-r--r--vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java164
-rw-r--r--vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java76
-rw-r--r--vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java69
-rw-r--r--vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java287
-rw-r--r--vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java65
-rw-r--r--vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp2
-rw-r--r--vespalib/src/tests/btree/btree_test.cpp2
-rw-r--r--vespalib/src/tests/datastore/array_store/array_store_test.cpp68
-rw-r--r--vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp22
-rw-r--r--vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp14
-rw-r--r--vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp160
-rw-r--r--vespalib/src/tests/datastore/datastore/datastore_test.cpp128
-rw-r--r--vespalib/src/tests/datastore/free_list/free_list_test.cpp12
-rw-r--r--vespalib/src/tests/datastore/unique_store/unique_store_test.cpp41
-rw-r--r--vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp18
-rw-r--r--vespalib/src/tests/signalhandler/CMakeLists.txt5
-rw-r--r--vespalib/src/vespa/vespalib/btree/btree.h4
-rw-r--r--vespalib/src/vespa/vespalib/btree/btreenodeallocator.h4
-rw-r--r--vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp8
-rw-r--r--vespalib/src/vespa/vespalib/btree/btreenodestore.h16
-rw-r--r--vespalib/src/vespa/vespalib/btree/btreenodestore.hpp10
-rw-r--r--vespalib/src/vespa/vespalib/btree/btreestore.h10
-rw-r--r--vespalib/src/vespa/vespalib/btree/btreestore.hpp26
-rw-r--r--vespalib/src/vespa/vespalib/datastore/allocator.h2
-rw-r--r--vespalib/src/vespa/vespalib/datastore/allocator.hpp26
-rw-r--r--vespalib/src/vespa/vespalib/datastore/array_store.h20
-rw-r--r--vespalib/src/vespa/vespalib/datastore/array_store.hpp105
-rw-r--r--vespalib/src/vespa/vespalib/datastore/array_store_config.cpp14
-rw-r--r--vespalib/src/vespa/vespalib/datastore/array_store_config.h28
-rw-r--r--vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h6
-rw-r--r--vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h2
-rw-r--r--vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp7
-rw-r--r--vespalib/src/vespa/vespalib/datastore/buffer_free_list.h7
-rw-r--r--vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp40
-rw-r--r--vespalib/src/vespa/vespalib/datastore/buffer_stats.h50
-rw-r--r--vespalib/src/vespa/vespalib/datastore/buffer_type.cpp155
-rw-r--r--vespalib/src/vespa/vespalib/datastore/buffer_type.h108
-rw-r--r--vespalib/src/vespa/vespalib/datastore/buffer_type.hpp74
-rw-r--r--vespalib/src/vespa/vespalib/datastore/bufferstate.cpp122
-rw-r--r--vespalib/src/vespa/vespalib/datastore/bufferstate.h39
-rw-r--r--vespalib/src/vespa/vespalib/datastore/datastore.h14
-rw-r--r--vespalib/src/vespa/vespalib/datastore/datastore.hpp14
-rw-r--r--vespalib/src/vespa/vespalib/datastore/datastorebase.cpp125
-rw-r--r--vespalib/src/vespa/vespalib/datastore/datastorebase.h57
-rw-r--r--vespalib/src/vespa/vespalib/datastore/free_list.h1
-rw-r--r--vespalib/src/vespa/vespalib/datastore/free_list_allocator.h2
-rw-r--r--vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp10
-rw-r--r--vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h2
-rw-r--r--vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp10
-rw-r--r--vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h8
-rw-r--r--vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp22
-rw-r--r--vespalib/src/vespa/vespalib/datastore/memory_stats.cpp16
-rw-r--r--vespalib/src/vespa/vespalib/datastore/memory_stats.h8
-rw-r--r--vespalib/src/vespa/vespalib/datastore/raw_allocator.h6
-rw-r--r--vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp14
-rw-r--r--vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h4
-rw-r--r--vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp14
-rw-r--r--vespalib/src/vespa/vespalib/datastore/unique_store.hpp2
-rw-r--r--vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp2
-rw-r--r--vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp2
-rw-r--r--vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp18
-rw-r--r--vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h8
-rw-r--r--vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp9
-rw-r--r--vespalib/src/vespa/vespalib/util/CMakeLists.txt4
-rw-r--r--vespalib/src/vespa/vespalib/util/time.cpp2
-rw-r--r--vespalib/src/vespa/vespalib/util/time.h2
-rw-r--r--zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java156
-rw-r--r--zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java50
-rw-r--r--zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java49
-rw-r--r--zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java12
395 files changed, 4957 insertions, 3730 deletions
diff --git a/client/go/internal/admin/prog/valgrind.go b/client/go/internal/admin/prog/valgrind.go
index 7d3fb059f8f..2d7f0a597d9 100644
--- a/client/go/internal/admin/prog/valgrind.go
+++ b/client/go/internal/admin/prog/valgrind.go
@@ -22,9 +22,10 @@ func (p *Spec) ConfigureValgrind() {
p.shouldUseValgrind = false
p.shouldUseCallgrind = false
env := p.Getenv(envvars.VESPA_USE_VALGRIND)
+ allValgrind := env == "all"
parts := strings.Split(env, " ")
for _, part := range parts {
- if p.BaseName == part {
+ if p.BaseName == part || allValgrind {
trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env)
backticks := util.BackTicksWithStderr
out, err := backticks.Run(VALGRIND_PROG, "--help")
diff --git a/client/go/internal/admin/prog/valgrind_test.go b/client/go/internal/admin/prog/valgrind_test.go
index 6ec622277c6..11d9424405f 100644
--- a/client/go/internal/admin/prog/valgrind_test.go
+++ b/client/go/internal/admin/prog/valgrind_test.go
@@ -52,6 +52,11 @@ func TestValgrindDetection(t *testing.T) {
assert.Equal(t, false, spec.shouldUseValgrind)
assert.Equal(t, false, spec.shouldUseCallgrind)
+ t.Setenv("VESPA_USE_VALGRIND", "all")
+ spec.ConfigureValgrind()
+ assert.Equal(t, true, spec.shouldUseValgrind)
+ assert.Equal(t, false, spec.shouldUseCallgrind)
+
t.Setenv("VESPA_USE_VALGRIND", "foo bar")
spec.ConfigureValgrind()
assert.Equal(t, false, spec.shouldUseValgrind)
diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go
index 43a1ed602bd..cccb37df8e5 100644
--- a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go
+++ b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go
@@ -18,9 +18,10 @@ func (p *ProgSpec) configureValgrind() {
p.shouldUseValgrind = false
p.shouldUseCallgrind = false
env := p.getenv(envvars.VESPA_USE_VALGRIND)
+ allValgrind := env == "all"
parts := strings.Split(env, " ")
for _, part := range parts {
- if p.BaseName == part {
+ if p.BaseName == part || allValgrind {
trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env)
backticks := util.BackTicksWithStderr
out, err := backticks.Run("which", "valgrind")
diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go
index 48cc78474ed..1a105d66c4a 100644
--- a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go
+++ b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go
@@ -52,6 +52,11 @@ func TestValgrindDetection(t *testing.T) {
assert.Equal(t, false, spec.shouldUseValgrind)
assert.Equal(t, false, spec.shouldUseCallgrind)
+ t.Setenv("VESPA_USE_VALGRIND", "all")
+ spec.configureValgrind()
+ assert.Equal(t, true, spec.shouldUseValgrind)
+ assert.Equal(t, false, spec.shouldUseCallgrind)
+
t.Setenv("VESPA_USE_VALGRIND", "foo bar")
spec.configureValgrind()
assert.Equal(t, false, spec.shouldUseValgrind)
diff --git a/client/go/internal/cli/auth/zts/zts.go b/client/go/internal/cli/auth/zts/zts.go
index 1e84912a271..caa2d03367d 100644
--- a/client/go/internal/cli/auth/zts/zts.go
+++ b/client/go/internal/cli/auth/zts/zts.go
@@ -37,7 +37,7 @@ func (c *Client) AccessToken(domain string, certificate tls.Certificate) (string
if err != nil {
return "", err
}
- util.SetCertificate(c.client, []tls.Certificate{certificate})
+ util.SetCertificates(c.client, []tls.Certificate{certificate})
response, err := c.client.Do(req, 10*time.Second)
if err != nil {
return "", err
diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go
index fd049864096..2d32c454842 100644
--- a/client/go/internal/cli/cmd/config.go
+++ b/client/go/internal/cli/cmd/config.go
@@ -544,12 +544,12 @@ func (c *Config) list(includeUnset bool) []string {
}
// flagValue returns the set value and default value of the named flag.
-func (c *Config) flagValue(name string) (string, string) {
+func (c *Config) flagValue(name string) (string, string, bool) {
f, ok := c.flags[name]
if !ok {
- return "", ""
+ return "", "", ok
}
- return f.Value.String(), f.DefValue
+ return f.Value.String(), f.DefValue, f.Changed
}
// getNonEmpty returns value of given option, if that value is non-empty
@@ -564,9 +564,9 @@ func (c *Config) getNonEmpty(option string) (string, bool) {
// get returns the value associated with option, from the most preferred source in the following order: flag > local
// config > global config.
func (c *Config) get(option string) (string, bool) {
- flagValue, flagDefault := c.flagValue(option)
+ flagValue, flagDefault, changed := c.flagValue(option)
// explicit flag value always takes precedence over everything else
- if flagValue != flagDefault {
+ if changed {
return flagValue, true
}
// ... then local config, if option is explicitly defined there
diff --git a/client/go/internal/cli/cmd/config_test.go b/client/go/internal/cli/cmd/config_test.go
index 612904061de..458878b4356 100644
--- a/client/go/internal/cli/cmd/config_test.go
+++ b/client/go/internal/cli/cmd/config_test.go
@@ -28,6 +28,7 @@ func TestConfig(t *testing.T) {
assertConfigCommand(t, configHome, "", "config", "set", "target", "http://127.0.0.1:8080")
assertConfigCommand(t, configHome, "", "config", "set", "target", "https://127.0.0.1")
assertConfigCommand(t, configHome, "target = https://127.0.0.1\n", "config", "get", "target")
+ assertConfigCommand(t, configHome, "target = local\n", "config", "get", "-t", "local", "target")
// application
assertConfigCommandErr(t, configHome, "Error: invalid application: \"foo\"\n", "config", "set", "application", "foo")
diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go
index f273c5aa826..895a22d2be5 100644
--- a/client/go/internal/cli/cmd/feed.go
+++ b/client/go/internal/cli/cmd/feed.go
@@ -9,16 +9,20 @@ import (
"time"
"github.com/spf13/cobra"
+ "github.com/vespa-engine/vespa/client/go/internal/util"
+ "github.com/vespa-engine/vespa/client/go/internal/vespa"
"github.com/vespa-engine/vespa/client/go/internal/vespa/document"
)
-func addFeedFlags(cmd *cobra.Command, concurrency *int) {
- cmd.PersistentFlags().IntVarP(concurrency, "concurrency", "T", 64, "Number of goroutines to use for dispatching")
+func addFeedFlags(cmd *cobra.Command, verbose *bool, connections *int) {
+ cmd.PersistentFlags().IntVarP(connections, "connections", "N", 8, "The number of connections to use")
+ cmd.PersistentFlags().BoolVarP(verbose, "verbose", "v", false, "Verbose mode. Print errors as they happen")
}
func newFeedCmd(cli *CLI) *cobra.Command {
var (
- concurrency int
+ verbose bool
+ connections int
)
cmd := &cobra.Command{
Use: "feed FILE",
@@ -26,39 +30,66 @@ func newFeedCmd(cli *CLI) *cobra.Command {
Long: `Feed documents to a Vespa cluster.
A high performance feeding client. This can be used to feed large amounts of
-documents to Vespa cluster efficiently.
+documents to a Vespa cluster efficiently.
The contents of FILE must be either a JSON array or JSON objects separated by
newline (JSONL).
+
+If FILE is a single dash ('-'), documents will be read from standard input.
`,
Example: `$ vespa feed documents.jsonl
+$ cat documents.jsonl | vespa feed -
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
SilenceUsage: true,
Hidden: true, // TODO(mpolden): Remove when ready for public use
RunE: func(cmd *cobra.Command, args []string) error {
- f, err := os.Open(args[0])
- if err != nil {
- return err
+ var r io.Reader
+ if args[0] == "-" {
+ r = cli.Stdin
+ } else {
+ f, err := os.Open(args[0])
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ r = f
}
- defer f.Close()
- return feed(f, cli, concurrency)
+ return feed(r, cli, verbose, connections)
},
}
- addFeedFlags(cmd, &concurrency)
+ addFeedFlags(cmd, &verbose, &connections)
return cmd
}
-func feed(r io.Reader, cli *CLI, concurrency int) error {
+func createServiceClients(service *vespa.Service, n int) []util.HTTPClient {
+ clients := make([]util.HTTPClient, 0, n)
+ for i := 0; i < n; i++ {
+ client := service.Client().Clone()
+ util.ForceHTTP2(client, service.TLSOptions.KeyPair) // Feeding should always use HTTP/2
+ clients = append(clients, client)
+ }
+ return clients
+}
+
+func feed(r io.Reader, cli *CLI, verbose bool, connections int) error {
service, err := documentService(cli)
if err != nil {
return err
}
+ clients := createServiceClients(service, connections)
client := document.NewClient(document.ClientOptions{
BaseURL: service.BaseURL,
- }, service)
- dispatcher := document.NewDispatcher(client, concurrency)
+ }, clients)
+ throttler := document.NewThrottler(connections)
+ // TODO(mpolden): Make doom duration configurable
+ circuitBreaker := document.NewCircuitBreaker(10*time.Second, 0)
+ errWriter := io.Discard
+ if verbose {
+ errWriter = cli.Stderr
+ }
+ dispatcher := document.NewDispatcher(client, throttler, circuitBreaker, errWriter)
dec := document.NewDecoder(r)
start := cli.now()
@@ -78,7 +109,7 @@ func feed(r io.Reader, cli *CLI, concurrency int) error {
return err
}
elapsed := cli.now().Sub(start)
- return writeSummaryJSON(cli.Stdout, client.Stats(), elapsed)
+ return writeSummaryJSON(cli.Stdout, dispatcher.Stats(), elapsed)
}
type number float32
diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go
index 1bf1ef6ab9b..521d2b2abd0 100644
--- a/client/go/internal/cli/cmd/feed_test.go
+++ b/client/go/internal/cli/cmd/feed_test.go
@@ -1,6 +1,7 @@
package cmd
import (
+ "bytes"
"os"
"path/filepath"
"testing"
@@ -42,7 +43,7 @@ func TestFeed(t *testing.T) {
require.Nil(t, cli.Run("feed", jsonFile))
assert.Equal(t, "", stderr.String())
- assert.Equal(t, `{
+ want := `{
"feeder.seconds": 1.000,
"feeder.ok.count": 1,
"feeder.ok.rate": 1.000,
@@ -63,5 +64,15 @@ func TestFeed(t *testing.T) {
"200": 1
}
}
-`, stdout.String())
+`
+ assert.Equal(t, want, stdout.String())
+
+ stdout.Reset()
+ cli.Stdin = bytes.NewBuffer([]byte(`{
+ "put": "id:ns:type::doc1",
+ "fields": {"foo": "123"}
+}`))
+ httpClient.NextResponseString(200, `{"message":"OK"}`)
+ require.Nil(t, cli.Run("feed", "-"))
+ assert.Equal(t, want, stdout.String())
}
diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go
index 0e00597221a..f322bd667df 100644
--- a/client/go/internal/cli/cmd/prod.go
+++ b/client/go/internal/cli/cmd/prod.go
@@ -105,21 +105,17 @@ https://cloud.vespa.ai/en/reference/deployment`,
func newProdSubmitCmd(cli *CLI) *cobra.Command {
return &cobra.Command{
Use: "submit",
- Short: "Submit your application for production deployment",
- Long: `Submit your application for production deployment.
+ Short: "Submit an application for production deployment",
+ Long: `Submit an application for production deployment.
-This commands uploads your application package to Vespa Cloud and deploys it to
+This commands uploads an application package to Vespa Cloud and deploys it to
the production zones specified in deployment.xml.
-Nodes are allocated to your application according to resources specified in
+Nodes are allocated to the application according to resources specified in
services.xml.
-While submitting an application from a local development environment is
-supported, it's strongly recommended that production deployments are performed
-by a continuous build system.
-
For more information about production deployments in Vespa Cloud see:
-https://cloud.vespa.ai/en/getting-to-production
+https://cloud.vespa.ai/en/production-deployment
https://cloud.vespa.ai/en/automated-deployments`,
DisableAutoGenTag: true,
SilenceUsage: true,
diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go
index 58e940d59ef..360af9d0dcf 100644
--- a/client/go/internal/cli/cmd/root.go
+++ b/client/go/internal/cli/cmd/root.go
@@ -366,7 +366,7 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta
return nil, errHint(err, "Deployment to cloud requires a certificate. Try 'vespa auth cert'")
}
deploymentTLSOptions = vespa.TLSOptions{
- KeyPair: &kp.KeyPair,
+ KeyPair: []tls.Certificate{kp.KeyPair},
CertificateFile: kp.CertificateFile,
PrivateKeyFile: kp.PrivateKeyFile,
}
@@ -377,7 +377,7 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta
return nil, errHint(err, "Deployment to hosted requires an Athenz certificate", "Try renewing certificate with 'athenz-user-cert'")
}
apiTLSOptions = vespa.TLSOptions{
- KeyPair: &kp.KeyPair,
+ KeyPair: []tls.Certificate{kp.KeyPair},
CertificateFile: kp.CertificateFile,
PrivateKeyFile: kp.PrivateKeyFile,
}
diff --git a/client/go/internal/cli/cmd/test.go b/client/go/internal/cli/cmd/test.go
index 4a53fe6bed3..05633b1135e 100644
--- a/client/go/internal/cli/cmd/test.go
+++ b/client/go/internal/cli/cmd/test.go
@@ -263,7 +263,7 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin
var response *http.Response
if externalEndpoint {
- util.SetCertificate(context.cli.httpClient, []tls.Certificate{})
+ util.SetCertificates(context.cli.httpClient, []tls.Certificate{})
response, err = context.cli.httpClient.Do(request, 60*time.Second)
} else {
response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout
diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go
index 4302680b9d9..b6e5b893e0b 100644
--- a/client/go/internal/cli/cmd/visit_test.go
+++ b/client/go/internal/cli/cmd/visit_test.go
@@ -47,7 +47,7 @@ func TestQuoteFunc(t *testing.T) {
res := quoteArgForUrl(s)
if i < 32 || i > 127 {
assert.Equal(t, "a+z", res)
- } else {
+ } else if testing.Verbose() { // go test -v
fmt.Printf("res %3d => '%s'\n", i, res)
}
}
diff --git a/client/go/internal/mock/http.go b/client/go/internal/mock/http.go
index 9c55f2e79bf..58614d7e5bd 100644
--- a/client/go/internal/mock/http.go
+++ b/client/go/internal/mock/http.go
@@ -6,6 +6,8 @@ import (
"net/http"
"strconv"
"time"
+
+ "github.com/vespa-engine/vespa/client/go/internal/util"
)
type HTTPClient struct {
@@ -58,3 +60,5 @@ func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Res
},
nil
}
+
+func (c *HTTPClient) Clone() util.HTTPClient { return c }
diff --git a/client/go/internal/util/http.go b/client/go/internal/util/http.go
index cb35932c8e7..dcf05ed3a14 100644
--- a/client/go/internal/util/http.go
+++ b/client/go/internal/util/http.go
@@ -2,9 +2,10 @@
package util
import (
- "bytes"
+ "context"
"crypto/tls"
"fmt"
+ "net"
"net/http"
"time"
@@ -14,6 +15,7 @@ import (
type HTTPClient interface {
Do(request *http.Request, timeout time.Duration) (response *http.Response, error error)
+ Clone() HTTPClient
}
type defaultHTTPClient struct {
@@ -31,50 +33,57 @@ func (c *defaultHTTPClient) Do(request *http.Request, timeout time.Duration) (re
return c.client.Do(request)
}
-func SetCertificate(client HTTPClient, certificates []tls.Certificate) {
+func (c *defaultHTTPClient) Clone() HTTPClient { return CreateClient(c.client.Timeout) }
+
+func SetCertificates(client HTTPClient, certificates []tls.Certificate) {
c, ok := client.(*defaultHTTPClient)
if !ok {
return
}
- // Use HTTP/2 transport explicitly. Connection reuse does not work properly when using regular http.Transport, even
- // though it upgrades to HTTP/2 automatically
- // https://github.com/golang/go/issues/16582
- // https://github.com/golang/go/issues/22091
- var transport *http2.Transport
- if _, ok := c.client.Transport.(*http.Transport); ok {
- transport = &http2.Transport{}
- c.client.Transport = transport
- } else if t, ok := c.client.Transport.(*http2.Transport); ok {
- transport = t
- } else {
- panic(fmt.Sprintf("unknown transport type: %T", c.client.Transport))
- }
- if ok && !c.hasCertificates(transport.TLSClientConfig, certificates) {
- transport.TLSClientConfig = &tls.Config{
+ var tlsConfig *tls.Config = nil
+ if certificates != nil {
+ tlsConfig = &tls.Config{
Certificates: certificates,
MinVersion: tls.VersionTLS12,
}
}
+ if tr, ok := c.client.Transport.(*http.Transport); ok {
+ tr.TLSClientConfig = tlsConfig
+ } else if tr, ok := c.client.Transport.(*http2.Transport); ok {
+ tr.TLSClientConfig = tlsConfig
+ } else {
+ panic(fmt.Sprintf("unknown transport type: %T", c.client.Transport))
+ }
}
-func (c *defaultHTTPClient) hasCertificates(tlsConfig *tls.Config, certs []tls.Certificate) bool {
- if tlsConfig == nil {
- return false
- }
- if len(tlsConfig.Certificates) != len(certs) {
- return false
+func ForceHTTP2(client HTTPClient, certificates []tls.Certificate) {
+ c, ok := client.(*defaultHTTPClient)
+ if !ok {
+ return
}
- for i := 0; i < len(certs); i++ {
- if len(tlsConfig.Certificates[i].Certificate) != len(certs[i].Certificate) {
- return false
+ var tlsConfig *tls.Config = nil
+ var dialFunc func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error)
+ if certificates != nil {
+ tlsConfig = &tls.Config{
+ Certificates: certificates,
+ MinVersion: tls.VersionTLS12,
}
- for j := 0; j < len(certs[i].Certificate); j++ {
- if !bytes.Equal(tlsConfig.Certificates[i].Certificate[j], certs[i].Certificate[j]) {
- return false
- }
+ } else {
+ // No certificate, so force H2C (HTTP/2 over clear-text) by using a non-TLS Dialer
+ dialer := net.Dialer{}
+ dialFunc = func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) {
+ return dialer.DialContext(ctx, network, addr)
}
}
- return true
+ // Use HTTP/2 transport explicitly. Connection reuse does not work properly when using regular http.Transport, even
+ // though it upgrades to HTTP/2 automatically
+ // https://github.com/golang/go/issues/16582
+ // https://github.com/golang/go/issues/22091
+ c.client.Transport = &http2.Transport{
+ AllowHTTP: true,
+ TLSClientConfig: tlsConfig,
+ DialTLSContext: dialFunc,
+ }
}
func CreateClient(timeout time.Duration) HTTPClient {
diff --git a/client/go/internal/vespa/document/circuit_breaker.go b/client/go/internal/vespa/document/circuit_breaker.go
new file mode 100644
index 00000000000..17fc595d58f
--- /dev/null
+++ b/client/go/internal/vespa/document/circuit_breaker.go
@@ -0,0 +1,77 @@
+package document
+
+import (
+ "math"
+ "sync/atomic"
+ "time"
+)
+
+type CircuitState int
+
+const (
+ // CircuitClosed represents a closed circuit. Documents are processed successfully
+ CircuitClosed CircuitState = iota
+ // CircuitHalfOpen represents a half-open circuit. Some errors have happend, but processing may still recover
+ CircuitHalfOpen
+ // CircuitOpen represents a open circuit. Something is broken. We should no longer process documents
+ CircuitOpen
+)
+
+type CircuitBreaker interface {
+ Success()
+ Error(error)
+ State() CircuitState
+}
+
+type timeCircuitBreaker struct {
+ graceDuration time.Duration
+ doomDuration time.Duration
+
+ failingSinceMillis int64
+ lastError atomic.Value
+ halfOpen atomic.Value
+ open atomic.Value
+
+ now func() time.Time
+}
+
+func (b *timeCircuitBreaker) Success() {
+ atomic.StoreInt64(&b.failingSinceMillis, math.MaxInt64)
+ if !b.open.Load().(bool) {
+ b.halfOpen.CompareAndSwap(true, false)
+ }
+}
+
+func (b *timeCircuitBreaker) Error(err error) {
+ if atomic.CompareAndSwapInt64(&b.failingSinceMillis, math.MaxInt64, b.now().UnixMilli()) {
+ b.lastError.Store(err)
+ }
+}
+
+func (b *timeCircuitBreaker) State() CircuitState {
+ failingDuration := b.now().Sub(time.UnixMilli(atomic.LoadInt64(&b.failingSinceMillis)))
+ if failingDuration > b.graceDuration {
+ b.halfOpen.CompareAndSwap(false, true)
+ }
+ if b.doomDuration > 0 && failingDuration > b.doomDuration {
+ b.open.CompareAndSwap(false, true)
+ }
+ if b.open.Load().(bool) {
+ return CircuitOpen
+ } else if b.halfOpen.Load().(bool) {
+ return CircuitHalfOpen
+ }
+ return CircuitClosed
+}
+
+func NewCircuitBreaker(graceDuration, doomDuration time.Duration) *timeCircuitBreaker {
+ b := &timeCircuitBreaker{
+ graceDuration: graceDuration,
+ doomDuration: doomDuration,
+ now: time.Now,
+ failingSinceMillis: math.MaxInt64,
+ }
+ b.open.Store(false)
+ b.halfOpen.Store(false)
+ return b
+}
diff --git a/client/go/internal/vespa/document/circuit_breaker_test.go b/client/go/internal/vespa/document/circuit_breaker_test.go
new file mode 100644
index 00000000000..7a4fffaae27
--- /dev/null
+++ b/client/go/internal/vespa/document/circuit_breaker_test.go
@@ -0,0 +1,49 @@
+package document
+
+import (
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestCircuitBreaker(t *testing.T) {
+ clock := &manualClock{}
+ breaker := NewCircuitBreaker(time.Second, time.Minute)
+ breaker.now = clock.now
+ err := errors.New("error")
+
+ assert.Equal(t, CircuitClosed, breaker.State(), "Initial state is closed")
+
+ clock.advance(100 * time.Second)
+ assert.Equal(t, CircuitClosed, breaker.State(), "State is closed after some time without activity")
+
+ breaker.Success()
+ assert.Equal(t, CircuitClosed, breaker.State(), "State is closed after a success")
+
+ clock.advance(100 * time.Second)
+ assert.Equal(t, CircuitClosed, breaker.State(), "State is closed some time after a success")
+
+ breaker.Error(err)
+ assert.Equal(t, CircuitClosed, breaker.State(), "State is closed right after a failure")
+
+ clock.advance(time.Second)
+ assert.Equal(t, CircuitClosed, breaker.State(), "State is closed until grace duration has passed")
+
+ clock.advance(time.Millisecond)
+ assert.Equal(t, CircuitHalfOpen, breaker.State(), "State is half-open when grace duration has passed")
+
+ breaker.Success()
+ assert.Equal(t, CircuitClosed, breaker.State(), "State is closed after a new success")
+
+ breaker.Error(err)
+ clock.advance(time.Minute)
+ assert.Equal(t, CircuitHalfOpen, breaker.State(), "State is half-open until doom duration has passed")
+
+ clock.advance(time.Millisecond)
+ assert.Equal(t, CircuitOpen, breaker.State(), "State is open when doom duration has passed")
+
+ breaker.Success()
+ assert.Equal(t, CircuitOpen, breaker.State(), "State remains open in spite of new successes")
+}
diff --git a/client/go/internal/vespa/document/dispatcher.go b/client/go/internal/vespa/document/dispatcher.go
index feb562a241a..838a7bc45ee 100644
--- a/client/go/internal/vespa/document/dispatcher.go
+++ b/client/go/internal/vespa/document/dispatcher.go
@@ -1,129 +1,212 @@
package document
import (
+ "container/list"
"fmt"
+ "io"
"sync"
+ "sync/atomic"
+ "time"
)
const maxAttempts = 10
// Dispatcher dispatches documents from a queue to a Feeder.
type Dispatcher struct {
- workers int
- feeder Feeder
- ready chan Id
- inflight map[string]*documentGroup
+ feeder Feeder
+ throttler Throttler
+ circuitBreaker CircuitBreaker
+ stats Stats
+
+ started bool
+ ready chan Id
+ results chan Result
+ inflight map[string]*documentGroup
+ inflightCount int64
+ errWriter io.Writer
+
mu sync.RWMutex
wg sync.WaitGroup
- closed bool
-}
-
-// documentGroup holds document operations which share their ID, and must be dispatched in order.
-type documentGroup struct {
- id Id
- failed bool
- operations []documentOp
- mu sync.Mutex
+ resultWg sync.WaitGroup
}
+// documentOp represents a document operation and the number of times it has been attempted.
type documentOp struct {
document Document
attempts int
}
-func (g *documentGroup) append(op documentOp) {
+// documentGroup holds document operations which share an ID, and must be dispatched in order.
+type documentGroup struct {
+ ops *list.List
+ mu sync.Mutex
+}
+
+func (g *documentGroup) add(op documentOp, first bool) {
g.mu.Lock()
defer g.mu.Unlock()
- g.operations = append(g.operations, op)
+ if g.ops == nil {
+ g.ops = list.New()
+ }
+ if first {
+ g.ops.PushFront(op)
+ } else {
+ g.ops.PushBack(op)
+ }
}
-func NewDispatcher(feeder Feeder, workers int) *Dispatcher {
- if workers < 1 {
- workers = 1
- }
+func NewDispatcher(feeder Feeder, throttler Throttler, breaker CircuitBreaker, errWriter io.Writer) *Dispatcher {
d := &Dispatcher{
- workers: workers,
- feeder: feeder,
- inflight: make(map[string]*documentGroup),
+ feeder: feeder,
+ throttler: throttler,
+ circuitBreaker: breaker,
+ inflight: make(map[string]*documentGroup),
+ errWriter: errWriter,
}
d.start()
return d
}
-func (d *Dispatcher) dispatchAll(g *documentGroup) int {
- g.mu.Lock()
- defer g.mu.Unlock()
- failCount := len(g.operations)
- for i := 0; !g.failed && i < len(g.operations); i++ {
- op := g.operations[i]
- ok := false
- for op.attempts <= maxAttempts && !ok {
- op.attempts += 1
- // TODO(mpolden): Extract function which does throttling/circuit-breaking
- result := d.feeder.Send(op.document)
- ok = result.Status.Success()
+func (d *Dispatcher) sendDocumentIn(group *documentGroup) {
+ group.mu.Lock()
+ defer group.mu.Unlock()
+ defer d.releaseSlot()
+ first := group.ops.Front()
+ if first == nil {
+ panic("sending from empty document group, this should not happen")
+ }
+ op := group.ops.Remove(first).(documentOp)
+ op.attempts++
+ result := d.feeder.Send(op.document)
+ d.results <- result
+ if d.shouldRetry(op, result) {
+ d.enqueue(op)
+ }
+}
+
+func (d *Dispatcher) shouldRetry(op documentOp, result Result) bool {
+ if result.HTTPStatus/100 == 2 || result.HTTPStatus == 404 || result.HTTPStatus == 412 {
+ d.throttler.Success()
+ d.circuitBreaker.Success()
+ return false
+ }
+ if result.HTTPStatus == 429 || result.HTTPStatus == 503 {
+ fmt.Fprintf(d.errWriter, "feed: %s was throttled with status %d: retrying\n", op.document, result.HTTPStatus)
+ d.throttler.Throttled(atomic.LoadInt64(&d.inflightCount))
+ return true
+ }
+ if result.Err != nil || result.HTTPStatus == 500 || result.HTTPStatus == 502 || result.HTTPStatus == 504 {
+ retry := op.attempts <= maxAttempts
+ msg := "feed: " + op.document.String() + " failed with "
+ if result.Err != nil {
+ msg += "error " + result.Err.Error()
+ } else {
+ msg += fmt.Sprintf("status %d", result.HTTPStatus)
}
- if ok {
- failCount--
+ if retry {
+ msg += ": retrying"
} else {
- g.failed = true
- failCount = len(g.operations) - i
+ msg += fmt.Sprintf(": giving up after %d attempts", maxAttempts)
+ }
+ fmt.Fprintln(d.errWriter, msg)
+ d.circuitBreaker.Error(fmt.Errorf("request failed with status %d", result.HTTPStatus))
+ if retry {
+ return true
}
}
- g.operations = nil
- return failCount
+ return false
}
func (d *Dispatcher) start() {
d.mu.Lock()
defer d.mu.Unlock()
- d.closed = false
- d.ready = make(chan Id, 4*d.workers)
- for i := 0; i < d.workers; i++ {
+ if d.started {
+ return
+ }
+ d.ready = make(chan Id, 4096)
+ d.results = make(chan Result, 4096)
+ d.started = true
+ d.wg.Add(1)
+ go func() {
+ defer d.wg.Done()
+ d.readDocuments()
+ }()
+ d.resultWg.Add(1)
+ go func() {
+ defer d.resultWg.Done()
+ d.readResults()
+ }()
+}
+
+func (d *Dispatcher) readDocuments() {
+ for id := range d.ready {
+ d.mu.RLock()
+ group := d.inflight[id.String()]
+ d.mu.RUnlock()
d.wg.Add(1)
go func() {
defer d.wg.Done()
- for id := range d.ready {
- d.mu.RLock()
- group := d.inflight[id.String()]
- d.mu.RUnlock()
- if group != nil {
- failedDocs := d.dispatchAll(group)
- d.feeder.AddStats(Stats{Errors: int64(failedDocs)})
- }
- }
+ d.sendDocumentIn(group)
}()
}
}
-func (d *Dispatcher) Enqueue(doc Document) error {
+func (d *Dispatcher) readResults() {
+ for result := range d.results {
+ d.stats.Add(result.Stats)
+ }
+}
+
+func (d *Dispatcher) enqueue(op documentOp) error {
d.mu.Lock()
- defer d.mu.Unlock()
- if d.closed {
+ if !d.started {
return fmt.Errorf("dispatcher is closed")
}
- group, ok := d.inflight[doc.Id.String()]
- if ok {
- group.append(documentOp{document: doc})
- } else {
- group = &documentGroup{
- id: doc.Id,
- operations: []documentOp{{document: doc}},
- }
- d.inflight[doc.Id.String()] = group
+ group, ok := d.inflight[op.document.Id.String()]
+ if !ok {
+ group = &documentGroup{}
+ d.inflight[op.document.Id.String()] = group
}
- d.ready <- doc.Id
+ d.mu.Unlock()
+ group.add(op, op.attempts > 0)
+ d.enqueueWithSlot(op.document.Id)
return nil
}
-// Close closes the dispatcher and waits for all inflight operations to complete.
-func (d *Dispatcher) Close() error {
+func (d *Dispatcher) enqueueWithSlot(id Id) {
+ d.acquireSlot()
+ d.ready <- id
+ d.throttler.Sent()
+}
+
+func (d *Dispatcher) acquireSlot() {
+ for atomic.LoadInt64(&d.inflightCount) >= d.throttler.TargetInflight() {
+ time.Sleep(time.Millisecond)
+ }
+ atomic.AddInt64(&d.inflightCount, 1)
+}
+
+func (d *Dispatcher) releaseSlot() { atomic.AddInt64(&d.inflightCount, -1) }
+
+func closeAndWait[T any](ch chan T, wg *sync.WaitGroup, d *Dispatcher, markClosed bool) {
d.mu.Lock()
- if !d.closed {
- close(d.ready)
- d.closed = true
+ if d.started {
+ close(ch)
+ if markClosed {
+ d.started = false
+ }
}
d.mu.Unlock()
- d.wg.Wait()
+ wg.Wait()
+}
+
+func (d *Dispatcher) Enqueue(doc Document) error { return d.enqueue(documentOp{document: doc}) }
+
+func (d *Dispatcher) Stats() Stats { return d.stats }
+
+// Close closes the dispatcher and waits for all inflight operations to complete.
+func (d *Dispatcher) Close() error {
+ closeAndWait(d.ready, &d.wg, d, false)
+ closeAndWait(d.results, &d.resultWg, d, true)
return nil
}
diff --git a/client/go/internal/vespa/document/dispatcher_test.go b/client/go/internal/vespa/document/dispatcher_test.go
index 04e0928f2a3..80bc5f603ae 100644
--- a/client/go/internal/vespa/document/dispatcher_test.go
+++ b/client/go/internal/vespa/document/dispatcher_test.go
@@ -1,8 +1,10 @@
package document
import (
+ "io"
"sync"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
)
@@ -10,7 +12,6 @@ import (
type mockFeeder struct {
failAfterNDocs int
documents []Document
- stats Stats
mu sync.Mutex
}
@@ -23,24 +24,24 @@ func (f *mockFeeder) failAfterN(docs int) {
func (f *mockFeeder) Send(doc Document) Result {
f.mu.Lock()
defer f.mu.Unlock()
+ result := Result{Id: doc.Id}
if f.failAfterNDocs > 0 && len(f.documents) >= f.failAfterNDocs {
- return Result{Id: doc.Id, Status: StatusVespaFailure}
+ result.Status = StatusVespaFailure
+ } else {
+ f.documents = append(f.documents, doc)
}
- f.documents = append(f.documents, doc)
- return Result{Id: doc.Id}
-}
-
-func (f *mockFeeder) Stats() Stats { return f.stats }
-
-func (f *mockFeeder) AddStats(stats Stats) {
- f.mu.Lock()
- defer f.mu.Unlock()
- f.stats.Add(stats)
+ if !result.Success() {
+ result.Stats.Errors = 1
+ }
+ return result
}
func TestDispatcher(t *testing.T) {
feeder := &mockFeeder{}
- dispatcher := NewDispatcher(feeder, 2)
+ clock := &manualClock{tick: time.Second}
+ throttler := newThrottler(8, clock.now)
+ breaker := NewCircuitBreaker(time.Second, 0)
+ dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard)
docs := []Document{
{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{"foo": "123"}}`)},
{Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Body: []byte(`{"fields":{"bar": "456"}}`)},
@@ -70,7 +71,10 @@ func TestDispatcherOrdering(t *testing.T) {
{Id: mustParseId("id:ns:type::doc8"), Operation: OperationPut},
{Id: mustParseId("id:ns:type::doc9"), Operation: OperationPut},
}
- dispatcher := NewDispatcher(feeder, len(docs))
+ clock := &manualClock{tick: time.Second}
+ throttler := newThrottler(8, clock.now)
+ breaker := NewCircuitBreaker(time.Second, 0)
+ dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard)
for _, d := range docs {
dispatcher.Enqueue(d)
}
@@ -90,7 +94,7 @@ func TestDispatcherOrdering(t *testing.T) {
}
assert.Equal(t, len(docs), len(feeder.documents))
assert.Equal(t, wantDocs, gotDocs)
- assert.Equal(t, int64(0), feeder.Stats().Errors)
+ assert.Equal(t, int64(0), dispatcher.Stats().Errors)
}
func TestDispatcherOrderingWithFailures(t *testing.T) {
@@ -103,26 +107,26 @@ func TestDispatcherOrderingWithFailures(t *testing.T) {
{Id: commonId, Operation: OperationRemove}, // fails
}
feeder.failAfterN(2)
- dispatcher := NewDispatcher(feeder, len(docs))
+ clock := &manualClock{tick: time.Second}
+ throttler := newThrottler(8, clock.now)
+ breaker := NewCircuitBreaker(time.Second, 0)
+ dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard)
for _, d := range docs {
dispatcher.Enqueue(d)
}
dispatcher.Close()
wantDocs := docs[:2]
assert.Equal(t, wantDocs, feeder.documents)
- assert.Equal(t, int64(2), feeder.Stats().Errors)
+ assert.Equal(t, int64(2), dispatcher.Stats().Errors)
- // Dispatching more documents for same ID fails implicitly
+ // Dispatching more documents for same ID succeed
feeder.failAfterN(0)
dispatcher.start()
dispatcher.Enqueue(Document{Id: commonId, Operation: OperationPut})
dispatcher.Enqueue(Document{Id: commonId, Operation: OperationRemove})
- // Other IDs are fine
- doc2 := Document{Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut}
- doc3 := Document{Id: mustParseId("id:ns:type::doc3"), Operation: OperationPut}
- dispatcher.Enqueue(doc2)
- dispatcher.Enqueue(doc3)
+ dispatcher.Enqueue(Document{Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut})
+ dispatcher.Enqueue(Document{Id: mustParseId("id:ns:type::doc3"), Operation: OperationPut})
dispatcher.Close()
- assert.Equal(t, int64(4), feeder.Stats().Errors)
- assert.Equal(t, 4, len(feeder.documents))
+ assert.Equal(t, int64(2), dispatcher.Stats().Errors)
+ assert.Equal(t, 6, len(feeder.documents))
}
diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go
index 98cb2d1b6c6..efb60ad8c0a 100644
--- a/client/go/internal/vespa/document/document.go
+++ b/client/go/internal/vespa/document/document.go
@@ -130,6 +130,27 @@ type Decoder struct {
jsonl bool
}
+func (d Document) String() string {
+ var sb strings.Builder
+ switch d.Operation {
+ case OperationPut:
+ sb.WriteString("put ")
+ case OperationUpdate:
+ sb.WriteString("update ")
+ case OperationRemove:
+ sb.WriteString("remove ")
+ }
+ sb.WriteString(d.Id.String())
+ if d.Condition != "" {
+ sb.WriteString(", condition=")
+ sb.WriteString(d.Condition)
+ }
+ if d.Create {
+ sb.WriteString(", create=true")
+ }
+ return sb.String()
+}
+
func (d *Decoder) guessMode() error {
for !d.array && !d.jsonl {
b, err := d.buf.ReadByte()
diff --git a/client/go/internal/vespa/document/feeder.go b/client/go/internal/vespa/document/feeder.go
index 6996e649d24..9e6768d0eb4 100644
--- a/client/go/internal/vespa/document/feeder.go
+++ b/client/go/internal/vespa/document/feeder.go
@@ -17,23 +17,24 @@ const (
// StatusTransportFailure indicates that there was failure in the transport layer error while sending the document
// operation to Vespa.
StatusTransportFailure
- // StatusError is a catch-all status for any other error that might occur.
- StatusError
)
// Result represents the result of a feeding operation.
type Result struct {
- Id Id
- Status Status
- Message string
- Trace string
- Err error
+ Id Id
+ Status Status
+ HTTPStatus int
+ Message string
+ Trace string
+ Err error
+ Stats Stats
}
-// Success returns whether status s is considered a success.
-func (s Status) Success() bool { return s == StatusSuccess || s == StatusConditionNotMet }
+func (r Result) Success() bool {
+ return r.Err == nil && (r.Status == StatusSuccess || r.Status == StatusConditionNotMet)
+}
-// Stats represents the summed statistics of a feeder.
+// Stats represents feeding operation statistics.
type Stats struct {
Requests int64
Responses int64
@@ -47,8 +48,6 @@ type Stats struct {
BytesRecv int64
}
-func NewStats() Stats { return Stats{ResponsesByCode: make(map[int]int64)} }
-
// AvgLatency returns the average latency for a request.
func (s Stats) AvgLatency() time.Duration {
requests := s.Requests
@@ -69,6 +68,9 @@ func (s Stats) Successes() int64 {
func (s *Stats) Add(other Stats) {
s.Requests += other.Requests
s.Responses += other.Responses
+ if s.ResponsesByCode == nil && other.ResponsesByCode != nil {
+ s.ResponsesByCode = make(map[int]int64)
+ }
for code, count := range other.ResponsesByCode {
_, ok := s.ResponsesByCode[code]
if ok {
@@ -91,8 +93,4 @@ func (s *Stats) Add(other Stats) {
}
// Feeder is the interface for a consumer of documents.
-type Feeder interface {
- Send(Document) Result
- Stats() Stats
- AddStats(Stats)
-}
+type Feeder interface{ Send(Document) Result }
diff --git a/client/go/internal/vespa/document/feeder_test.go b/client/go/internal/vespa/document/feeder_test.go
index 1368d871436..a7d92495889 100644
--- a/client/go/internal/vespa/document/feeder_test.go
+++ b/client/go/internal/vespa/document/feeder_test.go
@@ -7,7 +7,7 @@ import (
)
func TestStatsAdd(t *testing.T) {
- got := NewStats()
+ var got Stats
got.Add(Stats{Requests: 1})
got.Add(Stats{Requests: 1})
got.Add(Stats{Responses: 1})
diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go
index 2e01d4564ab..588330a0574 100644
--- a/client/go/internal/vespa/document/http.go
+++ b/client/go/internal/vespa/document/http.go
@@ -5,11 +5,12 @@ import (
"encoding/json"
"fmt"
"io"
+ "math"
"net/http"
"net/url"
"strconv"
"strings"
- "sync"
+ "sync/atomic"
"time"
"github.com/vespa-engine/vespa/client/go/internal/util"
@@ -17,11 +18,10 @@ import (
// Client represents a HTTP client for the /document/v1/ API.
type Client struct {
- options ClientOptions
- httpClient util.HTTPClient
- stats Stats
- mu sync.Mutex
- now func() time.Time
+ options ClientOptions
+ httpClients []countingHTTPClient
+ now func() time.Time
+ sendCount int32
}
// ClientOptions specifices the configuration options of a feed client.
@@ -32,6 +32,18 @@ type ClientOptions struct {
TraceLevel *int
}
+type countingHTTPClient struct {
+ client util.HTTPClient
+ inflight int64
+}
+
+func (c *countingHTTPClient) addInflight(n int64) { atomic.AddInt64(&c.inflight, n) }
+
+func (c *countingHTTPClient) Do(req *http.Request, timeout time.Duration) (*http.Response, error) {
+ defer c.addInflight(-1)
+ return c.client.Do(req, timeout)
+}
+
type countingReader struct {
reader io.Reader
bytesRead int64
@@ -43,14 +55,19 @@ func (r *countingReader) Read(p []byte) (int, error) {
return n, err
}
-func NewClient(options ClientOptions, httpClient util.HTTPClient) *Client {
- c := &Client{
- options: options,
- httpClient: httpClient,
- stats: NewStats(),
- now: time.Now,
+func NewClient(options ClientOptions, httpClients []util.HTTPClient) *Client {
+ if len(httpClients) < 1 {
+ panic("need at least one HTTP client")
+ }
+ countingClients := make([]countingHTTPClient, 0, len(httpClients))
+ for _, client := range httpClients {
+ countingClients = append(countingClients, countingHTTPClient{client: client})
+ }
+ return &Client{
+ options: options,
+ httpClients: countingClients,
+ now: time.Now,
}
- return c
}
func (c *Client) queryParams() url.Values {
@@ -113,52 +130,56 @@ func (c *Client) feedURL(d Document, queryParams url.Values) (string, *url.URL,
return httpMethod, u, nil
}
-// Send given document the URL configured in this client.
+func (c *Client) leastBusyClient() *countingHTTPClient {
+ leastBusy := c.httpClients[0]
+ min := int64(math.MaxInt64)
+ next := atomic.AddInt32(&c.sendCount, 1)
+ start := int(next) % len(c.httpClients)
+ for i := range c.httpClients {
+ j := (i + start) % len(c.httpClients)
+ client := c.httpClients[j]
+ inflight := atomic.LoadInt64(&client.inflight)
+ if inflight < min {
+ leastBusy = client
+ min = inflight
+ }
+ }
+ leastBusy.addInflight(1)
+ return &leastBusy
+}
+
+// Send given document to the endpoint configured in this client.
func (c *Client) Send(document Document) Result {
start := c.now()
- stats := NewStats()
- stats.Requests = 1
- defer func() {
- latency := c.now().Sub(start)
- stats.TotalLatency = latency
- stats.MinLatency = latency
- stats.MaxLatency = latency
- c.AddStats(stats)
- }()
+ result := Result{Id: document.Id, Stats: Stats{Requests: 1}}
method, url, err := c.feedURL(document, c.queryParams())
if err != nil {
- stats.Errors = 1
- return Result{Status: StatusError, Err: err}
+ return resultWithErr(result, err)
}
req, err := http.NewRequest(method, url.String(), bytes.NewReader(document.Body))
if err != nil {
- stats.Errors = 1
- return Result{Status: StatusError, Err: err}
+ return resultWithErr(result, err)
}
- resp, err := c.httpClient.Do(req, c.options.Timeout)
+ resp, err := c.leastBusyClient().Do(req, 190*time.Second)
if err != nil {
- stats.Errors = 1
- return Result{Status: StatusTransportFailure, Err: err}
+ return resultWithErr(result, err)
}
defer resp.Body.Close()
- stats.Responses = 1
- stats.ResponsesByCode = map[int]int64{
- resp.StatusCode: 1,
- }
- stats.BytesSent = int64(len(document.Body))
- return c.createResult(document.Id, &stats, resp)
+ elapsed := c.now().Sub(start)
+ return c.resultWithResponse(resp, result, document, elapsed)
}
-func (c *Client) Stats() Stats { return c.stats }
-
-func (c *Client) AddStats(stats Stats) {
- c.mu.Lock()
- defer c.mu.Unlock()
- c.stats.Add(stats)
+func resultWithErr(result Result, err error) Result {
+ result.Stats.Errors++
+ result.Status = StatusTransportFailure
+ result.Err = err
+ return result
}
-func (c *Client) createResult(id Id, stats *Stats, resp *http.Response) Result {
- result := Result{Id: id}
+func (c *Client) resultWithResponse(resp *http.Response, result Result, document Document, elapsed time.Duration) Result {
+ result.HTTPStatus = resp.StatusCode
+ result.Stats.Responses++
+ result.Stats.ResponsesByCode = map[int]int64{resp.StatusCode: 1}
switch resp.StatusCode {
case 200:
result.Status = StatusSuccess
@@ -176,14 +197,18 @@ func (c *Client) createResult(id Id, stats *Stats, resp *http.Response) Result {
cr := countingReader{reader: resp.Body}
jsonDec := json.NewDecoder(&cr)
if err := jsonDec.Decode(&body); err != nil {
- result.Status = StatusError
+ result.Status = StatusVespaFailure
result.Err = fmt.Errorf("failed to decode json response: %w", err)
}
result.Message = body.Message
result.Trace = string(body.Trace)
- stats.BytesRecv = cr.bytesRead
- if !result.Status.Success() {
- stats.Errors = 1
- }
+ result.Stats.BytesSent = int64(len(document.Body))
+ result.Stats.BytesRecv = cr.bytesRead
+ if !result.Success() {
+ result.Stats.Errors++
+ }
+ result.Stats.TotalLatency = elapsed
+ result.Stats.MinLatency = elapsed
+ result.Stats.MaxLatency = elapsed
return result
}
diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go
index f02c87730d5..43eaf1bfdf9 100644
--- a/client/go/internal/vespa/document/http_test.go
+++ b/client/go/internal/vespa/document/http_test.go
@@ -11,6 +11,7 @@ import (
"time"
"github.com/vespa-engine/vespa/client/go/internal/mock"
+ "github.com/vespa-engine/vespa/client/go/internal/util"
)
type manualClock struct {
@@ -19,9 +20,41 @@ type manualClock struct {
}
func (c *manualClock) now() time.Time {
- t := c.t
- c.t = c.t.Add(c.tick)
- return t
+ c.advance(c.tick)
+ return c.t
+}
+
+func (c *manualClock) advance(d time.Duration) { c.t = c.t.Add(d) }
+
+type mockHTTPClient struct {
+ id int
+ *mock.HTTPClient
+}
+
+func TestLeastBusyClient(t *testing.T) {
+ httpClient := mock.HTTPClient{}
+ var httpClients []util.HTTPClient
+ for i := 0; i < 4; i++ {
+ httpClients = append(httpClients, &mockHTTPClient{i, &httpClient})
+ }
+ client := NewClient(ClientOptions{}, httpClients)
+ client.httpClients[0].addInflight(1)
+ client.httpClients[1].addInflight(1)
+ assertLeastBusy(t, 2, client)
+ assertLeastBusy(t, 2, client)
+ assertLeastBusy(t, 3, client)
+ client.httpClients[3].addInflight(1)
+ client.httpClients[1].addInflight(-1)
+ assertLeastBusy(t, 1, client)
+}
+
+func assertLeastBusy(t *testing.T, id int, client *Client) {
+ t.Helper()
+ leastBusy := client.leastBusyClient()
+ got := leastBusy.client.(*mockHTTPClient).id
+ if got != id {
+ t.Errorf("got client.id=%d, want %d", got, id)
+ }
}
func TestClientSend(t *testing.T) {
@@ -34,19 +67,43 @@ func TestClientSend(t *testing.T) {
client := NewClient(ClientOptions{
BaseURL: "https://example.com:1337",
Timeout: time.Duration(5 * time.Second),
- }, &httpClient)
+ }, []util.HTTPClient{&httpClient})
clock := manualClock{t: time.Now(), tick: time.Second}
client.now = clock.now
+ var stats Stats
for i, doc := range docs {
+ wantRes := Result{
+ Id: doc.Id,
+ Stats: Stats{
+ Requests: 1,
+ Responses: 1,
+ TotalLatency: time.Second,
+ MinLatency: time.Second,
+ MaxLatency: time.Second,
+ BytesSent: 25,
+ },
+ }
if i < 2 {
httpClient.NextResponseString(200, `{"message":"All good!"}`)
+ wantRes.Status = StatusSuccess
+ wantRes.HTTPStatus = 200
+ wantRes.Message = "All good!"
+ wantRes.Stats.ResponsesByCode = map[int]int64{200: 1}
+ wantRes.Stats.BytesRecv = 23
} else {
httpClient.NextResponseString(502, `{"message":"Good bye, cruel world!"}`)
+ wantRes.Status = StatusVespaFailure
+ wantRes.HTTPStatus = 502
+ wantRes.Message = "Good bye, cruel world!"
+ wantRes.Stats.ResponsesByCode = map[int]int64{502: 1}
+ wantRes.Stats.Errors = 1
+ wantRes.Stats.BytesRecv = 36
}
res := client.Send(doc)
- if res.Err != nil {
- t.Fatalf("got unexpected error %q", res.Err)
+ if !reflect.DeepEqual(res, wantRes) {
+ t.Fatalf("got result %+v, want %+v", res, wantRes)
}
+ stats.Add(res.Stats)
r := httpClient.LastRequest
if r.Method != http.MethodPut {
t.Errorf("got r.Method = %q, want %q", r.Method, http.MethodPut)
@@ -64,7 +121,6 @@ func TestClientSend(t *testing.T) {
t.Errorf("got r.Body = %q, want %q", string(body), string(wantBody))
}
}
- stats := client.Stats()
want := Stats{
Requests: 3,
Responses: 3,
@@ -174,7 +230,7 @@ func TestClientFeedURL(t *testing.T) {
httpClient := mock.HTTPClient{}
client := NewClient(ClientOptions{
BaseURL: "https://example.com",
- }, &httpClient)
+ }, []util.HTTPClient{&httpClient})
for i, tt := range tests {
moreParams := url.Values{}
moreParams.Set("foo", "ba/r")
diff --git a/client/go/internal/vespa/document/throttler.go b/client/go/internal/vespa/document/throttler.go
new file mode 100644
index 00000000000..5b0aab6174e
--- /dev/null
+++ b/client/go/internal/vespa/document/throttler.go
@@ -0,0 +1,120 @@
+package document
+
+import (
+ "math"
+ "math/rand"
+ "sync/atomic"
+ "time"
+)
+
+const throttlerWeight = 0.7
+
+type Throttler interface {
+ // Sent notifies the the throttler that a document has been sent.
+ Sent()
+ // Success notifies the throttler that document operation succeeded.
+ Success()
+ // Throttled notifies the throttler that a throttling event occured while count documents were in-flight.
+ Throttled(count int64)
+ // TargetInflight returns the ideal number of documents to have in-flight now.
+ TargetInflight() int64
+}
+
+type dynamicThrottler struct {
+ minInflight int64
+ maxInflight int64
+ targetInflight int64
+ targetTimesTen int64
+
+ throughputs []float64
+ ok int64
+ sent int64
+
+ start time.Time
+ now func() time.Time
+}
+
+func newThrottler(connections int, nowFunc func() time.Time) *dynamicThrottler {
+ var (
+ minInflight = 16 * int64(connections)
+ maxInflight = 256 * minInflight // 4096 max streams per connection on the server side
+ )
+ return &dynamicThrottler{
+ minInflight: minInflight,
+ maxInflight: maxInflight,
+ targetInflight: 8 * minInflight,
+ targetTimesTen: 10 * maxInflight,
+
+ throughputs: make([]float64, 128),
+
+ start: nowFunc(),
+ now: nowFunc,
+ }
+}
+
+func NewThrottler(connections int) Throttler { return newThrottler(connections, time.Now) }
+
+func (t *dynamicThrottler) Sent() {
+ currentInflight := atomic.LoadInt64(&t.targetInflight)
+ t.sent++
+ if t.sent*t.sent*t.sent < 100*currentInflight*currentInflight {
+ return
+ }
+ t.sent = 0
+ now := t.now()
+ elapsed := now.Sub(t.start)
+ t.start = now
+ currentThroughput := float64(atomic.SwapInt64(&t.ok, 0)) / float64(elapsed)
+
+ // Use buckets for throughput over inflight, along the log-scale, in [minInflight, maxInflight).
+ index := int(float64(len(t.throughputs)) * math.Log(max(1, min(255, float64(currentInflight)/float64(t.minInflight)))) / math.Log(256))
+ t.throughputs[index] = currentThroughput
+
+ // Loop over throughput measurements and pick the one which optimises throughput and latency.
+ choice := float64(currentInflight)
+ maxObjective := float64(-1)
+ for i := len(t.throughputs) - 1; i >= 0; i-- {
+ if t.throughputs[i] == 0 {
+ continue // Skip unknown values
+ }
+ inflight := float64(t.minInflight) * math.Pow(256, (float64(i)+0.5)/float64(len(t.throughputs)))
+ objective := t.throughputs[i] * math.Pow(inflight, throttlerWeight-1) // Optimise throughput (weight), but also latency (1 - weight)
+ if objective > maxObjective {
+ maxObjective = objective
+ choice = inflight
+ }
+ }
+ target := int64((rand.Float64()*0.20 + 0.92) * choice) // Random walk, skewed towards increase
+ atomic.StoreInt64(&t.targetInflight, max(t.minInflight, min(t.maxInflight, target)))
+}
+
+func (t *dynamicThrottler) Success() {
+ atomic.AddInt64(&t.targetTimesTen, 1)
+ atomic.AddInt64(&t.ok, 1)
+}
+
+func (t *dynamicThrottler) Throttled(inflight int64) {
+ atomic.StoreInt64(&t.targetTimesTen, max(inflight*5, t.minInflight*10))
+}
+
+func (t *dynamicThrottler) TargetInflight() int64 {
+ staticTargetInflight := min(t.maxInflight, atomic.LoadInt64(&t.targetTimesTen)/10)
+ targetInflight := atomic.LoadInt64(&t.targetInflight)
+ return min(staticTargetInflight, targetInflight)
+}
+
+type number interface{ float64 | int64 }
+
+func min[T number](x, y T) T {
+ if x < y {
+ return x
+ }
+ return y
+}
+
+func max[T number](x, y T) T {
+ if x > y {
+ return x
+ }
+ return y
+}
diff --git a/client/go/internal/vespa/document/throttler_test.go b/client/go/internal/vespa/document/throttler_test.go
new file mode 100644
index 00000000000..a22f059207f
--- /dev/null
+++ b/client/go/internal/vespa/document/throttler_test.go
@@ -0,0 +1,21 @@
+package document
+
+import (
+ "testing"
+ "time"
+)
+
+func TestThrottler(t *testing.T) {
+ clock := &manualClock{tick: time.Second}
+ tr := newThrottler(8, clock.now)
+ for i := 0; i < 100; i++ {
+ tr.Sent()
+ }
+ if got, want := tr.TargetInflight(), int64(1024); got != want {
+ t.Errorf("got TargetInflight() = %d, but want %d", got, want)
+ }
+ tr.Throttled(5)
+ if got, want := tr.TargetInflight(), int64(128); got != want {
+ t.Errorf("got TargetInflight() = %d, but want %d", got, want)
+ }
+}
diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go
index 51861eb12ab..bc936623bcb 100644
--- a/client/go/internal/vespa/target.go
+++ b/client/go/internal/vespa/target.go
@@ -74,7 +74,7 @@ type Target interface {
// TLSOptions configures the client certificate to use for cloud API or service requests.
type TLSOptions struct {
- KeyPair *tls.Certificate
+ KeyPair []tls.Certificate
CertificateFile string
PrivateKeyFile string
AthenzDomain string
@@ -93,7 +93,7 @@ type LogOptions struct {
// Do sends request to this service. Any required authentication happens automatically.
func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) {
if s.TLSOptions.AthenzDomain != "" && s.TLSOptions.KeyPair != nil {
- accessToken, err := s.zts.AccessToken(s.TLSOptions.AthenzDomain, *s.TLSOptions.KeyPair)
+ accessToken, err := s.zts.AccessToken(s.TLSOptions.AthenzDomain, s.TLSOptions.KeyPair[0])
if err != nil {
return nil, err
}
@@ -105,6 +105,8 @@ func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Respon
return s.httpClient.Do(request, timeout)
}
+func (s *Service) Client() util.HTTPClient { return s.httpClient }
+
// Wait polls the health check of this service until it succeeds or timeout passes.
func (s *Service) Wait(timeout time.Duration) (int, error) {
url := s.BaseURL
@@ -139,18 +141,18 @@ type requestFunc func() *http.Request
// waitForOK queries url and returns its status code. If the url returns a non-200 status code, it is repeatedly queried
// until timeout elapses.
-func waitForOK(client util.HTTPClient, url string, certificate *tls.Certificate, timeout time.Duration) (int, error) {
+func waitForOK(client util.HTTPClient, url string, certificates []tls.Certificate, timeout time.Duration) (int, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return 0, err
}
okFunc := func(status int, response []byte) (bool, error) { return isOK(status), nil }
- return wait(client, okFunc, func() *http.Request { return req }, certificate, timeout)
+ return wait(client, okFunc, func() *http.Request { return req }, certificates, timeout)
}
-func wait(client util.HTTPClient, fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) {
- if certificate != nil {
- util.SetCertificate(client, []tls.Certificate{*certificate})
+func wait(client util.HTTPClient, fn responseFunc, reqFn requestFunc, certificates []tls.Certificate, timeout time.Duration) (int, error) {
+ if certificates != nil {
+ util.SetCertificates(client, certificates)
}
var (
httpErr error
diff --git a/client/go/internal/vespa/target_cloud.go b/client/go/internal/vespa/target_cloud.go
index 2335d4f3432..1fb3edd78c5 100644
--- a/client/go/internal/vespa/target_cloud.go
+++ b/client/go/internal/vespa/target_cloud.go
@@ -161,7 +161,7 @@ func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, c
}
if service.TLSOptions.KeyPair != nil {
- util.SetCertificate(service.httpClient, []tls.Certificate{*service.TLSOptions.KeyPair})
+ util.SetCertificates(service.httpClient, service.TLSOptions.KeyPair)
}
return service, nil
}
@@ -175,7 +175,7 @@ func (t *cloudTarget) SignRequest(req *http.Request, keyID string) error {
return t.addAuth0AccessToken(req)
}
} else {
- if t.apiOptions.TLSOptions.KeyPair.Certificate == nil {
+ if t.apiOptions.TLSOptions.KeyPair == nil {
return fmt.Errorf("system %s requires a certificate for authentication", t.apiOptions.System.Name)
}
return nil
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java
index 1d28a90352b..08265248959 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java
@@ -507,7 +507,7 @@ public class NodeStateChangeCheckerTest {
}
private Result transitionToSameState(String oldDescription, String newDescription) {
- return transitionToSameState(State.MAINTENANCE, oldDescription, newDescription);
+ return transitionToSameState(MAINTENANCE, oldDescription, newDescription);
}
@Test
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 21797f0f469..fef2354c452 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -72,7 +72,7 @@ public interface ModelContext {
* - Remove all flag data files from hosted-feature-flag repository
*/
interface FeatureFlags {
- @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); }
+ @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2023") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); }
@ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { return "THROUGHPUT"; }
@ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); }
@ModelFeatureFlag(owners = {"baldersheim"}) default String queryDispatchPolicy() { return "adaptive"; }
@@ -112,6 +112,7 @@ public interface ModelContext {
@ModelFeatureFlag(owners = {"arnej","baldersheim"}, removeAfter = "8.110") default boolean useOldJdiscContainerStartup() { return false; }
@ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; }
@ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return true; }
+ @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select summary decode type") default String summaryDecodePolicy() { return "eager"; }
//Below are all flags that must be kept until 7 is out of the door
@ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; }
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index ecbb990f096..c72aa23a836 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -43,6 +43,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
private double defaultTermwiseLimit = 1.0;
private String jvmGCOptions = null;
private String queryDispatchPolicy = "adaptive";
+ private String summaryDecodePolicy = "eager";
private String sequencerType = "THROUGHPUT";
private boolean firstTimeDeployment = false;
private String responseSequencerType = "ADAPTIVE";
@@ -134,6 +135,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
@Override public int heapSizePercentage() { return heapSizePercentage; }
@Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; }
@Override public String queryDispatchPolicy() { return queryDispatchPolicy; }
+ @Override public String summaryDecodePolicy() { return summaryDecodePolicy; }
@Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; }
@Override public Optional<CloudAccount> cloudAccount() { return cloudAccount; }
@Override public boolean allowUserFilters() { return allowUserFilters; }
@@ -191,6 +193,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
queryDispatchPolicy = policy;
return this;
}
+ public TestProperties setSummaryDecodePolicy(String type) {
+ summaryDecodePolicy = type;
+ return this;
+ }
public TestProperties setFeedSequencerType(String type) {
sequencerType = type;
return this;
diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java
index 272b668b5fb..90a27d1f036 100644
--- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java
+++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java
@@ -1,15 +1,17 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.schema;
+import com.yahoo.searchlib.rankingexpression.Reference;
import com.yahoo.tensor.TensorType;
import com.yahoo.vespa.model.ml.OnnxModelInfo;
-import com.yahoo.searchlib.rankingexpression.Reference;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
/**
* A global ONNX model distributed using file distribution, similar to ranking constants.
@@ -21,6 +23,7 @@ public class OnnxModel extends DistributableResource {
private OnnxModelInfo modelInfo = null;
private final Map<String, String> inputMap = new HashMap<>();
private final Map<String, String> outputMap = new HashMap<>();
+ private final Set<String> initializers = new HashSet<>();
private String statelessExecutionMode = null;
private Integer statelessInterOpThreads = null;
@@ -101,11 +104,13 @@ public class OnnxModel extends DistributableResource {
for (String onnxName : modelInfo.getOutputs()) {
addOutputNameMapping(onnxName, OnnxModelInfo.asValidIdentifier(onnxName), false);
}
+ initializers.addAll(modelInfo.getInitializers());
this.modelInfo = modelInfo;
}
public Map<String, String> getInputMap() { return Collections.unmodifiableMap(inputMap); }
public Map<String, String> getOutputMap() { return Collections.unmodifiableMap(outputMap); }
+ public Set<String> getInitializers() { return Set.copyOf(initializers); }
public String getDefaultOutput() {
return modelInfo != null ? modelInfo.getDefaultOutput() : "";
diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
index af072c5b59a..7f578f07fe3 100644
--- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
+++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
@@ -2,7 +2,6 @@
package com.yahoo.schema.expressiontransforms;
import com.yahoo.schema.FeatureNames;
-import com.yahoo.schema.RankProfile;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.Reference;
import com.yahoo.searchlib.rankingexpression.parser.ParseException;
@@ -12,13 +11,12 @@ import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode;
import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
-import com.yahoo.tensor.functions.DynamicTensor;
import com.yahoo.tensor.functions.Generate;
-import com.yahoo.tensor.functions.Slice;
import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;
+import java.util.logging.Logger;
/**
* Analyzes expression to figure out what inputs it needs
@@ -27,6 +25,8 @@ import java.util.Set;
*/
public class InputRecorder extends ExpressionTransformer<InputRecorderContext> {
+ private static final Logger log = Logger.getLogger(InputRecorder.class.getName());
+
private final Set<String> neededInputs;
private final Set<String> handled = new HashSet<>();
@@ -120,7 +120,11 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> {
if (model == null) {
throw new IllegalArgumentException("missing onnx model: " + arg);
}
- for (String onnxInput : model.getInputMap().values()) {
+ model.getInputMap().forEach((onnxName, onnxInput) -> {
+ if (model.getInitializers().contains(onnxName)) {
+ log.fine(() -> "For input '%s': skipping name '%s' as it's an initializer".formatted(onnxInput, onnxName));
+ return;
+ }
var reader = new StringReader(onnxInput);
try {
var asExpression = new RankingExpression(reader);
@@ -128,7 +132,7 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> {
} catch (ParseException e) {
throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage());
}
- }
+ });
return;
}
neededInputs.add(feature.toString());
diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java
index 5509d11885c..b414d3757e2 100644
--- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java
+++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java
@@ -20,7 +20,7 @@ import java.util.List;
* Class converting a collection of schemas from the intermediate format.
*
* @author arnej27959
- **/
+ */
public class ConvertSchemaCollection {
private final IntermediateCollection input;
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java
index d8c1fb3125f..3c7e9b4066f 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java
@@ -62,7 +62,7 @@ public class IndexingValidation extends Processor {
final Set<String> prevNames = new HashSet<>();
@Override
- protected ExpressionConverter branch() {
+ public ExpressionConverter branch() {
MyConverter ret = new MyConverter();
ret.outputs.addAll(outputs);
ret.prevNames.addAll(prevNames);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
deleted file mode 100644
index 9d4e732d3f8..00000000000
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.model.admin.monitoring;
-
-import java.util.Optional;
-
-/**
- * Helper object for CloudWatch configuration.
- *
- * @author gjoranv
- */
-public class CloudWatch {
-
- private final String region;
- private final String namespace;
- private final MetricsConsumer consumer;
-
- private HostedAuth hostedAuth;
- private SharedCredentials sharedCredentials;
-
- public CloudWatch(String region, String namespace, MetricsConsumer consumer) {
- this.region = region;
- this.namespace = namespace;
- this.consumer = consumer;
- }
-
- public String region() { return region; }
- public String namespace() { return namespace; }
- public String consumer() { return consumer.id(); }
-
- public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); }
- public Optional<SharedCredentials> sharedCredentials() {return Optional.ofNullable(sharedCredentials); }
-
- public void setHostedAuth(String accessKeyName, String secretKeyName) {
- hostedAuth = new HostedAuth(accessKeyName, secretKeyName);
- }
-
- public void setSharedCredentials(String file, Optional<String> profile) {
- sharedCredentials = new SharedCredentials(file, profile);
- }
-
- public static class HostedAuth {
- public final String accessKeyName;
- public final String secretKeyName;
-
- HostedAuth(String accessKeyName, String secretKeyName) {
- this.accessKeyName = accessKeyName;
- this.secretKeyName = secretKeyName;
- }
- }
-
- public static class SharedCredentials {
- public final String file;
- public final Optional<String> profile;
-
- SharedCredentials(String file, Optional<String> profile) {
- this.file = file;
- this.profile = profile;
- }
- }
-
-}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
index 671a90a0fee..c6bd4c6e295 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
@@ -41,9 +41,6 @@ public class MetricsConsumer {
private final String id;
private final MetricSet metricSet;
- // TODO: This shouldn't be here
- private final List<CloudWatch> cloudWatches = new ArrayList<>();
-
/**
* @param id the consumer
* @param metricSet the metrics for this consumer
@@ -66,14 +63,6 @@ public class MetricsConsumer {
return metricSet.getMetrics();
}
- public void addCloudWatch(CloudWatch cloudWatch) {
- cloudWatches.add(cloudWatch);
- }
-
- public List<CloudWatch> cloudWatches() {
- return unmodifiableList(cloudWatches);
- }
-
private static MetricsConsumer consumer(String id, MetricSet ... metricSets) {
return new MetricsConsumer(id, new MetricSet(id + "-consumer-metrics", List.of(), Arrays.asList(metricSets)));
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 9c0a6b1b224..ac354d66206 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -142,26 +142,26 @@ public class VespaMetricSet {
addMetric(metrics, ContainerMetrics.HANDLED_REQUESTS.count());
addMetric(metrics, ContainerMetrics.HANDLED_LATENCY, EnumSet.of(sum, count, max));
- addMetric(metrics, ContainerMetrics.SERVER_NUM_OPEN_CONNECTIONS, EnumSet.of(max, last, average));
- addMetric(metrics, ContainerMetrics.SERVER_NUM_CONNECTIONS, EnumSet.of(max, last, average));
+ addMetric(metrics, ContainerMetrics.SERVER_NUM_OPEN_CONNECTIONS, EnumSet.of(max, last, average)); // TODO: Vespa 9: Remove last
+ addMetric(metrics, ContainerMetrics.SERVER_NUM_CONNECTIONS, EnumSet.of(max, last, average)); // TODO: Vespa 9: Remove last
addMetric(metrics, ContainerMetrics.SERVER_BYTES_RECEIVED, EnumSet.of(sum, count));
addMetric(metrics, ContainerMetrics.SERVER_BYTES_SENT, EnumSet.of(sum, count));
- addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_REJECTED_TASKS, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_SIZE, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_ACTIVE_THREADS, EnumSet.of(sum, count, last, min, max));
-
- addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MAX_THREADS, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MIN_THREADS, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_RESERVED_THREADS, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_BUSY_THREADS, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_TOTAL_THREADS, EnumSet.of(sum, count, last, min, max));
- addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max));
+ addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min, max
+ addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min
+ addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last
+ addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_REJECTED_TASKS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min, max
+ addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min
+ addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min
+ addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_ACTIVE_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last
+
+ addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MAX_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove.
+ addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MIN_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove.
+ addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_RESERVED_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove.
+ addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_BUSY_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min
+ addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_TOTAL_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min
+ addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min
addMetric(metrics, ContainerMetrics.HTTPAPI_LATENCY, EnumSet.of(max, sum, count));
addMetric(metrics, ContainerMetrics.HTTPAPI_PENDING, EnumSet.of(max, sum, count));
@@ -192,8 +192,8 @@ public class VespaMetricSet {
addMetric(metrics, ContainerMetrics.JDISC_MEMORY_MAPPINGS.max());
addMetric(metrics, ContainerMetrics.JDISC_OPEN_FILE_DESCRIPTORS.max());
- addMetric(metrics, ContainerMetrics.JDISC_GC_COUNT, EnumSet.of(average, max, last));
- addMetric(metrics, ContainerMetrics.JDISC_GC_MS, EnumSet.of(average, max, last));
+ addMetric(metrics, ContainerMetrics.JDISC_GC_COUNT, EnumSet.of(average, max, last)); // TODO: Vespa 9: Remove last
+ addMetric(metrics, ContainerMetrics.JDISC_GC_MS, EnumSet.of(average, max, last)); // TODO: Vespa 9: Remove last
addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS.last());
addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS_WITH_RETAINED_REFS.last());
@@ -263,19 +263,19 @@ public class VespaMetricSet {
addMetric(metrics, ClusterControllerMetrics.STOPPING_COUNT.last());
addMetric(metrics, ClusterControllerMetrics.UP_COUNT.last());
addMetric(metrics, ClusterControllerMetrics.CLUSTER_STATE_CHANGE_COUNT.baseName());
- addMetric(metrics, ClusterControllerMetrics.BUSY_TICK_TIME_MS, EnumSet.of(last, max, sum, count));
- addMetric(metrics, ClusterControllerMetrics.IDLE_TICK_TIME_MS, EnumSet.of(last, max, sum, count));
+ addMetric(metrics, ClusterControllerMetrics.BUSY_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); // TODO: Vespa 9: Remove last
+ addMetric(metrics, ClusterControllerMetrics.IDLE_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); // TODO: Vespa 9: Remove last
- addMetric(metrics, ClusterControllerMetrics.WORK_MS, EnumSet.of(last, sum, count));
+ addMetric(metrics, ClusterControllerMetrics.WORK_MS, EnumSet.of(last, sum, count)); // TODO: Vespa 9: Remove last
addMetric(metrics, ClusterControllerMetrics.IS_MASTER.last());
addMetric(metrics, ClusterControllerMetrics.REMOTE_TASK_QUEUE_SIZE.last());
// TODO(hakonhall): Update this name once persistent "count" metrics has been implemented.
// DO NOT RELY ON THIS METRIC YET.
addMetric(metrics, ClusterControllerMetrics.NODE_EVENT_COUNT.baseName());
- addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(last, max));
- addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max));
- addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max));
+ addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last
+ addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last
+ addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last
addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MEMORY_LIMIT.last());
addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_DISK_LIMIT.last());
addMetric(metrics, ClusterControllerMetrics.REINDEXING_PROGRESS.last());
@@ -311,7 +311,7 @@ public class VespaMetricSet {
addMetric(metrics, ContainerMetrics.DOCUMENTS_COVERED.count());
addMetric(metrics, ContainerMetrics.DOCUMENTS_TOTAL.count());
addMetric(metrics, ContainerMetrics.DOCUMENTS_TARGET_TOTAL.count());
- addMetric(metrics, ContainerMetrics.JDISC_RENDER_LATENCY, EnumSet.of(min, max, count, sum, last, average));
+ addMetric(metrics, ContainerMetrics.JDISC_RENDER_LATENCY, EnumSet.of(min, max, count, sum, last, average)); // TODO: Vespa 9: Remove last, average
addMetric(metrics, ContainerMetrics.QUERY_ITEM_COUNT, EnumSet.of(max, sum, count));
addMetric(metrics, ContainerMetrics.TOTAL_HITS_PER_QUERY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile));
addMetric(metrics, ContainerMetrics.EMPTY_RESULTS.rate());
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
index 5e76311dce3..aa61d2ffb74 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
@@ -29,12 +29,4 @@ public class Metrics {
.anyMatch(existing -> existing.equalsIgnoreCase(id));
}
- /**
- * Returns true if any of the consumers have specified external metric systems.
- */
- public boolean usesExternalMetricSystems() {
- return consumers.values().stream()
- .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty());
- }
-
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
deleted file mode 100644
index ada4cb76d8e..00000000000
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.model.admin.monitoring.builder.xml;
-
-import com.yahoo.vespa.model.admin.monitoring.CloudWatch;
-import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
-import org.w3c.dom.Element;
-
-import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute;
-import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild;
-
-/**
- * @author gjoranv
- */
-public class CloudWatchBuilder {
-
- private static final String REGION_ATTRIBUTE = "region";
- private static final String NAMESPACE_ATTRIBUTE = "namespace";
- private static final String CREDENTIALS_ELEMENT = "credentials";
- private static final String ACCESS_KEY_ATTRIBUTE = "access-key-name";
- private static final String SECRET_KEY_ATTRIBUTE = "secret-key-name";
- private static final String SHARED_CREDENTIALS_ELEMENT = "shared-credentials";
- private static final String PROFILE_ATTRIBUTE = "profile";
- private static final String FILE_ATTRIBUTE = "file";
-
- public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) {
- CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE),
- cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE),
- consumer);
-
- getOptionalChild(cloudwatchElement, CREDENTIALS_ELEMENT)
- .ifPresent(elem -> cloudWatch.setHostedAuth(elem.getAttribute(ACCESS_KEY_ATTRIBUTE),
- elem.getAttribute(SECRET_KEY_ATTRIBUTE)));
-
- getOptionalChild(cloudwatchElement, SHARED_CREDENTIALS_ELEMENT)
- .ifPresent(elem -> cloudWatch.setSharedCredentials(elem.getAttribute(FILE_ATTRIBUTE),
- getOptionalAttribute(elem, PROFILE_ATTRIBUTE)));
-
- return cloudWatch;
- }
-
-}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
index cfd2ebb2fd1..f75b2a864f9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
@@ -41,9 +41,6 @@ public class MetricsBuilder {
MetricSet metricSet = buildMetricSet(consumerId, consumerElement);
var consumer = new MetricsConsumer(consumerId, metricSet);
- for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) {
- consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer));
- }
metrics.addConsumer(consumer);
}
return metrics;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java
deleted file mode 100644
index 847a3402dc5..00000000000
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.model.application.validation;
-
-import com.yahoo.config.model.ConfigModelContext;
-import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.vespa.model.VespaModel;
-import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
-
-import java.util.List;
-
-/**
- * @author gjoranv
- */
-public class CloudWatchValidator extends Validator {
-
- @Override
- public void validate(VespaModel model, DeployState deployState) {
- if (!deployState.isHosted()) return;
- if (deployState.zone().system().isPublic()) return;
- if (model.getAdmin().getApplicationType() != ConfigModelContext.ApplicationType.DEFAULT) return;
-
- var offendingConsumers = model.getAdmin().getUserMetrics().getConsumers().values().stream()
- .filter(consumer -> !consumer.cloudWatches().isEmpty())
- .toList();
-
- if (! offendingConsumers.isEmpty()) {
- throw new IllegalArgumentException("CloudWatch cannot be set up for non-public hosted Vespa and must " +
- "be removed for consumers: " + consumerIds(offendingConsumers));
- }
- }
-
- private List<String> consumerIds(List<MetricsConsumer> offendingConsumers) {
- return offendingConsumers.stream().map(MetricsConsumer::id).toList();
- }
-
-}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
index c7a363010b7..2576d9cb392 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -82,7 +82,6 @@ public class Validation {
new SecretStoreValidator().validate(model, deployState);
new EndpointCertificateSecretsValidator().validate(model, deployState);
new AccessControlFilterValidator().validate(model, deployState);
- new CloudWatchValidator().validate(model, deployState);
new QuotaValidator().validate(model, deployState);
new UriBindingsValidator().validate(model, deployState);
new CloudDataPlaneFilterValidator().validate(model, deployState);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java
index 0cc52edf3cc..b1eace947cc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java
@@ -2,11 +2,15 @@
package com.yahoo.vespa.model.application.validation.change;
import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.model.api.ServiceInfo;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.vespa.model.container.ApplicationContainer;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -19,16 +23,23 @@ public class ContentClusterRemovalValidator implements ChangeValidator {
@Override
public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) {
+ List<ConfigChangeAction> actions = new ArrayList<>();
for (String currentClusterId : current.getContentClusters().keySet()) {
ContentCluster nextCluster = next.getContentClusters().get(currentClusterId);
- if (nextCluster == null)
+ if (nextCluster == null) {
deployState.validationOverrides().invalid(ValidationId.contentClusterRemoval,
- "Content cluster '" + currentClusterId + "' is removed. " +
- "This will cause loss of all data in this cluster",
- deployState.now());
- }
+ "Content cluster '" + currentClusterId + "' is removed. " +
+ "This will cause loss of all data in this cluster",
+ deployState.now());
- return List.of();
+ // If we allow the removal, we must restart all containers to ensure mbus is OK.
+ for (ApplicationContainerCluster cluster : next.getContainerClusters().values()) {
+ actions.add(new VespaRestartAction(cluster.id(),
+ "Content cluster '" + currentClusterId + "' has been removed",
+ cluster.getContainers().stream().map(ApplicationContainer::getServiceInfo).toList()));
+ }
+ }
+ }
+ return actions;
}
-
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java
index 5e8bb85c29d..2b5dadf5512 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java
@@ -51,15 +51,18 @@ public class IdentityProvider extends SimpleComponent implements IdentityConfig.
builder.loadBalancerAddress(loadBalancerName.value());
builder.ztsUrl(ztsUrl != null ? ztsUrl.toString() : "");
builder.athenzDnsSuffix(athenzDnsSuffix != null ? athenzDnsSuffix : "");
- builder.nodeIdentityName("vespa.vespa.tenant"); // TODO Move to Oath configmodel amender
+ builder.nodeIdentityName(configServerDomain() + ".tenant"); // TODO Move to Oath configmodel amender
builder.configserverIdentityName(getConfigserverIdentityName());
}
// TODO Move to Oath configmodel amender
private String getConfigserverIdentityName() {
return String.format("%s.provider_%s_%s",
- zone.system() == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd",
+ configServerDomain(),
zone.environment().value(),
zone.region().value());
}
+ private String configServerDomain() {
+ return zone.system() == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd";
+ }
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
index e2bc9897c85..5184d4ef07a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
@@ -112,7 +112,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
private void addFieldChildren(QueryProfilesConfig.Queryprofile.Builder qpB, QueryProfile profile, String namePrefix) {
List<Map.Entry<String, Object>> content = new ArrayList<>(profile.declaredContent().entrySet());
- Collections.sort(content, new MapEntryKeyComparator());
+ content.sort(new MapEntryKeyComparator());
if (profile.getValue() != null) { // Add "prefix with dot removed"=value:
QueryProfilesConfig.Queryprofile.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Property.Builder();
String fullName = namePrefix.substring(0, namePrefix.length() - 1);
@@ -132,7 +132,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
QueryProfile profile,
String namePrefix) {
List<Map.Entry<String, Object>> content = new ArrayList<>(profile.declaredContent().entrySet());
- Collections.sort(content,new MapEntryKeyComparator());
+ content.sort(new MapEntryKeyComparator());
if (profile.getValue() != null) { // Add "prefix with dot removed"=value:
QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder();
String fullName = namePrefix.substring(0, namePrefix.length() - 1);
@@ -152,8 +152,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
private void addField(QueryProfilesConfig.Queryprofile.Builder qpB,
QueryProfile profile, Entry<String, Object> field, String namePrefix) {
String fullName=namePrefix + field.getKey();
- if (field.getValue() instanceof QueryProfile) {
- QueryProfile subProfile=(QueryProfile)field.getValue();
+ if (field.getValue() instanceof QueryProfile subProfile) {
if ( ! subProfile.isExplicit()) { // Implicitly defined profile - add content
addFieldChildren(qpB, subProfile,fullName + ".");
}
@@ -172,8 +171,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
private void addVariantField(QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder qpB,
Entry<String, Object> field, Boolean overridable, String namePrefix) {
String fullName = namePrefix + field.getKey();
- if (field.getValue() instanceof QueryProfile) {
- QueryProfile subProfile = (QueryProfile)field.getValue();
+ if (field.getValue() instanceof QueryProfile subProfile) {
if ( ! subProfile.isExplicit()) { // Implicitly defined profile - add content
addVariantFieldChildren(qpB, subProfile,fullName + ".");
}
@@ -203,7 +201,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
varB.inherit(inherited.getId().stringValue());
List<Map.Entry<String,Object>> content = new ArrayList<>(variant.getValues().entrySet());
- Collections.sort(content, new MapEntryKeyComparator());
+ content.sort(new MapEntryKeyComparator());
for (Map.Entry<String, Object> entry : content) {
addVariantField(varB, entry, variant.getOverriable().get(entry.getKey()), "");
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java
index 2742dc59fcd..7c89a349d7d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java
@@ -42,13 +42,16 @@ public class OnnxModelInfo {
private final Map<String, OnnxTypeInfo> inputs;
private final Map<String, OnnxTypeInfo> outputs;
private final Map<String, TensorType> vespaTypes = new HashMap<>();
+ private final Set<String> initializers;
- private OnnxModelInfo(ApplicationPackage app, String path, Map<String, OnnxTypeInfo> inputs, Map<String, OnnxTypeInfo> outputs, String defaultOutput) {
+ private OnnxModelInfo(ApplicationPackage app, String path, Map<String, OnnxTypeInfo> inputs,
+ Map<String, OnnxTypeInfo> outputs, Set<String> initializers, String defaultOutput) {
this.app = app;
this.modelPath = path;
this.inputs = Collections.unmodifiableMap(inputs);
this.outputs = Collections.unmodifiableMap(outputs);
this.defaultOutput = defaultOutput;
+ this.initializers = Set.copyOf(initializers);
}
public String getModelPath() {
@@ -63,6 +66,8 @@ public class OnnxModelInfo {
return outputs.keySet();
}
+ public Set<String> getInitializers() { return initializers; }
+
public String getDefaultOutput() {
return defaultOutput;
}
@@ -208,6 +213,14 @@ public class OnnxModelInfo {
}
g.writeEndArray();
+ g.writeArrayFieldStart("initializers");
+ for (Onnx.TensorProto initializers : model.getGraph().getInitializerList()) {
+ g.writeStartObject();
+ g.writeStringField("name", initializers.getName());
+ g.writeEndObject();
+ }
+ g.writeEndArray();
+
g.writeEndObject();
g.close();
return out.toString();
@@ -218,6 +231,7 @@ public class OnnxModelInfo {
JsonNode root = m.readTree(json);
Map<String, OnnxTypeInfo> inputs = new HashMap<>();
Map<String, OnnxTypeInfo> outputs = new HashMap<>();
+ Set<String> initializers = new HashSet<>();
String defaultOutput = "";
String path = null;
@@ -233,7 +247,13 @@ public class OnnxModelInfo {
if (root.get("outputs").has(0)) {
defaultOutput = root.get("outputs").get(0).get("name").textValue();
}
- return new OnnxModelInfo(app, path, inputs, outputs, defaultOutput);
+ var initializerRoot = root.get("initializers");
+ if (initializerRoot != null) {
+ for (JsonNode initializer : initializerRoot) {
+ initializers.add(initializer.get("name").textValue());
+ }
+ }
+ return new OnnxModelInfo(app, path, inputs, outputs, initializers, defaultOutput);
}
static private void onnxTypeToJson(JsonGenerator g, Onnx.ValueInfoProto valueInfo) throws IOException {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 4ed8c5ab2e8..670460a9f9f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -64,6 +64,7 @@ public class IndexedSearchCluster extends SearchCluster
private final List<SearchNode> searchNodes = new ArrayList<>();
private final DispatchTuning.DispatchPolicy defaultDispatchPolicy;
private final double dispatchWarmup;
+ private final String summaryDecodePolicy;
/**
* Returns the document selector that is able to resolve what documents are to be routed to this search cluster.
* This string uses the document selector language as defined in the "document" module.
@@ -80,6 +81,7 @@ public class IndexedSearchCluster extends SearchCluster
rootDispatch = new DispatchGroup(this);
defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy());
dispatchWarmup = featureFlags.queryDispatchWarmup();
+ summaryDecodePolicy = featureFlags.summaryDecodePolicy();
}
@Override
@@ -337,6 +339,15 @@ public class IndexedSearchCluster extends SearchCluster
builder.maxWaitAfterCoverageFactor(searchCoverage.getMaxWaitAfterCoverageFactor());
}
builder.warmuptime(dispatchWarmup);
+ builder.summaryDecodePolicy(toSummaryDecoding(summaryDecodePolicy));
+ }
+
+ private DispatchConfig.SummaryDecodePolicy.Enum toSummaryDecoding(String summaryDecodeType) {
+ return switch (summaryDecodeType.toLowerCase()) {
+ case "eager" -> DispatchConfig.SummaryDecodePolicy.EAGER;
+ case "ondemand","on-demand" -> DispatchConfig.SummaryDecodePolicy.Enum.ONDEMAND;
+ default -> DispatchConfig.SummaryDecodePolicy.Enum.EAGER;
+ };
}
@Override
diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj
index 61e8574bc87..9d6e16b3f67 100644
--- a/config-model/src/main/javacc/SchemaParser.jj
+++ b/config-model/src/main/javacc/SchemaParser.jj
@@ -164,13 +164,13 @@ TOKEN :
| < FIELD: "field" >
| < FIELDS: "fields" >
| < FIELDSET: "fieldset" >
-| < STRUCTFIELD: "struct-field" >
+| < STRUCT_FIELD: "struct-field" >
| < IMPORT: "import" >
| < AS: "as" >
| < INDEXING: "indexing" >
-| < SUMMARYTO: "summary-to" >
-| < DOCUMENTSUMMARY: "document-summary" >
-| < RANKTYPE: "rank-type" >
+| < SUMMARY_TO: "summary-to" >
+| < DOCUMENT_SUMMARY: "document-summary" >
+| < RANK_TYPE: "rank-type" >
| < WEIGHT: "weight" >
| < TYPE: "type" >
| < INDEX: "index" >
@@ -179,26 +179,29 @@ TOKEN :
| < TEXT: "text" >
| < WORD: "word" >
| < GRAM: "gram" >
-| < GRAMSIZE: "gram-size" >
-| < MAXLENGTH: "max-length" >
+| < GRAM_SIZE: "gram-size" >
+| < MAX_LENGTH: "max-length" >
| < PREFIX: "prefix" >
| < SUBSTRING: "substring" >
| < SUFFIX: "suffix" >
| < CONSTANT: "constant">
-| < ONNXMODEL: "onnx-model">
-| < INTRAOPTHREADS: "intraop-threads">
-| < INTEROPTHREADS: "interop-threads">
-| < GPUDEVICE: "gpu-device">
+| < ONNX_MODEL: "onnx-model">
+| < INTRAOP_THREADS: "intraop-threads">
+| < INTEROP_THREADS: "interop-threads">
+| < GPU_DEVICE: "gpu-device">
+| < EXECUTION_MODE: "execution-mode">
+| < PARALLEL: "parallel">
+| < SEQUENTIAL: "sequential">
| < MODEL: "model" >
| < MUTATE: "mutate" >
| < QUERY: "query" >
-| < RANKPROFILE: "rank-profile" >
+| < RANK_PROFILE: "rank-profile" >
| < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" >
| < SUMMARY: "summary" >
| < FULL: "full" >
| < STATIC: "static" >
| < DYNAMIC: "dynamic" >
-| < MATCHEDELEMENTSONLY: "matched-elements-only" >
+| < MATCHED_ELEMENTS_ONLY: "matched-elements-only" >
| < SSCONTEXTUAL: "contextual" >
| < SSOVERRIDE: "override" >
| < SSTITLE: "title" >
@@ -233,7 +236,7 @@ TOKEN :
| < TRUE: "true" >
| < FALSE: "false" >
| < SYMMETRIC: "symmetric" >
-| < QUERYCOMMAND: "query-command" >
+| < QUERY_COMMAND: "query-command" >
| < ALIAS: "alias" >
| < MATCH: "match" >
| < RANK: "rank" >
@@ -241,24 +244,24 @@ TOKEN :
| < EXACT: "exact" >
| < FILTER: "filter" >
| < NORMAL: "normal" >
-| < EXACTTERMINATOR: "exact-terminator" >
-| < IGNOREDEFAULTRANKFEATURES: "ignore-default-rank-features" >
+| < EXACT_TERMINATOR: "exact-terminator" >
+| < IGNORE_DEFAULT_RANK_FEATURES: "ignore-default-rank-features" >
| < ID: "id" >
| < SOURCE: "source" >
| < TO: "to" >
| < DIRECT: "direct" >
-| < FROMDISK: "from-disk" >
-| < OMITSUMMARYFEATURES: "omit-summary-features" >
+| < FROM_DISK: "from-disk" >
+| < OMIT_SUMMARY_FEATURES: "omit-summary-features" >
| < ALWAYS: "always" >
-| < ONDEMAND: "on-demand" >
+| < ON_DEMAND: "on-demand" >
| < NEVER: "never" >
-| < ENABLEBITVECTORS: "enable-bit-vectors" >
-| < ENABLEONLYBITVECTOR: "enable-only-bit-vector" >
-| < FASTACCESS: "fast-access" >
+| < ENABLE_BIT_VECTORS: "enable-bit-vectors" >
+| < ENABLE_ONLY_BIT_VECTOR: "enable-only-bit-vector" >
+| < FAST_ACCESS: "fast-access" >
| < MUTABLE: "mutable" >
| < PAGED: "paged" >
-| < FASTRANK: "fast-rank" >
-| < FASTSEARCH: "fast-search" >
+| < FAST_RANK: "fast-rank" >
+| < FAST_SEARCH: "fast-search" >
| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" >
| < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? >
| < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? >
@@ -272,30 +275,30 @@ TOKEN :
| < MAP: "map" >
| < REFERENCE: "reference" >
| < QUESTIONMARK: "?" >
-| < CREATEIFNONEXISTENT: "create-if-nonexistent" >
-| < REMOVEIFZERO: "remove-if-zero" >
-| < MATCHPHASE: "match-phase" >
+| < CREATE_IF_NONEXISTENT: "create-if-nonexistent" >
+| < REMOVE_IF_ZERO: "remove-if-zero" >
+| < MATCH_PHASE: "match-phase" >
| < EVALUATION_POINT: "evaluation-point" >
| < PRE_POST_FILTER_TIPPING_POINT: "pre-post-filter-tipping-point" >
| < ORDER: "order" >
-| < MAXFILTERCOVERAGE: "max-filter-coverage" >
-| < MAXHITS: "max-hits" >
-| < FIRSTPHASE: "first-phase" >
-| < SECONDPHASE: "second-phase" >
-| < GLOBALPHASE: "global-phase" >
+| < MAX_FILTER_COVERAGE: "max-filter-coverage" >
+| < MAX_HITS: "max-hits" >
+| < FIRST_PHASE: "first-phase" >
+| < SECOND_PHASE: "second-phase" >
+| < GLOBAL_PHASE: "global-phase" >
| < MACRO: "macro" >
| < INLINE: "inline" >
| < ARITY: "arity" >
-| < LOWERBOUND: "lower-bound" >
-| < UPPERBOUND: "upper-bound" >
-| < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" >
+| < LOWER_BOUND: "lower-bound" >
+| < UPPER_BOUND: "upper-bound" >
+| < DENSE_POSTING_LIST_THRESHOLD: "dense-posting-list-threshold" >
| < ENABLE_BM25: "enable-bm25" >
| < HNSW: "hnsw" >
-| < MAXLINKSPERNODE: "max-links-per-node" >
+| < MAX_LINKS_PER_NODE: "max-links-per-node" >
| < DOUBLE_KEYWORD: "double" >
-| < DISTANCEMETRIC: "distance-metric" >
-| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" >
-| < MULTITHREADEDINDEXING: "multi-threaded-indexing" >
+| < DISTANCE_METRIC: "distance-metric" >
+| < NEIGHBORS_TO_EXPLORE_AT_INSERT: "neighbors-to-explore-at-insert" >
+| < MULTI_THREADED_INDEXING: "multi-threaded-indexing" >
| < MATCHFEATURES_SL: "match-features" (" ")* ":" (~["}","\n"])* ("\n")? >
| < MATCHFEATURES_ML: "match-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
| < MATCHFEATURES_ML_INHERITS: "match-features inherits " (<IDENTIFIER_WITH_DASH>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
@@ -315,16 +318,16 @@ TOKEN :
| < #BRACE_ML_LEVEL_3: <BRACE_ML_CONTENT> "}" >
| < #BRACE_ML_CONTENT: (~["{","}"])* >
| < #SEARCHLIB_SKIP: ([" ","\f","\n","\r","\t"])+ >
-| < RANKPROPERTIES: "rank-properties" >
-| < RERANKCOUNT: "rerank-count" >
-| < NUMTHREADSPERSEARCH: "num-threads-per-search" >
-| < MINHITSPERTHREAD: "min-hits-per-thread" >
-| < NUMSEARCHPARTITIONS: "num-search-partitions" >
-| < TERMWISELIMIT: "termwise-limit" >
-| < POSTFILTERTHRESHOLD: "post-filter-threshold" >
-| < APPROXIMATETHRESHOLD: "approximate-threshold" >
-| < KEEPRANKCOUNT: "keep-rank-count" >
-| < RANKSCOREDROPLIMIT: "rank-score-drop-limit" >
+| < RANK_PROPERTIES: "rank-properties" >
+| < RERANK_COUNT: "rerank-count" >
+| < NUM_THREADS_PER_SEARCH: "num-threads-per-search" >
+| < MIN_HITS_PER_THREAD: "min-hits-per-thread" >
+| < NUM_SEARCH_PARTITIONS: "num-search-partitions" >
+| < TERMWISE_LIMIT: "termwise-limit" >
+| < POST_FILTER_THRESHOLD: "post-filter-threshold" >
+| < APPROXIMATE_THRESHOLD: "approximate-threshold" >
+| < KEEP_RANK_COUNT: "keep-rank-count" >
+| < RANK_SCORE_DROP_LIMIT: "rank-score-drop-limit" >
| < CONSTANTS: "constants" >
| < FILE: "file" >
| < URI: "uri" >
@@ -608,7 +611,7 @@ void fieldSet(ParsedSchema schema) :
((
( <FIELDS><COLON> name = identifier() { fieldSet.addField(name); }
( <COMMA> name = identifier() { fieldSet.addField(name); } )* )
- | ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { fieldSet.addQueryCommand(queryCommand); }
+ | ( <QUERY_COMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { fieldSet.addQueryCommand(queryCommand); }
| matchSettings(fieldSet.matchSettings())
) (<NL>)* )+
<RBRACE>
@@ -825,7 +828,7 @@ void structField(ParsedField field) :
ParsedField structField;
}
{
- <STRUCTFIELD> name = identifier() {
+ <STRUCT_FIELD> name = identifier() {
if (name != null && Schema.isReservedName(name.toLowerCase())) {
throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
}
@@ -917,7 +920,7 @@ void summaryTo(ParsedField field) :
ParsedSummaryField psf;
}
{
- <SUMMARYTO> [name = identifier()] <COLON> destination = identifier()
+ <SUMMARY_TO> [name = identifier()] <COLON> destination = identifier()
{
psf = field.summaryFieldFor(name);
psf.addDestination(destination);
@@ -956,8 +959,8 @@ void weightedset(ParsedType fieldType) : { }
*/
void weightedsetBody(ParsedType type) : { }
{
- ( <CREATEIFNONEXISTENT> { type.setCreateIfNonExistent(true); }
- | <REMOVEIFZERO> { type.setRemoveIfZero(true); } )
+ ( <CREATE_IF_NONEXISTENT> { type.setCreateIfNonExistent(true); }
+ | <REMOVE_IF_ZERO> { type.setRemoveIfZero(true); } )
}
/**
@@ -971,7 +974,7 @@ void rankType(ParsedField field) :
String indexName = "";
}
{
- <RANKTYPE> [indexName = identifier()] <COLON> typeName = identifier()
+ <RANK_TYPE> [indexName = identifier()] <COLON> typeName = identifier()
{
field.addRankType(indexName, typeName);
}
@@ -1049,18 +1052,18 @@ void attributeSetting(ParsedAttribute attribute) :
}
{
(
- <FASTRANK> { attribute.setFastRank(true); }
- | <FASTSEARCH> { attribute.setFastSearch(true); }
- | <FASTACCESS> { attribute.setFastAccess(true); }
+ <FAST_RANK> { attribute.setFastRank(true); }
+ | <FAST_SEARCH> { attribute.setFastSearch(true); }
+ | <FAST_ACCESS> { attribute.setFastAccess(true); }
| <MUTABLE> { attribute.setMutable(true); }
| <PAGED> { attribute.setPaged(true); }
- | <ENABLEBITVECTORS> { deployLogger.logApplicationPackage(Level.WARNING, "'enable-bit-vectors' is deprecated and void -> remove it. Will be removed in vespa-9"); }
- | <ENABLEONLYBITVECTOR> { attribute.setEnableOnlyBitVector(true); }
+ | <ENABLE_BIT_VECTORS> { deployLogger.logApplicationPackage(Level.WARNING, "'enable-bit-vectors' is deprecated and void -> remove it. Will be removed in vespa-9"); }
+ | <ENABLE_ONLY_BIT_VECTOR> { attribute.setEnableOnlyBitVector(true); }
| attributeSorting(attribute)
| <ALIAS> { String alias; String aliasedName=attribute.name(); } [aliasedName = identifier()] <COLON> alias = identifierWithDash() {
attribute.addAlias(aliasedName, alias);
}
- | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); }
+ | <DISTANCE_METRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); }
)
}
@@ -1116,7 +1119,7 @@ void summaryInFieldShort(ParsedField field) :
psf = field.summaryFieldFor(name);
}
<COLON> ( <DYNAMIC> { psf.setDynamic(); }
- | <MATCHEDELEMENTSONLY> { psf.setMatchedElementsOnly(); }
+ | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); }
| (<FULL> | <STATIC>) { psf.setFull(); }
)
}
@@ -1161,7 +1164,7 @@ void summaryItem(ParsedSummaryField field) : { }
void summaryTransform(ParsedSummaryField field) : { }
{
( <DYNAMIC> { field.setDynamic(); }
- | <MATCHEDELEMENTSONLY> { field.setMatchedElementsOnly(); }
+ | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); }
| (<FULL> | <STATIC>) { field.setFull(); }
)
}
@@ -1295,7 +1298,7 @@ void queryCommand(ParsedField field) :
String command;
}
{
- <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() )
+ <QUERY_COMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() )
{
field.addQueryCommand(command);
}
@@ -1349,7 +1352,7 @@ void exactTerminator(ParsedMatchSettings matchInfo) :
String terminator;
}
{
- <EXACTTERMINATOR> <COLON> terminator = quotedString()
+ <EXACT_TERMINATOR> <COLON> terminator = quotedString()
{
matchInfo.setExactTerminator(terminator);
}
@@ -1360,7 +1363,7 @@ void gramSize(ParsedMatchSettings matchInfo) :
int gramSize;
}
{
- <GRAMSIZE> <COLON> gramSize = integer()
+ <GRAM_SIZE> <COLON> gramSize = integer()
{
matchInfo.setGramSize(gramSize);
}
@@ -1371,7 +1374,7 @@ void matchSize(ParsedMatchSettings matchInfo) :
int matchSize;
}
{
- <MAXLENGTH> <COLON> matchSize = integer() {
+ <MAX_LENGTH> <COLON> matchSize = integer() {
matchInfo.setMaxLength(matchSize);
}
}
@@ -1426,13 +1429,13 @@ void documentSummary(ParsedSchema schema) :
ParsedDocumentSummary summary;
}
{
- ( <DOCUMENTSUMMARY>
+ ( <DOCUMENT_SUMMARY>
name = identifierWithDash() { summary = new ParsedDocumentSummary(name); }
[inheritsDocumentSummary(summary)]
lbrace()
(
- <FROMDISK> { summary.setFromDisk(true); } |
- <OMITSUMMARYFEATURES> { summary.setOmitSummaryFeatures(true); } |
+ <FROM_DISK> { summary.setFromDisk(true); } |
+ <OMIT_SUMMARY_FEATURES> { summary.setOmitSummaryFeatures(true); } |
documentSummaryItem(summary) |
<NL>
)*
@@ -1529,9 +1532,9 @@ void indexBody(ParsedIndex index) :
| <ALIAS> <COLON> str = identifierWithDash() { index.addAlias(str); }
| <STEMMING> <COLON> str = identifierWithDash() { index.setStemming(Stemming.get(str)); }
| <ARITY> <COLON> arity = integer() { index.setArity(arity); }
- | <LOWERBOUND> <COLON> num = longValue() { index.setLowerBound(num); }
- | <UPPERBOUND> <COLON> num = longValue() { index.setUpperBound(num); }
- | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = floatValue() { index.setDensePostingListThreshold(threshold); }
+ | <LOWER_BOUND> <COLON> num = longValue() { index.setLowerBound(num); }
+ | <UPPER_BOUND> <COLON> num = longValue() { index.setUpperBound(num); }
+ | <DENSE_POSTING_LIST_THRESHOLD> <COLON> threshold = floatValue() { index.setDensePostingListThreshold(threshold); }
| <ENABLE_BM25> { index.setEnableBm25(true); }
| hnswIndex(index) { }
)
@@ -1556,9 +1559,9 @@ void hnswIndexBody(HnswIndexParams.Builder params) :
boolean bool;
}
{
- ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); }
- | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); }
- | <MULTITHREADEDINDEXING> <COLON> bool = bool() { params.setMultiThreadedIndexing(bool); } )
+ ( <MAX_LINKS_PER_NODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); }
+ | <NEIGHBORS_TO_EXPLORE_AT_INSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); }
+ | <MULTI_THREADED_INDEXING> <COLON> bool = bool() { params.setMultiThreadedIndexing(bool); } )
}
void onnxModelInSchema(ParsedSchema schema) :
@@ -1584,7 +1587,7 @@ OnnxModel onnxModel() :
OnnxModel onnxModel;
}
{
- ( <ONNXMODEL> name = identifier() { onnxModel = new OnnxModel(name); }
+ ( <ONNX_MODEL> name = identifier() { onnxModel = new OnnxModel(name); }
lbrace() (onnxModelItem(onnxModel) (<NL>)*)+ <RBRACE> )
{ return onnxModel; }
}
@@ -1603,9 +1606,11 @@ void onnxModelItem(OnnxModel onnxModel) :
(
(path = fileItem()) { onnxModel.setFileName(path); } |
(path = uriItem()) { onnxModel.setUri(path); } |
- <GPUDEVICE> <COLON> num = integer() { onnxModel.setGpuDevice(num, false); } |
- <INTRAOPTHREADS> <COLON> num = integer() { onnxModel.setStatelessIntraOpThreads(num); } |
- <INTEROPTHREADS> <COLON> num = integer() { onnxModel.setStatelessInterOpThreads(num); } |
+ <GPU_DEVICE> <COLON> num = integer() { onnxModel.setGpuDevice(num, false); } |
+ <INTRAOP_THREADS> <COLON> num = integer() { onnxModel.setStatelessIntraOpThreads(num); } |
+ <INTEROP_THREADS> <COLON> num = integer() { onnxModel.setStatelessInterOpThreads(num); } |
+ <EXECUTION_MODE> <COLON> ( <PARALLEL> { onnxModel.setStatelessExecutionMode("parallel"); }
+ | <SEQUENTIAL> { onnxModel.setStatelessExecutionMode("sequential"); } ) |
(<ONNX_INPUT_SL>) {
String name = token.image.substring(5, token.image.lastIndexOf(":")).trim();
if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); }
@@ -1680,7 +1685,7 @@ void rankProfile(ParsedSchema schema) :
ParsedRankProfile profile;
}
{
- ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash()
+ ( <MODEL> | <RANK_PROFILE> ) name = identifierWithDash()
{ profile = new ParsedRankProfile(name); }
[inheritsRankProfile(profile)]
lbrace() (rankProfileItem(schema, profile) (<NL>)*)* <RBRACE>
@@ -1819,7 +1824,7 @@ void matchPhase(ParsedRankProfile profile) :
MatchPhaseSettings settings = new MatchPhaseSettings();
}
{
- <MATCHPHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE>
+ <MATCH_PHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE>
{
settings.checkValid();
profile.setMatchPhaseSettings(settings);
@@ -1838,8 +1843,8 @@ void matchPhaseItem(MatchPhaseSettings settings) :
| diversity(settings)
| <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); }
| <DESCENDING> { settings.setAscending(false); } )
- | <MAXHITS> <COLON> num = integer() { settings.setMaxHits(num); }
- | <MAXFILTERCOVERAGE> <COLON> coverage = floatValue() { settings.setMaxFilterCoverage(coverage); }
+ | <MAX_HITS> <COLON> num = integer() { settings.setMaxHits(num); }
+ | <MAX_FILTER_COVERAGE> <COLON> coverage = floatValue() { settings.setMaxFilterCoverage(coverage); }
| <EVALUATION_POINT> <COLON> multiplier = floatValue() { settings.setEvaluationPoint(multiplier); }
| <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = floatValue() { settings.setPrePostFilterTippingPoint(multiplier); }
)
@@ -1890,7 +1895,7 @@ void firstPhase(ParsedRankProfile profile) :
String exp;
}
{
- <FIRSTPHASE> lbrace() (firstPhaseItem(profile) (<NL>)*)* <RBRACE>
+ <FIRST_PHASE> lbrace() (firstPhaseItem(profile) (<NL>)*)* <RBRACE>
}
void firstPhaseItem(ParsedRankProfile profile) :
@@ -1901,8 +1906,8 @@ void firstPhaseItem(ParsedRankProfile profile) :
}
{
( expression = expression() { profile.setFirstPhaseRanking(expression); }
- | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); }
- | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = floatValue()) { profile.setRankScoreDropLimit(dropLimit); }
+ | (<KEEP_RANK_COUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); }
+ | (<RANK_SCORE_DROP_LIMIT> <COLON> dropLimit = floatValue()) { profile.setRankScoreDropLimit(dropLimit); }
)
}
@@ -1913,7 +1918,7 @@ void firstPhaseItem(ParsedRankProfile profile) :
*/
void secondPhase(ParsedRankProfile profile) : { }
{
- <SECONDPHASE> lbrace() (secondPhaseItem(profile) (<NL>)*)* <RBRACE>
+ <SECOND_PHASE> lbrace() (secondPhaseItem(profile) (<NL>)*)* <RBRACE>
}
/**
@@ -1928,7 +1933,7 @@ void secondPhaseItem(ParsedRankProfile profile) :
}
{
( expression = expression() { profile.setSecondPhaseRanking(expression); }
- | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); }
+ | (<RERANK_COUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); }
)
}
@@ -1939,7 +1944,7 @@ void secondPhaseItem(ParsedRankProfile profile) :
*/
void globalPhase(ParsedRankProfile profile) : { }
{
- <GLOBALPHASE> lbrace() (globalPhaseItem(profile) (<NL>)*)* <RBRACE>
+ <GLOBAL_PHASE> lbrace() (globalPhaseItem(profile) (<NL>)*)* <RBRACE>
}
/**
@@ -1954,7 +1959,7 @@ void globalPhaseItem(ParsedRankProfile profile) :
}
{
( expression = expression() { profile.setGlobalPhaseExpression(expression); }
- | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setGlobalPhaseRerankCount(rerankCount); }
+ | (<RERANK_COUNT> <COLON> rerankCount = integer()) { profile.setGlobalPhaseRerankCount(rerankCount); }
)
}
@@ -2092,7 +2097,7 @@ void rankFeatures(ParsedRankProfile profile) :
*/
void ignoreRankFeatures(ParsedRankProfile profile) : { }
{
- <IGNOREDEFAULTRANKFEATURES> { profile.setIgnoreDefaultRankFeatures(true); }
+ <IGNORE_DEFAULT_RANK_FEATURES> { profile.setIgnoreDefaultRankFeatures(true); }
}
/**
@@ -2105,7 +2110,7 @@ void numThreadsPerSearch(ParsedRankProfile profile) :
int num;
}
{
- (<NUMTHREADSPERSEARCH> <COLON> num = integer()) { profile.setNumThreadsPerSearch(num); }
+ (<NUM_THREADS_PER_SEARCH> <COLON> num = integer()) { profile.setNumThreadsPerSearch(num); }
}
/**
@@ -2118,7 +2123,7 @@ void minHitsPerThread(ParsedRankProfile profile) :
int num;
}
{
- (<MINHITSPERTHREAD> <COLON> num = integer()) { profile.setMinHitsPerThread(num); }
+ (<MIN_HITS_PER_THREAD> <COLON> num = integer()) { profile.setMinHitsPerThread(num); }
}
/**
@@ -2131,7 +2136,7 @@ void numSearchPartitions(ParsedRankProfile profile) :
int num;
}
{
- (<NUMSEARCHPARTITIONS> <COLON> num = integer()) { profile.setNumSearchPartitions(num); }
+ (<NUM_SEARCH_PARTITIONS> <COLON> num = integer()) { profile.setNumSearchPartitions(num); }
}
/**
@@ -2144,7 +2149,7 @@ void termwiseLimit(ParsedRankProfile profile) :
double num;
}
{
- (<TERMWISELIMIT> <COLON> num = floatValue()) { profile.setTermwiseLimit(num); }
+ (<TERMWISE_LIMIT> <COLON> num = floatValue()) { profile.setTermwiseLimit(num); }
}
/**
@@ -2157,7 +2162,7 @@ void postFilterThreshold(ParsedRankProfile profile) :
double threshold;
}
{
- (<POSTFILTERTHRESHOLD> <COLON> threshold = floatValue()) { profile.setPostFilterThreshold(threshold); }
+ (<POST_FILTER_THRESHOLD> <COLON> threshold = floatValue()) { profile.setPostFilterThreshold(threshold); }
}
/**
@@ -2170,7 +2175,7 @@ void approximateThreshold(ParsedRankProfile profile) :
double threshold;
}
{
- (<APPROXIMATETHRESHOLD> <COLON> threshold = floatValue()) { profile.setApproximateThreshold(threshold); }
+ (<APPROXIMATE_THRESHOLD> <COLON> threshold = floatValue()) { profile.setApproximateThreshold(threshold); }
}
/**
@@ -2182,7 +2187,7 @@ void approximateThreshold(ParsedRankProfile profile) :
*/
void rankProperties(ParsedRankProfile profile) : { }
{
- <RANKPROPERTIES> lbrace() (LOOKAHEAD(rankPropertyItem() <COLON> rankPropertyItem() <NL>)
+ <RANK_PROPERTIES> lbrace() (LOOKAHEAD(rankPropertyItem() <COLON> rankPropertyItem() <NL>)
rankProperty(profile) (<NL>)+)* [rankProperty(profile)] <RBRACE>
}
@@ -2245,7 +2250,7 @@ void fieldRankType(ParsedRankProfile profile) :
String type;
}
{
- <RANKTYPE> name = identifier() <COLON> type = identifier()
+ <RANK_TYPE> name = identifier() <COLON> type = identifier()
{ profile.addFieldRankType(name, type); }
}
@@ -2566,7 +2571,68 @@ String identifierWithDash() :
{
( identifier = identifier() { return identifier; } )
|
- ( <IDENTIFIER_WITH_DASH> { return token.image; } )
+ ( <IDENTIFIER_WITH_DASH>
+ | <APPROXIMATE_THRESHOLD>
+ | <CREATE_IF_NONEXISTENT>
+ | <CUTOFF_FACTOR>
+ | <CUTOFF_STRATEGY>
+ | <DENSE_POSTING_LIST_THRESHOLD>
+ | <DISTANCE_METRIC>
+ | <DOCUMENT_SUMMARY>
+ | <ENABLE_BIT_VECTORS>
+ | <ENABLE_BM25>
+ | <ENABLE_ONLY_BIT_VECTOR>
+ | <EVALUATION_POINT>
+ | <EXACT_TERMINATOR>
+ | <EXECUTION_MODE>
+ | <FAST_ACCESS>
+ | <FAST_RANK>
+ | <FAST_SEARCH>
+ | <FIRST_PHASE>
+ | <FROM_DISK>
+ | <GLOBAL_PHASE>
+ | <GPU_DEVICE>
+ | <GRAM_SIZE>
+ | <IGNORE_DEFAULT_RANK_FEATURES>
+ | <INTEROP_THREADS>
+ | <INTRAOP_THREADS>
+ | <KEEP_RANK_COUNT>
+ | <LOWER_BOUND>
+ | <MATCHED_ELEMENTS_ONLY>
+ | <MATCH_PHASE>
+ | <MAX_FILTER_COVERAGE>
+ | <MAX_HITS>
+ | <MAX_LENGTH>
+ | <MAX_LINKS_PER_NODE>
+ | <MIN_GROUPS>
+ | <MIN_HITS_PER_THREAD>
+ | <MULTI_THREADED_INDEXING>
+ | <NEIGHBORS_TO_EXPLORE_AT_INSERT>
+ | <NUM_SEARCH_PARTITIONS>
+ | <NUM_THREADS_PER_SEARCH>
+ | <OMIT_SUMMARY_FEATURES>
+ | <ON_DEMAND>
+ | <ON_FIRST_PHASE>
+ | <ON_MATCH>
+ | <ONNX_MODEL>
+ | <ON_SECOND_PHASE>
+ | <ON_SUMMARY>
+ | <POST_FILTER_THRESHOLD>
+ | <PRE_POST_FILTER_TIPPING_POINT>
+ | <QUERY_COMMAND>
+ | <RANK_PROFILE>
+ | <RANK_PROPERTIES>
+ | <RANK_SCORE_DROP_LIMIT>
+ | <RANK_TYPE>
+ | <RAW_AS_BASE64_IN_SUMMARY>
+ | <REMOVE_IF_ZERO>
+ | <RERANK_COUNT>
+ | <SECOND_PHASE>
+ | <STRUCT_FIELD>
+ | <SUMMARY_TO>
+ | <TERMWISE_LIMIT>
+ | <UPPER_BOUND>
+ ) { return token.image; }
}
/**
@@ -2592,38 +2658,28 @@ String identifier() : { }
| <CONSTANT>
| <CONSTANTS>
| <CONTEXT>
- | <GLOBALPHASE>
- | <CREATEIFNONEXISTENT>
- | <DENSEPOSTINGLISTTHRESHOLD>
| <DESCENDING>
| <DICTIONARY>
| <DIRECT>
+ | <DIVERSITY>
| <DOCUMENT>
- | <DOCUMENTSUMMARY>
| <DOUBLE_KEYWORD>
| <DYNAMIC>
- | <ENABLEBITVECTORS>
- | <ENABLEONLYBITVECTOR>
| <EXACT>
- | <EXACTTERMINATOR>
| <FALSE>
- | <FASTACCESS>
- | <FASTRANK>
- | <FASTSEARCH>
| <FIELD>
| <FIELDS>
| <FIELDSET>
| <FILE>
| <FILTER>
- | <FIRSTPHASE>
| <FULL>
| <FUNCTION>
| <GRAM>
| <HASH>
+ | <HNSW>
| <ID>
| <IDENTICAL>
| <IDENTIFIER>
- | <IGNOREDEFAULTRANKFEATURES>
| <IMPORT>
| <INDEX>
| <INDEXING>
@@ -2631,52 +2687,42 @@ String identifier() : { }
| <INLINE>
| <INPUTS>
| <INTEGER>
- | <KEEPRANKCOUNT>
| <LITERAL>
| <LOCALE>
| <LONG>
| <LOOSE>
- | <LOWERBOUND>
| <LOWERCASE>
| <MACRO>
| <MAP>
| <MATCH>
- | <MATCHPHASE>
- | <MAXFILTERCOVERAGE>
- | <MAXHITS>
+ | <MODEL>
| <MTOKEN>
| <MUTABLE>
+ | <MUTATE>
| <NEVER>
| <NONE>
| <NORMAL>
| <NORMALIZING>
| <OFF>
| <ON>
- | <ONDEMAND>
+ | <OPERATION>
| <ORDER>
+ | <PAGED>
+ | <PARALLEL>
| <PREFIX>
| <PRIMARY>
| <PROPERTIES>
| <QUATERNARY>
| <QUERY>
- | <QUERYCOMMAND>
| <RANK>
- | <MODEL>
- | <RANKPROFILE>
- | <RANKPROPERTIES>
- | <RANKSCOREDROPLIMIT>
- | <RANKTYPE>
| <RAW>
| <REFERENCE>
- | <REMOVEIFZERO>
- | <RERANKCOUNT>
| <SCHEMA>
| <SEARCH>
| <SECONDARY>
- | <SECONDPHASE>
+ | <SEQUENTIAL>
| <SORTING>
| <SOURCE>
- | <PAGED>
| <SSCONTEXTUAL>
| <SSOVERRIDE>
| <SSTITLE>
@@ -2690,7 +2736,6 @@ String identifier() : { }
| <SUBSTRING>
| <SUFFIX>
| <SUMMARY>
- | <SUMMARYTO>
| <SYMMETRIC>
| <TERTIARY>
| <TEXT>
@@ -2700,7 +2745,6 @@ String identifier() : { }
| <UCA>
| <UNCASED>
| <URI>
- | <UPPERBOUND>
| <VARIABLE>
| <WEIGHT>
| <WEIGHTEDSET>
diff --git a/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg b/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg
index f705f896540..d63e85e2f19 100644
--- a/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg
+++ b/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg
@@ -25,7 +25,7 @@ model[].input[].source "rankingExpression(indirect_x)"
model[].output[].name "vector_Y"
model[].output[].as "foobar"
model[].dry_run_on_setup true
-model[].stateless_execution_mode ""
+model[].stateless_execution_mode "parallel"
model[].stateless_interop_threads 5
model[].stateless_intraop_threads 3
model[].gpu_device 2
diff --git a/config-model/src/test/derived/globalphase_onnx_inside/test.sd b/config-model/src/test/derived/globalphase_onnx_inside/test.sd
index cd3dfcea3d0..f7749e648e0 100644
--- a/config-model/src/test/derived/globalphase_onnx_inside/test.sd
+++ b/config-model/src/test/derived/globalphase_onnx_inside/test.sd
@@ -52,6 +52,7 @@ schema test {
intraop-threads: 3
interop-threads: 5
gpu-device: 2
+ execution-mode: parallel
}
first-phase {
expression: sum(attribute(aa))
diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg
index 77ac18e3261..5cda0a9fdc7 100644
--- a/config-model/src/test/derived/indexswitches/ilscripts.cfg
+++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg
@@ -4,7 +4,7 @@ ilscript[].doctype "indexswitches"
ilscript[].docfield[] "title"
ilscript[].docfield[] "descr"
ilscript[].docfield[] "source_src"
-ilscript[].content[] "clear_state | guard { input source_src | switch { case \"theweb\": input source_src | tokenize normalize | summary source | index source; case \"amg\": input source_src | tokenize normalize | summary source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }"
+ilscript[].content[] "clear_state | guard { input source_src | switch { case \"amg\": input source_src | tokenize normalize | summary source; case \"theweb\": input source_src | tokenize normalize | summary source | index source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }"
ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }"
ilscript[].content[] "clear_state | guard { input descr | tokenize normalize stem:\"BEST\" | summary descr | index descr; }"
ilscript[].content[] "input source_src | passthrough source_src"
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java
deleted file mode 100644
index f600900d3cd..00000000000
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.model.application.validation;
-
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.config.model.NullConfigModelRegistry;
-import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.config.model.deploy.TestProperties;
-import com.yahoo.config.model.test.MockApplicationPackage;
-import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.RegionName;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.model.VespaModel;
-import org.junit.jupiter.api.Test;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-
-import static com.yahoo.config.provision.Environment.prod;
-import static org.junit.jupiter.api.Assertions.*;
-
-
-/**
- * @author gjoranv
- */
-public class CloudWatchValidatorTest {
-
- @Test
- void cloudwatch_in_public_zones_passes_validation() throws IOException, SAXException {
- DeployState deployState = deployState(servicesWithCloudwatch(), true, true);
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new CloudWatchValidator().validate(model, deployState);
- }
-
- @Test
- void cloudwatch_passes_validation_for_self_hosted_vespa() throws IOException, SAXException {
- DeployState deployState = deployState(servicesWithCloudwatch(), false, false);
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new CloudWatchValidator().validate(model, deployState);
- }
-
- @Test
- void cloudwatch_in_non_public_zones_fails_validation() throws IOException, SAXException {
- DeployState deployState = deployState(servicesWithCloudwatch(), true, false);
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- try {
- new CloudWatchValidator().validate(model, deployState);
- fail();
- } catch (IllegalArgumentException e) {
- assertEquals("CloudWatch cannot be set up for non-public hosted Vespa and must be removed for consumers: [cloudwatch-consumer]",
- e.getMessage());
- }
- }
-
- private static DeployState deployState(String servicesXml, boolean isHosted, boolean isPublic) {
- ApplicationPackage app = new MockApplicationPackage.Builder()
- .withServices(servicesXml)
- .build();
-
- DeployState.Builder builder = new DeployState.Builder()
- .applicationPackage(app)
- .properties(new TestProperties().setHostedVespa(isHosted));
- if (isHosted) {
- var system = isPublic ? SystemName.Public : SystemName.main;
- builder.zone(new Zone(system, Environment.prod, RegionName.from("foo")));
- }
- final DeployState deployState = builder.build();
-
- if (isHosted) {
- assertTrue(deployState.isHosted(), "Test must emulate a hosted deployment.");
- assertEquals(prod, deployState.zone().environment(), "Test must emulate a prod environment.");
- }
- return deployState;
- }
-
- private String servicesWithCloudwatch() {
- return String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='cloudwatch-consumer'>",
- " <metric id='my-metric'/>",
- " <cloudwatch region='us-east-1' namespace='my-namespace' >",
- " <credentials access-key-name='my-access-key' ",
- " secret-key-name='my-secret-key' />",
- " </cloudwatch>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- }
-
-}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
index 5c360a9343f..65dfce8ff6c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java
@@ -1,14 +1,18 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.application.validation.change;
+import com.yahoo.collections.Pair;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.model.api.ConfigChangeAction;
import com.yahoo.config.provision.Environment;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.application.validation.ValidationTester;
import com.yahoo.yolean.Exceptions;
import org.junit.jupiter.api.Test;
+import java.util.List;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
@@ -36,7 +40,12 @@ public class ContentClusterRemovalValidatorTest {
@Test
void testOverridingContentRemovalValidation() {
VespaModel previous = tester.deploy(null, getServices("contentClusterId"), Environment.prod, null).getFirst();
- tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, removalOverride); // Allowed due to override
+ var result = tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, removalOverride); // Allowed due to override
+ assertEquals(result.getFirst().getContainerClusters().values().stream()
+ .flatMap(cluster -> cluster.getContainers().stream())
+ .map(container -> container.getServiceInfo())
+ .toList(),
+ result.getSecond().stream().flatMap(action -> action.getServices().stream()).toList());
}
private static String getServices(String contentClusterId) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index b2175014cd4..488ad9f8484 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1099,6 +1099,30 @@ public class ContentClusterTest extends ContentBaseTest {
}
}
+ private void verifySummaryDecodeType(String policy, DispatchConfig.SummaryDecodePolicy.Enum expected) {
+ TestProperties properties = new TestProperties();
+ if (policy != null) {
+ properties.setSummaryDecodePolicy(policy);
+ }
+ VespaModel model = createEnd2EndOneNode(properties);
+
+ ContentCluster cc = model.getContentClusters().get("storage");
+ DispatchConfig.Builder builder = new DispatchConfig.Builder();
+ cc.getSearch().getConfig(builder);
+
+ DispatchConfig cfg = new DispatchConfig(builder);
+ assertEquals(expected, cfg.summaryDecodePolicy());
+ }
+
+ @Test
+ public void verify_summary_decoding_controlled_by_properties() {
+ verifySummaryDecodeType(null, DispatchConfig.SummaryDecodePolicy.EAGER);
+ verifySummaryDecodeType("illegal-config", DispatchConfig.SummaryDecodePolicy.EAGER);
+ verifySummaryDecodeType("eager", DispatchConfig.SummaryDecodePolicy.EAGER);
+ verifySummaryDecodeType("ondemand", DispatchConfig.SummaryDecodePolicy.ONDEMAND);
+ verifySummaryDecodeType("on-demand", DispatchConfig.SummaryDecodePolicy.ONDEMAND);
+ }
+
private int resolveMaxCompactBuffers(OptionalInt maxCompactBuffers) {
TestProperties testProperties = new TestProperties();
if (maxCompactBuffers.isPresent()) {
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
index 5d5775275c3..17025b10568 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java
@@ -23,7 +23,9 @@ import java.util.stream.Collectors;
import static java.nio.file.Files.readAttributes;
/**
- * Deletes file references and url downloads that have not been used for some time
+ * Deletes file references and url downloads that have not been used for some time.
+ * See {@link com.yahoo.vespa.config.proxy.filedistribution.RequestTracker} for how we track
+ * when a file reference or download was last used.
*
* @author hmusum
*/
@@ -32,7 +34,7 @@ class FileReferencesAndDownloadsMaintainer implements Runnable {
private static final Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName());
private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir;
private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory;
- private static final Duration defaultDurationToKeepFiles = Duration.ofDays(14);
+ private static final Duration defaultDurationToKeepFiles = Duration.ofDays(21);
private static final Duration interval = Duration.ofMinutes(1);
private final ScheduledExecutorService executor =
diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def
index 936b1400053..01fc5d48dfa 100644
--- a/configdefinitions/src/vespa/dispatch.def
+++ b/configdefinitions/src/vespa/dispatch.def
@@ -3,11 +3,11 @@
namespace=vespa.config.search
-# The active docs a group must have as a % of the average active docs of all other groups,
-# for that group to be included in queries
+## The active docs a group must have as a % of the average active docs of all other groups,
+## for that group to be included in queries
minActivedocsPercentage double default=97.0
-# Distribution policy for group selection
+## Distribution policy for group selection
distributionPolicy enum { ROUNDROBIN, BEST_OF_RANDOM_2, LATENCY_AMORTIZED_OVER_REQUESTS, LATENCY_AMORTIZED_OVER_TIME, ADAPTIVE } default=ADAPTIVE
## Maximum number of hits that will be requested from a single node
@@ -26,24 +26,28 @@ maxHitsPerNode int default=2147483647
## a setting of 1.0. This is a significant optimisation with with very little loss in presicion.
topKProbability double default=0.9999
-# Number of document replicas _per group_ that will be present in a stable cluster.
-# Should always be >= searchableCopies.
+## Number of document replicas _per group_ that will be present in a stable cluster.
+## Should always be >= searchableCopies.
redundancy long default=1
-# Minimum search coverage required before returning the results of a query
+## Minimum search coverage required before returning the results of a query
minSearchCoverage double default=100
-# Minimum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage
+## Minimum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage
minWaitAfterCoverageFactor double default=0
-# Maximum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage
+## Maximum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage
maxWaitAfterCoverageFactor double default=1
-# Number of JRT transport threads
+## Number of JRT transport threads
numJrtTransportThreads int default=8
-# Number of JRT connections per backend node
+## Number of JRT connections per backend node
numJrtConnectionsPerNode int default=8
-# Number of seconds to spend warming up code to prevent JIT cold start issues.
+## Number of seconds to spend warming up code to prevent JIT cold start issues.
warmuptime double default=0.1
+
+## Specifies how summary data are decoded
+## Eager will build java objects immediately, while ONDEMAND will do so when it is needed
+summaryDecodePolicy enum {EAGER, ONDEMAND} default=EAGER
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index f6988a6b566..955b1bc8f4f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -534,7 +534,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
NestedTransaction transaction = new NestedTransaction();
Optional<ApplicationTransaction> applicationTransaction = hostProvisioner.map(provisioner -> provisioner.lock(applicationId))
.map(lock -> new ApplicationTransaction(lock, transaction));
- try (var applicationLock = tenantApplications.lock(applicationId)) {
+ try (@SuppressWarnings("unused") var applicationLock = tenantApplications.lock(applicationId)) {
Optional<Long> activeSession = tenantApplications.activeSessionOf(applicationId);
CompletionWaiter waiter;
if (activeSession.isPresent()) {
@@ -796,7 +796,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
NestedTransaction transaction = new NestedTransaction();
Optional<ApplicationTransaction> applicationTransaction = hostProvisioner.map(provisioner -> provisioner.lock(applicationId))
.map(lock -> new ApplicationTransaction(lock, transaction));
- try (var sessionLock = tenant.getApplicationRepo().lock(applicationId)) {
+ try (@SuppressWarnings("unused") var sessionLock = tenant.getApplicationRepo().lock(applicationId)) {
Optional<Session> activeSession = getActiveSession(applicationId);
var sessionZooKeeperClient = tenant.getSessionRepository().createSessionZooKeeperClient(session.getSessionId());
CompletionWaiter waiter = sessionZooKeeperClient.createActiveWaiter();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
index 88e3134ccad..cddcb0f316d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
@@ -74,7 +74,7 @@ public class TenantApplications implements RequestHandler, HostValidator {
private final MetricUpdater tenantMetricUpdater;
private final Clock clock;
private final TenantFileSystemDirs tenantFileSystemDirs;
- private final ConfigserverConfig configserverConfig;
+ private final String serverId;
private final ListFlag<String> incompatibleVersions;
public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor,
@@ -95,7 +95,7 @@ public class TenantApplications implements RequestHandler, HostValidator {
this.hostRegistry = hostRegistry;
this.tenantFileSystemDirs = tenantFileSystemDirs;
this.clock = clock;
- this.configserverConfig = configserverConfig;
+ this.serverId = configserverConfig.serverId();
this.incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource);
}
@@ -230,7 +230,7 @@ public class TenantApplications implements RequestHandler, HostValidator {
*/
public void activateApplication(ApplicationSet applicationSet, long activeSessionId) {
ApplicationId id = applicationSet.getId();
- try (Lock lock = lock(id)) {
+ try (@SuppressWarnings("unused") Lock lock = lock(id)) {
if ( ! exists(id))
return; // Application was deleted before activation.
if (applicationSet.getApplicationGeneration() != activeSessionId)
@@ -269,7 +269,7 @@ public class TenantApplications implements RequestHandler, HostValidator {
public void removeApplicationsExcept(Set<ApplicationId> applications) {
for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) {
if ( ! applications.contains(activeApplication)) {
- try (var applicationLock = lock(activeApplication)){
+ try (@SuppressWarnings("unused") var applicationLock = lock(activeApplication)){
removeApplication(activeApplication);
}
}
@@ -404,11 +404,11 @@ public class TenantApplications implements RequestHandler, HostValidator {
public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; }
public CompletionWaiter createRemoveApplicationWaiter(ApplicationId applicationId) {
- return RemoveApplicationWaiter.createAndInitialize(curator, applicationId, configserverConfig.serverId());
+ return RemoveApplicationWaiter.createAndInitialize(curator, applicationId, serverId);
}
public CompletionWaiter getRemoveApplicationWaiter(ApplicationId applicationId) {
- return RemoveApplicationWaiter.create(curator, applicationId, configserverConfig.serverId());
+ return RemoveApplicationWaiter.create(curator, applicationId, serverId);
}
/**
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index ad5423f0a94..7a2377594a1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -203,6 +203,7 @@ public class ModelContextImpl implements ModelContext {
private final boolean useRestrictedDataPlaneBindings;
private final int heapPercentage;
private final boolean enableGlobalPhase;
+ private final String summaryDecodePolicy;
public FeatureFlags(FlagSource source, ApplicationId appId, Version version) {
this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -248,11 +249,13 @@ public class ModelContextImpl implements ModelContext {
this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS);
this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE);
this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE);
+ this.summaryDecodePolicy = flagValue(source, appId, version, Flags.SUMMARY_DECODE_POLICY);
}
@Override public int heapSizePercentage() { return heapPercentage; }
@Override public String queryDispatchPolicy() { return queryDispatchPolicy; }
@Override public double queryDispatchWarmup() { return queryDispatchWarmup; }
+ @Override public String summaryDecodePolicy() { return summaryDecodePolicy; }
@Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
@Override public String feedSequencerType() { return feedSequencer; }
@Override public String responseSequencerType() { return responseSequencer; }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 86c0c90ca12..62a1704b350 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -97,8 +97,6 @@ public class ApplicationHandler extends HttpHandler {
if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request);
if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request);
if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path));
- // TODO: Remove when all usage has migrated to .../metrics/searchnode
- if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return searchNodeMetrics(applicationId(path));
if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/searchnode")) return searchNodeMetrics(applicationId(path));
if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path));
if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
index 2948b82dd96..22ef6cc2547 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
@@ -88,7 +88,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
createLocalSessionIfMissing(applicationId, sessionId);
}
}
- return asSuccessFactor(attempts, failures);
+ return asSuccessFactorDeviation(attempts, failures);
}
private static FileDownloader createFileDownloader(List<String> otherConfigServersInCluster,
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
index dad687aae67..ef48c3bdb98 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
@@ -46,9 +46,9 @@ public abstract class ConfigServerMaintainer extends Maintainer {
}
@Override
- public void completed(String job, double successFactor, long durationMs) {
+ public void completed(String job, double successFactorDeviation, long durationMs) {
var context = metric.createContext(Map.of("job", job));
- metric.set("maintenance.successFactor", successFactor, context);
+ metric.set("maintenance.successFactorDeviation", successFactorDeviation, context);
metric.set("maintenance.duration", durationMs, context);
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
index a9672455d09..e81c9b6bcbd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
@@ -77,7 +77,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
}
});
}
- return asSuccessFactor(attempts.get(), failures.get());
+ return asSuccessFactorDeviation(attempts.get(), failures.get());
}
private Supplier<Long> lazyGeneration(Application application) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java
index 728f3e8510f..2ad04fdd572 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java
@@ -12,8 +12,8 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.TenantName;
import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.ConfigKey;
-import com.yahoo.vespa.config.server.ConfigServerDB;
import com.yahoo.vespa.config.server.ConfigActivationListener;
+import com.yahoo.vespa.config.server.ConfigServerDB;
import com.yahoo.vespa.config.server.ServerCache;
import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs;
import com.yahoo.vespa.config.server.host.HostRegistry;
@@ -26,7 +26,6 @@ import com.yahoo.vespa.config.server.tenant.TestTenantRepository;
import com.yahoo.vespa.curator.CompletionTimeoutException;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
-import com.yahoo.vespa.curator.mock.MockCuratorFramework;
import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.model.VespaModel;
@@ -37,22 +36,17 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.xml.sax.SAXException;
-
import java.io.File;
import java.io.IOException;
import java.time.Clock;
import java.time.Duration;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.IntStream;
-import static com.yahoo.vespa.config.server.application.TenantApplications.RemoveApplicationWaiter;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -195,7 +189,6 @@ public class TenantApplicationsTest {
public static class MockConfigActivationListener implements ConfigActivationListener {
public final AtomicInteger activated = new AtomicInteger(0);
final AtomicInteger removed = new AtomicInteger(0);
- final Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>();
@Override
public void configActivated(ApplicationSet application) {
@@ -247,22 +240,22 @@ public class TenantApplicationsTest {
@Test
public void testRemoveApplication2of3Respond() throws InterruptedException {
- Curator curator = new MockCurator3ConfigServers();
- Thread t1 = setupWaiter(curator);
- notifyCompletion(curator, 2);
+ TenantApplications applications = createZKAppRepo(new InMemoryFlagSource());
+ Thread t1 = setupWaiter(applications);
+ notifyCompletion(applications, 2);
t1.join();
}
@Test
public void testRemoveApplicationAllRespond() throws InterruptedException {
- Curator curator = new MockCurator3ConfigServers();
- Thread t1 = setupWaiter(curator);
- notifyCompletion(curator, 3);
+ TenantApplications applications = createZKAppRepo(new InMemoryFlagSource());
+ Thread t1 = setupWaiter(applications);
+ notifyCompletion(applications, 3);
t1.join();
}
- private Thread setupWaiter(Curator curator) {
- Curator.CompletionWaiter waiter = RemoveApplicationWaiter.createAndInitialize(curator, createApplicationId(), "cfg1", Duration.ofSeconds(1));
+ private Thread setupWaiter(TenantApplications applications) {
+ Curator.CompletionWaiter waiter = applications.getRemoveApplicationWaiter(createApplicationId());
Thread t1 = new Thread(() -> {
try {
waiter.awaitCompletion(Duration.ofSeconds(120));
@@ -274,10 +267,10 @@ public class TenantApplicationsTest {
return t1;
}
- private void notifyCompletion(Curator curator, int respondentCount) {
+ private void notifyCompletion(TenantApplications applications, int respondentCount) {
IntStream.range(0, respondentCount)
- .forEach(i -> RemoveApplicationWaiter.create(curator, createApplicationId(), "cfg" + i, Duration.ofSeconds(1))
- .notifyCompletion());
+ .forEach(i -> applications.createRemoveApplicationWaiter(createApplicationId())
+ .notifyCompletion());
}
private TenantApplications createZKAppRepo() {
@@ -332,12 +325,4 @@ public class TenantApplicationsTest {
flagSource);
}
- private static class MockCurator3ConfigServers extends Curator {
-
- public MockCurator3ConfigServers() {
- super("host1:2181,host2:2181,host3:2181", "host1:2181,host2:2181,host3:2181", (retryPolicy) -> new MockCuratorFramework(true, false));
- }
-
- }
-
}
diff --git a/container-core/src/main/java/com/yahoo/processing/Request.java b/container-core/src/main/java/com/yahoo/processing/Request.java
index cff546f1fd4..313df855cd4 100644
--- a/container-core/src/main/java/com/yahoo/processing/Request.java
+++ b/container-core/src/main/java/com/yahoo/processing/Request.java
@@ -30,13 +30,13 @@ public class Request extends FreezableClass implements Cloneable {
* The name of the chain of Processor instances which will be invoked when
* executing a request.
*/
- public static final CompoundName CHAIN = new CompoundName("chain");
+ public static final CompoundName CHAIN = CompoundName.from("chain");
/**
* The name of the request property used in the processing framework to
* store the incoming JDisc request.
*/
- public static final CompoundName JDISC_REQUEST = new CompoundName("jdisc.request");
+ public static final CompoundName JDISC_REQUEST = CompoundName.from("jdisc.request");
/**
* Creates a request with no properties
diff --git a/container-core/src/main/java/com/yahoo/processing/Response.java b/container-core/src/main/java/com/yahoo/processing/Response.java
index 59533900a0c..9f87f42ba65 100644
--- a/container-core/src/main/java/com/yahoo/processing/Response.java
+++ b/container-core/src/main/java/com/yahoo/processing/Response.java
@@ -35,7 +35,7 @@ import java.util.concurrent.TimeoutException;
*/
public class Response extends ListenableFreezableClass {
- private final static CompoundName freezeListenerKey =new CompoundName("processing.freezeListener");
+ private final static CompoundName freezeListenerKey = CompoundName.from("processing.freezeListener");
private final DataList<?> data;
diff --git a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java
index f7aea8abbd1..837f356d4d9 100644
--- a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java
+++ b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java
@@ -46,7 +46,7 @@ import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry;
@SuppressWarnings("deprecation") // super class is deprecated
public abstract class AbstractProcessingHandler<COMPONENT extends Processor> extends LoggingRequestHandler {
- private final static CompoundName freezeListenerKey =new CompoundName("processing.freezeListener");
+ private final static CompoundName freezeListenerKey = CompoundName.from("processing.freezeListener");
public final static String DEFAULT_RENDERER_ID = "default";
@@ -112,7 +112,7 @@ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> ext
public HttpResponse handle(HttpRequest request, ContentChannel channel) {
com.yahoo.processing.Request processingRequest = new com.yahoo.processing.Request();
populate("", request.propertyMap(), processingRequest.properties());
- populate("context", request.getJDiscRequest().context(), processingRequest.properties());
+ populate("context.", request.getJDiscRequest().context(), processingRequest.properties());
processingRequest.properties().set(Request.JDISC_REQUEST, request);
FreezeListener freezeListener = new FreezeListener(processingRequest, renderers, defaultRenderer, channel, renderingExecutor);
@@ -183,10 +183,9 @@ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> ext
return properties.getString(Request.CHAIN,"default");
}
- private void populate(String prefixName,Map<String,?> parameters,Properties properties) {
- CompoundName prefix = new CompoundName(prefixName);
+ private void populate(String prefixName, Map<String,?> parameters,Properties properties) {
for (Map.Entry<String,?> entry : parameters.entrySet())
- properties.set(prefix.append(entry.getKey()), entry.getValue());
+ properties.set(CompoundName.from(prefixName + entry.getKey()), entry.getValue());
}
private static class FreezeListener implements Runnable, ResponseReceiver {
diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java
index ac7b3d24d08..0edff9162b5 100644
--- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java
+++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java
@@ -21,9 +21,12 @@ import static com.yahoo.text.Lowercase.toLowerCase;
*/
public final class CompoundName {
- /**
- * The string name of this compound.
- */
+ private static final int MAX_CACHE_SIZE = 10_000;
+ private static final Map<String, CompoundName> cache = new CopyOnWriteHashMap<>();
+ /** The empty compound */
+ public static final CompoundName empty = CompoundName.from("");
+
+ /* The string name of this compound. */
private final String name;
private final String lowerCasedName;
@@ -37,17 +40,16 @@ public final class CompoundName {
/** This name with the last component removed */
private final CompoundName first;
- /** The empty compound */
- public static final CompoundName empty = new CompoundName("");
- private static final Map<String, CompoundName> cache = new CopyOnWriteHashMap<>();
- private static final int MAX_CACHE_SIZE = 10_000;
/**
* Constructs this from a string which may contains dot-separated components
*
* @throws NullPointerException if name is null
*/
public CompoundName(String name) {
- this(name, parse(name).toArray(new String[1]));
+ this(name, false);
+ }
+ private CompoundName(String name, boolean useCache) {
+ this(name, parse(name).toArray(new String[0]), useCache);
}
/** Constructs this from an array of name components which are assumed not to contain dots */
@@ -61,7 +63,7 @@ public final class CompoundName {
}
private CompoundName(String [] compounds) {
- this(toCompoundString(compounds), compounds);
+ this(toCompoundString(compounds), compounds, false);
}
/**
@@ -71,7 +73,7 @@ public final class CompoundName {
* @param name the string representation of the compounds
* @param compounds the compounds of this name
*/
- private CompoundName(String name, String [] compounds) {
+ private CompoundName(String name, String [] compounds, boolean useCache) {
if (name == null) throw new NullPointerException("Name can not be null");
this.name = name;
@@ -86,12 +88,27 @@ public final class CompoundName {
this.compounds = new ImmutableArrayList(compounds);
this.hashCode = this.compounds.hashCode();
- rest = (compounds.length > 1)
- ? new CompoundName(name.substring(compounds[0].length()+1), Arrays.copyOfRange(compounds, 1, compounds.length))
- : empty;
- first = (compounds.length > 1)
- ? new CompoundName(name.substring(0, name.length() - (compounds[compounds.length-1].length()+1)), Arrays.copyOfRange(compounds, 0, compounds.length-1))
- : empty;
+ if (compounds.length > 1) {
+ String restName = name.substring(compounds[0].length()+1);
+ if (useCache) {
+ rest = cache.computeIfAbsent(restName, (key) -> new CompoundName(key, Arrays.copyOfRange(compounds, 1, compounds.length), useCache));
+ } else {
+ rest = new CompoundName(restName, Arrays.copyOfRange(compounds, 1, compounds.length), useCache);
+ }
+ } else {
+ rest = empty;
+ }
+
+ if (compounds.length > 1) {
+ String firstName = name.substring(0, name.length() - (compounds[compounds.length-1].length()+1));
+ if (useCache) {
+ first = cache.computeIfAbsent(firstName, (key) -> new CompoundName(key, Arrays.copyOfRange(compounds, 0, compounds.length-1), useCache));
+ } else {
+ first = new CompoundName(firstName, Arrays.copyOfRange(compounds, 0, compounds.length-1), useCache);
+ }
+ } else {
+ first = empty;
+ }
}
private static List<String> parse(String s) {
@@ -138,7 +155,7 @@ public final class CompoundName {
int count = 0;
for (String s : compounds) { newCompounds[count++] = s; }
for (String s : name.compounds) { newCompounds[count++] = s; }
- return new CompoundName(concat(this.name, name.name), newCompounds);
+ return new CompoundName(concat(this.name, name.name), newCompounds, false);
}
private static String concat(String name1, String name2) {
@@ -310,11 +327,12 @@ public final class CompoundName {
CompoundName found = cache.get(name);
if (found != null) return found;
- CompoundName compound = new CompoundName(name);
if (cache.size() < MAX_CACHE_SIZE) {
+ CompoundName compound = new CompoundName(name, true);
cache.put(name, compound);
+ return compound;
}
- return compound;
+ return new CompoundName(name, false);
}
private static class ImmutableArrayList extends AbstractList<String> {
diff --git a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java
index 4bbece0af29..eccc4dd8842 100644
--- a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java
+++ b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java
@@ -18,59 +18,61 @@ import static org.junit.jupiter.api.Assertions.*;
public class CompoundNameTestCase {
private static final String NAME = "com.yahoo.processing.request.CompoundNameTestCase";
- private final CompoundName cn = new CompoundName(NAME);
+ private static final CompoundName C_NAME = CompoundName.from(NAME);
+ private static final CompoundName C_abcde = CompoundName.from("a.b.c.d.e");
void verifyStrict(CompoundName expected, CompoundName actual) {
assertEquals(expected, actual);
assertEquals(expected.asList(), actual.asList());
}
void verifyStrict(String expected, CompoundName actual) {
- verifyStrict(new CompoundName(expected), actual);
+ verifyStrict(CompoundName.from(expected), actual);
}
@Test
final void testLast() {
- assertEquals(NAME.substring(NAME.lastIndexOf('.') + 1), cn.last());
+ assertEquals(NAME.substring(NAME.lastIndexOf('.') + 1), C_NAME.last());
}
@Test
final void testFirst() {
- assertEquals(NAME.substring(0, NAME.indexOf('.')), cn.first());
+ assertEquals(NAME.substring(0, NAME.indexOf('.')), C_NAME.first());
}
@Test
final void testRest() {
- verifyStrict(NAME.substring(NAME.indexOf('.') + 1), cn.rest());
+ verifyStrict(NAME.substring(NAME.indexOf('.') + 1), C_NAME.rest());
}
@Test
final void testRestN() {
- verifyStrict("a.b.c.d.e", new CompoundName("a.b.c.d.e").rest(0));
- verifyStrict("b.c.d.e", new CompoundName("a.b.c.d.e").rest(1));
- verifyStrict("c.d.e", new CompoundName("a.b.c.d.e").rest(2));
- verifyStrict("d.e", new CompoundName("a.b.c.d.e").rest(3));
- verifyStrict("e", new CompoundName("a.b.c.d.e").rest(4));
- verifyStrict(CompoundName.empty, new CompoundName("a.b.c.d.e").rest(5));
+ verifyStrict("a.b.c.d.e", C_abcde.rest(0));
+ verifyStrict("b.c.d.e", C_abcde.rest(1));
+ verifyStrict("c.d.e", C_abcde.rest(2));
+ verifyStrict("d.e", C_abcde.rest(3));
+ verifyStrict("e", C_abcde.rest(4));
+ verifyStrict(CompoundName.empty, C_abcde.rest(5));
}
@Test
final void testFirstN() {
- verifyStrict("a.b.c.d.e", new CompoundName("a.b.c.d.e").first(5));
- verifyStrict("a.b.c.d", new CompoundName("a.b.c.d.e").first(4));
- verifyStrict("a.b.c", new CompoundName("a.b.c.d.e").first(3));
- verifyStrict("a.b", new CompoundName("a.b.c.d.e").first(2));
- verifyStrict("a", new CompoundName("a.b.c.d.e").first(1));
- verifyStrict(CompoundName.empty, new CompoundName("a.b.c.d.e").first(0));
+ verifyStrict("a.b.c.d.e", C_abcde.first(5));
+ verifyStrict("a.b.c.d", C_abcde.first(4));
+ verifyStrict("a.b.c", C_abcde.first(3));
+ verifyStrict("a.b", C_abcde.first(2));
+ verifyStrict("a", C_abcde.first(1));
+ verifyStrict(CompoundName.empty, C_abcde.first(0));
}
@Test
final void testPrefix() {
- assertTrue(new CompoundName("a.b.c").hasPrefix(new CompoundName("")));
- assertTrue(new CompoundName("a.b.c").hasPrefix(new CompoundName("a")));
- assertTrue(new CompoundName("a.b.c").hasPrefix(new CompoundName("a.b")));
- assertTrue(new CompoundName("a.b.c").hasPrefix(new CompoundName("a.b.c")));
+ CompoundName abc = CompoundName.from("a.b.c");
+ assertTrue(abc.hasPrefix(CompoundName.empty));
+ assertTrue(abc.hasPrefix(CompoundName.from("a")));
+ assertTrue(abc.hasPrefix(CompoundName.from("a.b")));
+ assertTrue(abc.hasPrefix(CompoundName.from("a.b.c")));
- assertFalse(new CompoundName("a.b.c").hasPrefix(new CompoundName("a.b.c.d")));
- assertFalse(new CompoundName("a.b.c").hasPrefix(new CompoundName("a.b.d")));
+ assertFalse(abc.hasPrefix(CompoundName.from("a.b.c.d")));
+ assertFalse(abc.hasPrefix(CompoundName.from("a.b.d")));
}
@Test
@@ -87,28 +89,28 @@ public class CompoundNameTestCase {
for (@SuppressWarnings("unused") String x : i) {
++n;
}
- assertEquals(n, cn.size());
+ assertEquals(n, C_NAME.size());
}
@Test
final void testGet() {
- String s = cn.get(0);
+ String s = C_NAME.get(0);
assertEquals(NAME.substring(0, NAME.indexOf('.')), s);
}
@Test
final void testIsCompound() {
- assertTrue(cn.isCompound());
+ assertTrue(C_NAME.isCompound());
}
@Test
final void testIsEmpty() {
- assertFalse(cn.isEmpty());
+ assertFalse(C_NAME.isEmpty());
}
@Test
final void testAsList() {
- List<String> l = cn.asList();
+ List<String> l = C_NAME.asList();
Splitter peoplesFront = Splitter.on('.');
Iterable<String> answer = peoplesFront.split(NAME);
Iterator<String> expected = answer.iterator();
@@ -120,35 +122,35 @@ public class CompoundNameTestCase {
@Test
final void testEqualsObject() {
- assertNotEquals(cn, NAME);
- assertNotEquals(cn, null);
- verifyStrict(cn, cn);
- verifyStrict(cn, new CompoundName(NAME));
+ assertNotEquals(C_NAME, NAME);
+ assertNotEquals(C_NAME, null);
+ verifyStrict(C_NAME, C_NAME);
+ verifyStrict(C_NAME, CompoundName.from(NAME));
}
@Test
final void testEmptyNonEmpty() {
- assertTrue(new CompoundName("").isEmpty());
- assertEquals(0, new CompoundName("").size());
- assertFalse(new CompoundName("a").isEmpty());
- assertEquals(1, new CompoundName("a").size());
- CompoundName empty = new CompoundName("a.b.c");
+ assertTrue(CompoundName.empty.isEmpty());
+ assertEquals(0, CompoundName.empty.size());
+ assertFalse(CompoundName.from("a").isEmpty());
+ assertEquals(1, CompoundName.from("a").size());
+ CompoundName empty = CompoundName.from("a.b.c");
verifyStrict(empty, empty.rest(0));
assertNotEquals(empty, empty.rest(1));
}
@Test
final void testGetLowerCasedName() {
- assertEquals(Lowercase.toLowerCase(NAME), cn.getLowerCasedName());
+ assertEquals(Lowercase.toLowerCase(NAME), C_NAME.getLowerCasedName());
}
@Test
void testAppendCompound() {
- verifyStrict("a.b.c.d", new CompoundName("").append(new CompoundName("a.b.c.d")));
- verifyStrict("a.b.c.d", new CompoundName("a").append(new CompoundName("b.c.d")));
- verifyStrict("a.b.c.d", new CompoundName("a.b").append(new CompoundName("c.d")));
- verifyStrict("a.b.c.d", new CompoundName("a.b.c").append(new CompoundName("d")));
- verifyStrict("a.b.c.d", new CompoundName("a.b.c.d").append(new CompoundName("")));
+ verifyStrict("a.b.c.d", CompoundName.empty.append(CompoundName.from("a.b.c.d")));
+ verifyStrict("a.b.c.d", CompoundName.from("a").append(CompoundName.from("b.c.d")));
+ verifyStrict("a.b.c.d", CompoundName.from("a.b").append(CompoundName.from("c.d")));
+ verifyStrict("a.b.c.d", CompoundName.from("a.b.c").append(CompoundName.from("d")));
+ verifyStrict("a.b.c.d", CompoundName.from("a.b.c.d").append(CompoundName.empty));
}
@Test
@@ -156,13 +158,13 @@ public class CompoundNameTestCase {
CompoundName empty = new CompoundName("");
assertTrue(empty.hasPrefix(empty));
- assertTrue(new CompoundName("a").hasPrefix(empty));
+ assertTrue(CompoundName.from("a").hasPrefix(empty));
}
@Test
void whole_components_must_match_to_be_prefix() {
- CompoundName stringPrefix = new CompoundName("a");
- CompoundName name = new CompoundName("aa");
+ CompoundName stringPrefix = CompoundName.from("a");
+ CompoundName name = CompoundName.from("aa");
assertFalse(name.hasPrefix(stringPrefix));
}
@@ -171,7 +173,7 @@ public class CompoundNameTestCase {
void testFirstRest() {
verifyStrict(CompoundName.empty, CompoundName.empty.rest());
- CompoundName n = new CompoundName("on.two.three");
+ CompoundName n = CompoundName.from("on.two.three");
assertEquals("on", n.first());
verifyStrict("two.three", n.rest());
n = n.rest();
@@ -190,7 +192,7 @@ public class CompoundNameTestCase {
@Test
void testHashCodeAndEquals() {
- CompoundName n1 = new CompoundName("venn.d.a");
+ CompoundName n1 = CompoundName.from("venn.d.a");
CompoundName n2 = new CompoundName(n1.asList());
assertEquals(n1.hashCode(), n2.hashCode());
verifyStrict(n1, n2);
@@ -198,12 +200,12 @@ public class CompoundNameTestCase {
@Test
void testAppendString() {
- verifyStrict("a", new CompoundName("a").append(""));
- verifyStrict("a", new CompoundName("").append("a"));
- verifyStrict("a.b", new CompoundName("a").append("b"));
- verifyStrict("a.b.c.d", new CompoundName("a.b").append("c.d"));
+ verifyStrict("a", CompoundName.from("a").append(""));
+ verifyStrict("a", CompoundName.empty.append("a"));
+ verifyStrict("a.b", CompoundName.from("a").append("b"));
+ verifyStrict("a.b.c.d", CompoundName.from("a.b").append("c.d"));
- CompoundName name = new CompoundName("a.b");
+ CompoundName name = CompoundName.from("a.b");
verifyStrict("a.b.c", name.append("c"));
verifyStrict("a.b.d", name.append("d"));
verifyStrict("a.b.d.e", name.append("d.e"));
@@ -218,7 +220,7 @@ public class CompoundNameTestCase {
@Test
void testAsList2() {
- assertEquals("[one]", new CompoundName("one").asList().toString());
- assertEquals("[one, two, three]", new CompoundName("one.two.three").asList().toString());
+ assertEquals("[one]", CompoundName.from("one").asList().toString());
+ assertEquals("[one, two, three]", CompoundName.from("one.two.three").asList().toString());
}
}
diff --git a/container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java
index c82b85d84d9..8e960bd1305 100644
--- a/container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java
+++ b/container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java
@@ -9,7 +9,9 @@ import com.yahoo.processing.request.properties.PropertyMap;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Tests using requests
@@ -19,9 +21,26 @@ import static org.junit.jupiter.api.Assertions.assertNull;
public class RequestTestCase {
private static final double delta = 0.0000000001;
+ private static final CompoundName C_a = CompoundName.from("a");
+
+ private static final CompoundName C_B = CompoundName.from("b");
+ private static final CompoundName C_C = CompoundName.from("c");
+ private static final CompoundName C_D = CompoundName.from("d");
+ private static final CompoundName C_f = CompoundName.from("f");
+ private static final CompoundName C_g = CompoundName.from("g");
+ private static final CompoundName C_I = CompoundName.from("i");
+ private static final CompoundName C_L = CompoundName.from("l");
+ private static final CompoundName C_M = CompoundName.from("m");
+ private static final CompoundName C_N = CompoundName.from("n");
+ private static final CompoundName C_o = CompoundName.from("o");
+ private static final CompoundName C_x = CompoundName.from("x");
+ private static final CompoundName C_none = CompoundName.from("none");
+
+
@Test
void testProperties() {
+
Properties p = new PropertyMap();
p.set("a", "a1");
Request r = new Request(p);
@@ -33,38 +52,38 @@ public class RequestTestCase {
assertEquals("b1", r.properties().get("b", "default"));
assertEquals("default", r.properties().get("c", "default"));
assertNull(r.properties().get("c"));
- assertEquals("b1", r.properties().get(new CompoundName("b")));
- assertEquals("b1", r.properties().get(new CompoundName("b"), "default"));
- assertEquals("default", r.properties().get(new CompoundName("c"), "default"));
- assertNull(r.properties().get(new CompoundName("c")));
+ assertEquals("b1", r.properties().get(C_B));
+ assertEquals("b1", r.properties().get(C_B, "default"));
+ assertEquals("default", r.properties().get(C_C, "default"));
+ assertNull(r.properties().get(C_C));
assertEquals("b1", r.properties().getString("b"));
assertEquals("b1", r.properties().getString("b", "default"));
assertEquals("default", r.properties().getString("c", "default"));
assertNull(r.properties().getString("c"));
- assertEquals("b1", r.properties().getString(new CompoundName("b")));
- assertEquals("b1", r.properties().getString(new CompoundName("b"), "default"));
- assertEquals("default", r.properties().getString(new CompoundName("c"), "default"));
- assertNull(r.properties().getString(new CompoundName("c")));
+ assertEquals("b1", r.properties().getString(C_B));
+ assertEquals("b1", r.properties().getString(C_B, "default"));
+ assertEquals("default", r.properties().getString(C_C, "default"));
+ assertNull(r.properties().getString(C_C));
r.properties().set("i", 7);
assertEquals(7, (int) r.properties().getInteger("i"));
assertEquals(7, (int) r.properties().getInteger("i", 3));
assertEquals(3, (int) r.properties().getInteger("n", 3));
assertNull(r.properties().getInteger("n"));
- assertEquals(7, (int) r.properties().getInteger(new CompoundName("i")));
- assertEquals(7, (int) r.properties().getInteger(new CompoundName("i"), 3));
- assertEquals(3, (int) r.properties().getInteger(new CompoundName("n"), 3));
+ assertEquals(7, (int) r.properties().getInteger(C_I));
+ assertEquals(7, (int) r.properties().getInteger(C_I, 3));
+ assertEquals(3, (int) r.properties().getInteger(C_N, 3));
assertNull(r.properties().getInteger("n"));
- r.properties().set(new CompoundName("l"), 7);
+ r.properties().set(C_L, 7);
assertEquals(7, (long) r.properties().getLong("l"));
- assertEquals(7, (long) r.properties().getLong("l", 3l));
- assertEquals(3, (long) r.properties().getLong("m", 3l));
+ assertEquals(7, (long) r.properties().getLong("l", 3L));
+ assertEquals(3, (long) r.properties().getLong("m", 3L));
assertNull(r.properties().getInteger("m"));
- assertEquals(7, (long) r.properties().getLong(new CompoundName("l")));
- assertEquals(7, (long) r.properties().getLong(new CompoundName("l"), 3l));
- assertEquals(3, (long) r.properties().getLong(new CompoundName("m"), 3l));
+ assertEquals(7, (long) r.properties().getLong(C_L));
+ assertEquals(7, (long) r.properties().getLong(C_L, 3L));
+ assertEquals(3, (long) r.properties().getLong(C_M, 3L));
assertNull(r.properties().getInteger("m"));
r.properties().set("d", 7.3);
@@ -72,29 +91,29 @@ public class RequestTestCase {
assertEquals(7.3, r.properties().getDouble("d", 3.4d), delta);
assertEquals(3.4, r.properties().getDouble("f", 3.4d), delta);
assertNull(r.properties().getDouble("f"));
- assertEquals(7.3, r.properties().getDouble(new CompoundName("d")), delta);
- assertEquals(7.3, r.properties().getDouble(new CompoundName("d"), 3.4d), delta);
- assertEquals(3.4, r.properties().getDouble(new CompoundName("f"), 3.4d), delta);
+ assertEquals(7.3, r.properties().getDouble(C_D), delta);
+ assertEquals(7.3, r.properties().getDouble(C_D, 3.4d), delta);
+ assertEquals(3.4, r.properties().getDouble(C_f, 3.4d), delta);
assertNull(r.properties().getDouble("f"));
r.properties().set("o", true);
- assertEquals(true, r.properties().getBoolean("o"));
- assertEquals(true, r.properties().getBoolean("o", true));
- assertEquals(true, r.properties().getBoolean("g", true));
- assertEquals(false, r.properties().getBoolean("g"));
- assertEquals(true, r.properties().getBoolean(new CompoundName("o")));
- assertEquals(true, r.properties().getBoolean(new CompoundName("o"), true));
- assertEquals(true, r.properties().getBoolean(new CompoundName("g"), true));
- assertEquals(false, r.properties().getBoolean("g"));
-
- r.properties().set(new CompoundName("x.y"), "x1.y1");
+ assertTrue(r.properties().getBoolean("o"));
+ assertTrue(r.properties().getBoolean("o", true));
+ assertTrue(r.properties().getBoolean("g", true));
+ assertFalse(r.properties().getBoolean("g"));
+ assertTrue(r.properties().getBoolean(C_o));
+ assertTrue(r.properties().getBoolean(C_o, true));
+ assertTrue(r.properties().getBoolean(C_g, true));
+ assertFalse(r.properties().getBoolean("g"));
+
+ r.properties().set(CompoundName.from("x.y"), "x1.y1");
r.properties().set("x.z", "x1.z1");
assertEquals(8, r.properties().listProperties().size());
assertEquals(0, r.properties().listProperties("a").size());
- assertEquals(0, r.properties().listProperties(new CompoundName("a")).size());
- assertEquals(0, r.properties().listProperties(new CompoundName("none")).size());
- assertEquals(2, r.properties().listProperties(new CompoundName("x")).size());
+ assertEquals(0, r.properties().listProperties(C_a).size());
+ assertEquals(0, r.properties().listProperties(C_none).size());
+ assertEquals(2, r.properties().listProperties(C_x).size());
assertEquals(2, r.properties().listProperties("x").size());
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
index 88a37ea5a02..92ce6abb319 100644
--- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
+++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
@@ -6,11 +6,11 @@ import com.yahoo.search.Query;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import static com.yahoo.text.Lowercase.toLowerCase;
@@ -32,6 +32,16 @@ public class IndexFacts {
private Map<String, List<String>> clusterByDocument;
+ private static class DocumentTypeListOffset {
+ public final int offset;
+ public final SearchDefinition searchDefinition;
+
+ public DocumentTypeListOffset(int offset, SearchDefinition searchDefinition) {
+ this.offset = offset;
+ this.searchDefinition = searchDefinition;
+ }
+ }
+
/** A Map of all known search definitions indexed by name */
private Map<String, SearchDefinition> searchDefinitions = new LinkedHashMap<>();
@@ -100,32 +110,34 @@ public class IndexFacts {
private boolean isIndexFromDocumentTypes(String indexName, List<String> documentTypes) {
if ( ! isInitialized()) return true;
- if (documentTypes.isEmpty()) return unionSearchDefinition.getIndex(indexName) != null;
+ if (documentTypes.isEmpty()) {
+ return unionSearchDefinition.getIndex(indexName) != null;
+ }
- for (String docName : documentTypes) {
- SearchDefinition sd = searchDefinitions.get(docName);
- if (sd != null) {
- Index index = sd.getIndex(indexName);
- if (index != null) return true;
+ DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0);
+ while (sd != null) {
+ Index index = sd.searchDefinition.getIndex(indexName);
+ if (index != null) {
+ return true;
}
+ sd = chooseSearchDefinition(documentTypes, sd.offset);
}
+
return false;
}
private String getCanonicNameFromDocumentTypes(String indexName, List<String> documentTypes) {
if (!isInitialized()) return indexName;
- String lowerCased = toLowerCase(indexName);
if (documentTypes.isEmpty()) {
- Index index = unionSearchDefinition.getIndexByLowerCase(lowerCased);
+ Index index = unionSearchDefinition.getIndexByLowerCase(toLowerCase(indexName));
return index == null ? indexName : index.getName();
}
- for (String docName : documentTypes) {
- SearchDefinition sd = searchDefinitions.get(docName);
- if (sd != null) {
- Index index = sd.getIndexByLowerCase(lowerCased);
- if (index != null) return index.getName();
- }
+ DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0);
+ while (sd != null) {
+ Index index = sd.searchDefinition.getIndexByLowerCase(toLowerCase(indexName));
+ if (index != null) return index.getName();
+ sd = chooseSearchDefinition(documentTypes, sd.offset);
}
return indexName;
}
@@ -146,12 +158,13 @@ public class IndexFacts {
return index;
}
- for (String docName : documentTypes) {
- SearchDefinition sd = searchDefinitions.get(docName);
- if (sd != null) {
- Index index = sd.getIndex(canonicName);
- if (index != null) return index;
- }
+ DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0);
+
+ while (sd != null) {
+ Index index = sd.searchDefinition.getIndex(canonicName);
+
+ if (index != null) return index;
+ sd = chooseSearchDefinition(documentTypes, sd.offset);
}
return Index.nullIndex;
}
@@ -174,7 +187,7 @@ public class IndexFacts {
* Given a search list which is a mixture of document types and cluster
* names, and a restrict list which is a list of document types, return a
* set of all valid document types for this combination. Most use-cases for
- * fetching index settings will involve calling this method with the
+ * fetching index settings will involve calling this method with the the
* incoming query's {@link com.yahoo.search.query.Model#getSources()} and
* {@link com.yahoo.search.query.Model#getRestrict()} as input parameters
* before calling any other method of this class.
@@ -183,20 +196,20 @@ public class IndexFacts {
* @param restrict the restrict list for a query
* @return a (possibly empty) set of valid document types
*/
- private Set<String> resolveDocumentTypes(Collection<String> sources, Set<String> restrict,
+ private Set<String> resolveDocumentTypes(Collection<String> sources, Collection<String> restrict,
Set<String> candidateDocumentTypes) {
sources = emptyCollectionIfNull(sources);
- restrict = emptySetIfNull(restrict);
+ restrict = emptyCollectionIfNull(restrict);
if (sources.isEmpty()) {
if ( ! restrict.isEmpty()) {
- return Set.copyOf(restrict);
+ return new TreeSet<>(restrict);
} else {
return candidateDocumentTypes;
}
}
- Set<String> toSearch = new HashSet<>();
+ Set<String> toSearch = new TreeSet<>();
for (String source : sources) { // source: a document type or a cluster containing them
List<String> clusterDocTypes = clusters.get(source);
if (clusterDocTypes == null) { // source was a document type
@@ -222,8 +235,21 @@ public class IndexFacts {
private Collection<String> emptyCollectionIfNull(Collection<String> collection) {
return collection == null ? List.of() : collection;
}
- private Set<String> emptySetIfNull(Set<String> collection) {
- return collection == null ? Set.of() : collection;
+
+ /**
+ * Chooses the correct search definition, default if in doubt.
+ *
+ * @return the search definition to use
+ */
+ private DocumentTypeListOffset chooseSearchDefinition(List<String> documentTypes, int index) {
+ while (index < documentTypes.size()) {
+ String docName = documentTypes.get(index++);
+ SearchDefinition sd = searchDefinitions.get(docName);
+ if (sd != null) {
+ return new DocumentTypeListOffset(index, sd);
+ }
+ }
+ return null;
}
/**
@@ -253,6 +279,10 @@ public class IndexFacts {
return frozen;
}
+ private void ensureNotFrozen() {
+ if (frozen) throw new IllegalStateException("Tried to modify frozen IndexFacts instance.");
+ }
+
public String getDefaultPosition(String sdName) {
SearchDefinition sd;
if (sdName == null) {
@@ -270,16 +300,12 @@ public class IndexFacts {
return new Session(query);
}
- public Session newSession() {
- return new Session(Set.of(), Set.of());
- }
-
- public Session newSession(Collection<String> sources, Set<String> restrict) {
+ public Session newSession(Collection<String> sources, Collection<String> restrict) {
return new Session(sources, restrict);
}
public Session newSession(Collection<String> sources,
- Set<String> restrict,
+ Collection<String> restrict,
Set<String> candidateDocumentTypes) {
return new Session(sources, restrict, candidateDocumentTypes);
}
@@ -297,12 +323,12 @@ public class IndexFacts {
documentTypes = List.copyOf(resolveDocumentTypes(query));
}
- private Session(Collection<String> sources, Set<String> restrict) {
+ private Session(Collection<String> sources, Collection<String> restrict) {
// Assumption: Search definition name equals document name.
documentTypes = List.copyOf(resolveDocumentTypes(sources, restrict, searchDefinitions.keySet()));
}
- private Session(Collection<String> sources, Set<String> restrict, Set<String> candidateDocumentTypes) {
+ private Session(Collection<String> sources, Collection<String> restrict, Set<String> candidateDocumentTypes) {
documentTypes = List.copyOf(resolveDocumentTypes(sources, restrict, candidateDocumentTypes));
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
index f0e3e3f3e44..46332d632fe 100644
--- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
@@ -1,8 +1,8 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.prelude.cluster;
-import com.yahoo.component.ComponentId;
import com.yahoo.component.annotation.Inject;
+import com.yahoo.component.ComponentId;
import com.yahoo.component.chain.dependencies.After;
import com.yahoo.component.provider.ComponentRegistry;
import com.yahoo.container.QrSearchersConfig;
@@ -28,6 +28,10 @@ import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher;
import com.yahoo.yolean.Exceptions;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
@@ -53,7 +57,8 @@ public class ClusterSearcher extends Searcher {
private final String searchClusterName;
- private final SchemaResolver schemaResolver;
+ // The set of document types contained in this search cluster
+ private final Set<String> schemas;
private final long maxQueryTimeout; // in milliseconds
private final long maxQueryCacheTimeout; // in milliseconds
@@ -79,7 +84,7 @@ public class ClusterSearcher extends Searcher {
searchClusterName = clusterConfig.clusterName();
QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName);
this.globalPhaseRanker = searchClusterConfig.globalphase() ? globalPhaseRanker : null;
- this.schemaResolver = new SchemaResolver(documentDbConfig);
+ schemas = new LinkedHashSet<>();
maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT);
maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT);
@@ -88,6 +93,9 @@ public class ClusterSearcher extends Searcher {
.com().yahoo().prelude().fastsearch().FastSearcher().docsum()
.defaultclass());
+ for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb())
+ schemas.add(docDb.name());
+
String uniqueServerId = UUID.randomUUID().toString();
if (searchClusterConfig.indexingmode() == STREAMING) {
server = vdsCluster(uniqueServerId, searchClusterIndex,
@@ -149,7 +157,7 @@ public class ClusterSearcher extends Searcher {
/** Do not use, for internal testing purposes only. **/
ClusterSearcher(Set<String> schemas, VespaBackEndSearcher searcher, Executor executor) {
- this.schemaResolver = new SchemaResolver(schemas);
+ this.schemas = schemas;
searchClusterName = "testScenario";
maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT;
maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT;
@@ -221,9 +229,8 @@ public class ClusterSearcher extends Searcher {
}
private Result doSearch(Searcher searcher, Query query, Execution execution) {
- var schemas = schemaResolver.resolve(query, execution);
if (schemas.size() > 1) {
- return searchMultipleDocumentTypes(searcher, query, execution, schemas);
+ return searchMultipleDocumentTypes(searcher, query, execution);
} else {
String docType = schemas.iterator().next();
query.getModel().setRestrict(docType);
@@ -259,7 +266,8 @@ public class ClusterSearcher extends Searcher {
}
}
- private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution, Set<String> schemas) {
+ private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution) {
+ Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts());
List<Query> queries = createQueries(query, schemas);
if (queries.size() == 1) {
return perSchemaSearch(searcher, queries.get(0), execution);
@@ -293,7 +301,25 @@ public class ClusterSearcher extends Searcher {
}
Set<String> resolveSchemas(Query query, IndexFacts indexFacts) {
- return schemaResolver.resolve(query, indexFacts);
+ Set<String> restrict = query.getModel().getRestrict();
+ if (restrict == null || restrict.isEmpty()) {
+ Set<String> sources = query.getModel().getSources();
+ return (sources == null || sources.isEmpty())
+ ? schemas
+ : new HashSet<>(indexFacts.newSession(sources, Collections.emptyList(), schemas).documentTypes());
+ } else {
+ return filterValidDocumentTypes(restrict);
+ }
+ }
+
+ private Set<String> filterValidDocumentTypes(Collection<String> restrict) {
+ Set<String> retval = new LinkedHashSet<>();
+ for (String docType : restrict) {
+ if (docType != null && schemas.contains(docType)) {
+ retval.add(docType);
+ }
+ }
+ return retval;
}
private List<Query> createQueries(Query query, Set<String> docTypes) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java b/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java
deleted file mode 100644
index 3a2125d1d38..00000000000
--- a/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-package com.yahoo.prelude.cluster;
-
-import com.yahoo.prelude.IndexFacts;
-import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
-import com.yahoo.search.Query;
-import com.yahoo.search.searchchain.Execution;
-
-import java.util.Collection;
-import java.util.LinkedHashSet;
-import java.util.Set;
-
-/**
- * Resolves schemas from query and execution context
- *
- * @author bjorncs
- */
-class SchemaResolver {
-
- private final Set<String> schemas;
-
- SchemaResolver(DocumentdbInfoConfig cfg) {
- this(cfg.documentdb().stream().map(DocumentdbInfoConfig.Documentdb::name).toList());
- }
-
- SchemaResolver(Collection<String> schemas) {
- this.schemas = new LinkedHashSet<>(schemas);
- }
-
- Set<String> resolve(Query query, Execution execution) {
- return resolve(query, execution.context().getIndexFacts());
- }
-
- Set<String> resolve(Query query, IndexFacts indexFacts) {
- if (schemas.size() == 1) return Set.of(schemas.iterator().next());
- var restrict = query.getModel().getRestrict();
- if (restrict == null || restrict.isEmpty()) {
- Set<String> sources = query.getModel().getSources();
- return (sources == null || sources.isEmpty())
- ? schemas
- : new LinkedHashSet<>(indexFacts.newSession(sources, Set.of(), schemas).documentTypes());
- } else {
- return filterValidDocumentTypes(restrict);
- }
- }
-
- private Set<String> filterValidDocumentTypes(Collection<String> restrict) {
- Set<String> retval = new LinkedHashSet<>();
- for (String docType : restrict) {
- if (docType != null && schemas.contains(docType)) {
- retval.add(docType);
- }
- }
- return retval;
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java
index 2bd408220cd..e3b2278475b 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java
@@ -6,6 +6,7 @@ import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.query.Item;
import com.yahoo.search.query.parser.Parser;
+import java.util.Collections;
import java.util.Set;
/**
@@ -22,7 +23,7 @@ public interface CustomParser extends Parser {
Set<String> toSearch, IndexFacts indexFacts, String defaultIndexName) {
if (indexFacts == null)
indexFacts = new IndexFacts();
- return parse(queryToParse, filterToParse, parsingLanguage, indexFacts.newSession(toSearch, Set.of()), defaultIndexName);
+ return parse(queryToParse, filterToParse, parsingLanguage, indexFacts.newSession(toSearch, Collections.emptySet()), defaultIndexName);
}
Item parse(String queryToParse, String filterToParse, Language parsingLanguage,
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
index 9952ec64d13..c1d415b8e27 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
@@ -8,6 +8,7 @@ import com.yahoo.prelude.Index;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.query.Substring;
+import java.util.Collections;
import java.util.List;
import static com.yahoo.prelude.query.parser.Token.Kind.*;
@@ -62,7 +63,7 @@ public final class Tokenizer {
* @return a read-only list of tokens. This list can only be used by this thread
*/
public List<Token> tokenize(String string) {
- return tokenize(string, new IndexFacts().newSession());
+ return tokenize(string, new IndexFacts().newSession(Collections.emptySet(), Collections.emptySet()));
}
/**
@@ -170,10 +171,13 @@ public final class Tokenizer {
// this is a heuristic to check whether we probably have reached the end of an URL element
for (int i = tokens.size() - 1; i >= 0; --i) {
switch (tokens.get(i).kind) {
- case COLON -> { if (i == indexLastExplicitlyChangedAt) return false; }
- case SPACE -> { return true; }
- default -> { }
- // do nothing
+ case COLON:
+ if (i == indexLastExplicitlyChangedAt) return false;
+ break;
+ case SPACE:
+ return true;
+ default:
+ // do nothing
}
}
// really not sure whether we should choose false instead, on cause of the guard at
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java
index 6bae5175d84..1eb7eb8abe7 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java
@@ -22,7 +22,7 @@ import java.util.List;
@Before("transformedQuery")
public class NonPhrasingSearcher extends Searcher {
- private static final CompoundName suggestonly=new CompoundName("suggestonly");
+ private static final CompoundName suggestonly = CompoundName.from("suggestonly");
private PhraseMatcher phraseMatcher;
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java
index b67ddade7ee..4da61f5eeca 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java
@@ -29,7 +29,7 @@ import java.util.List;
@Provides(PhrasingSearcher.PHRASE_REPLACEMENT)
public class PhrasingSearcher extends Searcher {
- private static final CompoundName suggestonly = new CompoundName("suggestonly");
+ private static final CompoundName suggestonly = CompoundName.from("suggestonly");
public static final String PHRASE_REPLACEMENT = "PhraseReplacement";
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java
index 2c7d78c0671..9f8facd2b28 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java
@@ -39,7 +39,7 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING;
@Before({STEMMING, ACCENT_REMOVAL})
public class RecallSearcher extends Searcher {
- public static final CompoundName recallName=new CompoundName("recall");
+ public static final CompoundName recallName = CompoundName.from("recall");
@Override
public Result search(Query query, Execution execution) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
index 6aa262ca49e..7c4bcb38c41 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
@@ -9,11 +9,33 @@ import com.yahoo.language.Language;
import com.yahoo.language.Linguistics;
import com.yahoo.language.process.StemMode;
import com.yahoo.language.process.StemList;
+
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
import java.util.logging.Level;
import com.yahoo.prelude.Index;
import com.yahoo.prelude.IndexFacts;
-import com.yahoo.prelude.query.*;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.AndSegmentItem;
+import com.yahoo.prelude.query.BlockItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Highlight;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.PhraseSegmentItem;
+import com.yahoo.prelude.query.PrefixItem;
+import com.yahoo.prelude.query.SegmentingRule;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.TaggableItem;
+import com.yahoo.prelude.query.TermItem;
+import com.yahoo.prelude.query.WordAlternativesItem;
import com.yahoo.prelude.query.WordAlternativesItem.Alternative;
+import com.yahoo.prelude.query.WordItem;
import com.yahoo.processing.request.CompoundName;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
@@ -21,7 +43,6 @@ import com.yahoo.search.Searcher;
import com.yahoo.search.searchchain.Execution;
import com.yahoo.search.searchchain.PhaseNames;
-import java.util.*;
import static com.yahoo.prelude.querytransform.CJKSearcher.TERM_ORDER_RELAXATION;
@@ -46,7 +67,7 @@ public class StemmingSearcher extends Searcher {
}
public static final String STEMMING = "Stemming";
- public static final CompoundName DISABLE = new CompoundName("nostemming");
+ public static final CompoundName DISABLE = CompoundName.from("nostemming");
private final Linguistics linguistics;
public StemmingSearcher(Linguistics linguistics) {
@@ -107,15 +128,13 @@ public class StemmingSearcher extends Searcher {
}
private Map<Item, TaggableItem> populateReverseConnectivityMap(Item root, Map<Item, TaggableItem> reverseConnectivity) {
- if (root instanceof TaggableItem) {
- TaggableItem asTaggable = (TaggableItem) root;
+ if (root instanceof TaggableItem asTaggable) {
Item connectsTo = asTaggable.getConnectedItem();
if (connectsTo != null) {
reverseConnectivity.put(connectsTo, asTaggable);
}
}
- if (root instanceof CompositeItem && !(root instanceof BlockItem)) {
- CompositeItem c = (CompositeItem) root;
+ if (root instanceof CompositeItem c && !(root instanceof BlockItem)) {
for (Iterator<Item> i = c.getItemIterator(); i.hasNext();) {
Item item = i.next();
populateReverseConnectivityMap(item, reverseConnectivity);
@@ -134,8 +153,7 @@ public class StemmingSearcher extends Searcher {
}
if (item instanceof BlockItem) {
item = checkBlock((BlockItem) item, context);
- } else if (item instanceof CompositeItem) {
- CompositeItem comp = (CompositeItem) item;
+ } else if (item instanceof CompositeItem comp) {
ListIterator<Item> i = comp.getItemIterator();
while (i.hasNext()) {
@@ -220,8 +238,7 @@ public class StemmingSearcher extends Searcher {
copyAttributes(blockAsItem, composite);
composite.lock();
- if (composite instanceof PhraseSegmentItem) {
- PhraseSegmentItem replacement = (PhraseSegmentItem) composite;
+ if (composite instanceof PhraseSegmentItem replacement) {
setSignificance(replacement, current);
phraseSegmentConnectivity(current, context.reverseConnectivity, replacement);
}
@@ -258,10 +275,9 @@ public class StemmingSearcher extends Searcher {
}
private Connectivity getConnectivity(BlockItem current) {
- if (!(current instanceof TaggableItem)) {
+ if (!(current instanceof TaggableItem t)) {
return null;
}
- TaggableItem t = (TaggableItem) current;
if (t.getConnectedItem() == null) {
return null;
}
@@ -294,7 +310,7 @@ public class StemmingSearcher extends Searcher {
Substring substring,
boolean insidePhrase) {
String indexName = current.getIndexName();
- if (insidePhrase == false && ((index.getLiteralBoost() || index.getStemMode() == StemMode.ALL))) {
+ if (!insidePhrase && ((index.getLiteralBoost() || index.getStemMode() == StemMode.ALL))) {
List<Alternative> terms = new ArrayList<>(segment.size() + 1);
terms.add(new Alternative(current.stringValue(), 1.0d));
for (String term : segment) {
@@ -305,8 +321,7 @@ public class StemmingSearcher extends Searcher {
return alternatives;
}
}
- WordItem first = singleStemSegment((Item) current, segment.get(0), indexName, substring);
- return first;
+ return singleStemSegment((Item) current, segment.get(0), indexName, substring);
}
private void setMetaData(BlockItem current, Map<Item, TaggableItem> reverseConnectivity, TaggableItem replacement) {
@@ -353,14 +368,13 @@ public class StemmingSearcher extends Searcher {
if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT)
return chooseComposite(current, parent, indexName);
- switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only
- case PHRASE: return createPhraseSegment(current, indexName);
- case BOOLEAN_AND: return createAndSegment(current);
- default:
- throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() +
- ". This is a bug in Vespa, as the implementation has gotten out of sync." +
- " Please create an issue.");
- }
+ return switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only
+ case PHRASE -> createPhraseSegment(current, indexName);
+ case BOOLEAN_AND -> createAndSegment(current);
+ default -> throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() +
+ ". This is a bug in Vespa, as the implementation has gotten out of sync." +
+ " Please create an issue.");
+ };
}
private AndSegmentItem createAndSegment(BlockItem current) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java
index ead6ad53715..9927880c476 100644
--- a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java
@@ -25,10 +25,10 @@ import java.util.Map;
@Before(PhaseNames.TRANSFORMED_QUERY)
public class FieldCollapsingSearcher extends Searcher {
- private static final CompoundName collapse = new CompoundName("collapse");
- private static final CompoundName collapsefield = new CompoundName("collapsefield");
- private static final CompoundName collapsesize = new CompoundName("collapsesize");
- private static final CompoundName collapseSummaryName = new CompoundName("collapse.summary");
+ private static final CompoundName collapse = CompoundName.from("collapse");
+ private static final CompoundName collapsefield = CompoundName.from("collapsefield");
+ private static final CompoundName collapsesize = CompoundName.from("collapsesize");
+ private static final CompoundName collapseSummaryName = CompoundName.from("collapse.summary");
/** Maximum number of queries to send next searcher */
private static final int maxQueries = 4;
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java
index 409b502f086..7750bd17108 100644
--- a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java
@@ -25,7 +25,7 @@ public class JSONDebugSearcher extends Searcher {
public static final String STRUCT_FIELD = "Structured data field (as json): ";
public static final String FEATURE_FIELD = "Feature data field (as json): ";
- private static final CompoundName PROPERTYNAME = new CompoundName("dumpjson");
+ private static final CompoundName PROPERTYNAME = CompoundName.from("dumpjson");
@Override
public Result search(Query query, Execution execution) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java
index 3c61a361cbb..dd2c4a1da7f 100644
--- a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java
@@ -10,7 +10,12 @@ import com.yahoo.search.result.Hit;
import com.yahoo.search.result.HitGroup;
import com.yahoo.search.searchchain.Execution;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* Groups hits according to document type.
@@ -21,9 +26,9 @@ import java.util.*;
public class MultipleResultsSearcher extends Searcher {
private final static String propertyPrefix = "multipleresultsets.";
- private static final CompoundName additionalHitsFactorName=new CompoundName(propertyPrefix + "additionalHitsFactor");
- private static final CompoundName maxTimesRetrieveHeterogeneousHitsName=new CompoundName(propertyPrefix + "maxTimesRetrieveHeterogeneousHits");
- private static final CompoundName numHits=new CompoundName(propertyPrefix + "numHits");
+ private static final CompoundName additionalHitsFactorName = CompoundName.from(propertyPrefix + "additionalHitsFactor");
+ private static final CompoundName maxTimesRetrieveHeterogeneousHitsName = CompoundName.from(propertyPrefix + "maxTimesRetrieveHeterogeneousHits");
+ private static final CompoundName numHits = CompoundName.from(propertyPrefix + "numHits");
@Override
public Result search(Query query, Execution e) {
@@ -162,9 +167,9 @@ public class MultipleResultsSearcher extends Searcher {
// Assumes that field sddocname is available
private static class PartitionedResult {
- private Map<String, HitGroup> resultSets = new HashMap<>();
+ private final Map<String, HitGroup> resultSets = new HashMap<>();
- private List<Hit> otherHits = new ArrayList<>();
+ private final List<Hit> otherHits = new ArrayList<>();
PartitionedResult(List<DocumentGroup> documentGroups,Result result) throws ParameterException {
for (DocumentGroup group : documentGroups)
@@ -174,9 +179,8 @@ public class MultipleResultsSearcher extends Searcher {
}
void addHits(Result result, boolean addOtherHits) {
- Iterator<Hit> i = result.hits().iterator();
- while (i.hasNext()) {
- add(i.next(), addOtherHits);
+ for (Hit hit : result.hits()) {
+ add(hit, addOtherHits);
}
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java
index 0df21df8842..fd4c9e89f3b 100644
--- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java
@@ -37,12 +37,11 @@ public class PosSearcher extends Searcher {
public static final String POSITION_PARSING = "PositionParsing";
- private static final CompoundName posBb = new CompoundName("pos.bb");
- private static final CompoundName posLl = new CompoundName("pos.ll");
- private static final CompoundName posXy = new CompoundName("pos.xy");
- private static final CompoundName posAttributeName = new CompoundName("pos.attribute");
- private static final CompoundName posRadius = new CompoundName("pos.radius");
- private static final CompoundName posUnits = new CompoundName("pos.units");
+ private static final CompoundName posBb = CompoundName.from("pos.bb");
+ private static final CompoundName posLl = CompoundName.from("pos.ll");
+ private static final CompoundName posXy = CompoundName.from("pos.xy");
+ private static final CompoundName posAttributeName = CompoundName.from("pos.attribute");
+ private static final CompoundName posRadius = CompoundName.from("pos.radius");
// according to wikipedia:
// Earth's equatorial radius = 6378137 meter - not used
@@ -117,13 +116,13 @@ public class PosSearcher extends Searcher {
if (radius == null) {
radiusdegrees = 50.0 * km2deg;
} else if (radius.endsWith("km")) {
- double radiuskm = Double.valueOf(radius.substring(0, radius.length()-2));
+ double radiuskm = Double.parseDouble(radius.substring(0, radius.length()-2));
radiusdegrees = radiuskm * km2deg;
} else if (radius.endsWith("m")) {
- double radiusm = Double.valueOf(radius.substring(0, radius.length()-1));
+ double radiusm = Double.parseDouble(radius.substring(0, radius.length()-1));
radiusdegrees = radiusm * km2deg / 1000.0;
} else if (radius.endsWith("mi")) {
- double radiusmiles = Double.valueOf(radius.substring(0, radius.length()-2));
+ double radiusmiles = Double.parseDouble(radius.substring(0, radius.length()-2));
radiusdegrees = radiusmiles * mi2deg;
} else {
radiusdegrees = Integer.parseInt(radius) * 0.000001;
@@ -151,15 +150,15 @@ public class PosSearcher extends Searcher {
double radiusdegrees = radiuskm * km2deg;
radiusUnits = (int)(radiusdegrees * 1000000);
} else if (radius.endsWith("km")) {
- double radiuskm = Double.valueOf(radius.substring(0, radius.length()-2));
+ double radiuskm = Double.parseDouble(radius.substring(0, radius.length()-2));
double radiusdegrees = radiuskm * km2deg;
radiusUnits = (int)(radiusdegrees * 1000000);
} else if (radius.endsWith("m")) {
- double radiusm = Double.valueOf(radius.substring(0, radius.length()-1));
+ double radiusm = Double.parseDouble(radius.substring(0, radius.length()-1));
double radiusdegrees = radiusm * km2deg / 1000.0;
radiusUnits = (int)(radiusdegrees * 1000000);
} else if (radius.endsWith("mi")) {
- double radiusmiles = Double.valueOf(radius.substring(0, radius.length()-2));
+ double radiusmiles = Double.parseDouble(radius.substring(0, radius.length()-2));
double radiusdegrees = radiusmiles * mi2deg;
radiusUnits = (int)(radiusdegrees * 1000000);
} else {
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java
index 42a2b4f4e9b..5b3806485b5 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java
@@ -29,9 +29,9 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING;
@Before({PhaseNames.TRANSFORMED_QUERY, STEMMING})
public class SemanticSearcher extends Searcher {
- private static final CompoundName rulesRulebase = new CompoundName("rules.rulebase");
- private static final CompoundName rulesOff = new CompoundName("rules.off");
- private static final CompoundName tracelevelRules = new CompoundName("tracelevel.rules");
+ private static final CompoundName rulesRulebase = CompoundName.from("rules.rulebase");
+ private static final CompoundName rulesOff = CompoundName.from("rules.off");
+ private static final CompoundName tracelevelRules = CompoundName.from("tracelevel.rules");
/** The default rule base of this */
private RuleBase defaultRuleBase;
diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java
index c13bdff1f50..de24196327e 100644
--- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java
@@ -56,7 +56,7 @@ import static com.yahoo.container.protect.Error.UNSPECIFIED;
@Before(PhaseNames.RAW_QUERY)
public class StatisticsSearcher extends Searcher {
- private static final CompoundName IGNORE_QUERY = new CompoundName("metrics.ignore");
+ private static final CompoundName IGNORE_QUERY = CompoundName.from("metrics.ignore");
private static final String MAX_QUERY_LATENCY_METRIC = ContainerMetrics.MAX_QUERY_LATENCY.baseName();
private static final String EMPTY_RESULTS_METRIC = ContainerMetrics.EMPTY_RESULTS.baseName();
private static final String HITS_PER_QUERY_METRIC = ContainerMetrics.HITS_PER_QUERY.baseName();
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 39fd372d2a7..73ed89687fe 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -48,7 +48,6 @@ import com.yahoo.yolean.Exceptions;
import java.net.URI;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -180,23 +179,23 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
//---------------- Static property handling ------------------------------------
- public static final CompoundName OFFSET = new CompoundName("offset");
- public static final CompoundName HITS = new CompoundName("hits");
+ public static final CompoundName OFFSET = CompoundName.from("offset");
+ public static final CompoundName HITS = CompoundName.from("hits");
- public static final CompoundName QUERY_PROFILE = new CompoundName("queryProfile");
- public static final CompoundName SEARCH_CHAIN = new CompoundName("searchChain");
+ public static final CompoundName QUERY_PROFILE = CompoundName.from("queryProfile");
+ public static final CompoundName SEARCH_CHAIN = CompoundName.from("searchChain");
- public static final CompoundName NO_CACHE = new CompoundName("noCache");
- public static final CompoundName GROUPING_SESSION_CACHE = new CompoundName("groupingSessionCache");
- public static final CompoundName TIMEOUT = new CompoundName("timeout");
+ public static final CompoundName NO_CACHE = CompoundName.from("noCache");
+ public static final CompoundName GROUPING_SESSION_CACHE = CompoundName.from("groupingSessionCache");
+ public static final CompoundName TIMEOUT = CompoundName.from("timeout");
/** @deprecated use Trace.LEVEL */
@Deprecated // TODO: Remove on Vespa 9
- public static final CompoundName TRACE_LEVEL = new CompoundName("traceLevel");
+ public static final CompoundName TRACE_LEVEL = CompoundName.from("traceLevel");
/** @deprecated use Trace.EXPLAIN_LEVEL */
@Deprecated // TODO: Remove on Vespa 9
- public static final CompoundName EXPLAIN_LEVEL = new CompoundName("explainLevel");
+ public static final CompoundName EXPLAIN_LEVEL = CompoundName.from("explainLevel");
private static final QueryProfileType argumentType;
static {
@@ -228,17 +227,17 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
private static final Map<String, CompoundName> propertyAliases;
static {
Map<String, CompoundName> propertyAliasesBuilder = new HashMap<>();
- addAliases(Query.getArgumentType(), CompoundName.empty, propertyAliasesBuilder);
+ addAliases(Query.getArgumentType(), "", propertyAliasesBuilder);
propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder);
}
- private static void addAliases(QueryProfileType arguments, CompoundName prefix, Map<String, CompoundName> aliases) {
+ private static void addAliases(QueryProfileType arguments, String prefix, Map<String, CompoundName> aliases) {
for (FieldDescription field : arguments.fields().values()) {
for (String alias : field.getAliases())
- aliases.put(alias, prefix.append(field.getName()));
+ aliases.put(alias, CompoundName.from(append(prefix, field.getName())));
if (field.getType() instanceof QueryProfileFieldType) {
var type = ((QueryProfileFieldType) field.getType()).getQueryProfileType();
if (type != null)
- addAliases(type, prefix.append(type.getComponentIdAsCompoundName()), aliases);
+ addAliases(type, append(prefix, type.getComponentIdAsCompoundName().toString()), aliases);
}
}
}
@@ -261,18 +260,18 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
/** Returns an unmodifiable list of all the native properties under a Query */
public static final List<CompoundName> nativeProperties =
- List.copyOf(namesUnder(CompoundName.empty, Query.getArgumentType()));
+ List.copyOf(namesUnder("", Query.getArgumentType()));
- private static List<CompoundName> namesUnder(CompoundName prefix, QueryProfileType type) {
- if (type == null) return Collections.emptyList(); // Names not known statically
+ private static List<CompoundName> namesUnder(String prefix, QueryProfileType type) {
+ if (type == null) return List.of(); // Names not known statically
List<CompoundName> names = new ArrayList<>();
for (Map.Entry<String, FieldDescription> field : type.fields().entrySet()) {
+ String name = append(prefix, field.getKey());
if (field.getValue().getType() instanceof QueryProfileFieldType) {
- names.addAll(namesUnder(prefix.append(field.getKey()),
- ((QueryProfileFieldType) field.getValue().getType()).getQueryProfileType()));
+ names.addAll(namesUnder(name, ((QueryProfileFieldType) field.getValue().getType()).getQueryProfileType()));
}
else {
- names.add(prefix.append(field.getKey()));
+ names.add(CompoundName.from(name));
}
}
return names;
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index 09db1be4732..4e4b77422c1 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -52,7 +52,7 @@ public class Dispatcher extends AbstractComponent {
private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3;
/** If set will control computation of how many hits will be fetched from each partition.*/
- public static final CompoundName topKProbability = CompoundName.fromComponents(DISPATCH, TOP_K_PROBABILITY);
+ public static final CompoundName topKProbability = CompoundName.from(DISPATCH + "." + TOP_K_PROBABILITY);
private final DispatchConfig dispatchConfig;
private final RpcResourcePool rpcResourcePool;
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java
index 9e7fc9b5b29..c10375a3014 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java
@@ -13,7 +13,7 @@ import com.yahoo.search.Query;
*/
public class CompressService implements CompressPayload {
/** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */
- public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression");
+ public static final CompoundName dispatchCompression = CompoundName.from("dispatch.compression");
private final Compressor compressor = new Compressor(CompressionType.LZ4, 5, 0.95, 256);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
index 4466b03a713..154002c4f77 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
@@ -20,11 +20,20 @@ public class RpcInvokerFactory extends InvokerFactory {
private final RpcConnectionPool rpcResourcePool;
private final CompressPayload compressor;
+ private final RpcProtobufFillInvoker.DecodePolicy decodeType;
+
+ private static RpcProtobufFillInvoker.DecodePolicy convert(DispatchConfig.SummaryDecodePolicy.Enum decoding) {
+ return switch (decoding) {
+ case EAGER -> RpcProtobufFillInvoker.DecodePolicy.EAGER;
+ case ONDEMAND -> RpcProtobufFillInvoker.DecodePolicy.ONDEMAND;
+ };
+ }
public RpcInvokerFactory(RpcConnectionPool rpcResourcePool, SearchGroups cluster, DispatchConfig dispatchConfig) {
super(cluster, dispatchConfig);
this.rpcResourcePool = rpcResourcePool;
this.compressor = new CompressService();
+ decodeType = convert(dispatchConfig.summaryDecodePolicy());
}
@Override
@@ -37,6 +46,7 @@ public class RpcInvokerFactory extends InvokerFactory {
Query query = result.getQuery();
boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query);
- return new RpcProtobufFillInvoker(rpcResourcePool, compressor, searcher.getDocumentDatabase(query), searcher.getServerId(), summaryNeedsQuery);
+ return new RpcProtobufFillInvoker(rpcResourcePool, compressor, searcher.getDocumentDatabase(query),
+ searcher.getServerId(), decodeType, summaryNeedsQuery);
}
}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java
index 4e538fb54dc..2bdafecfaba 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java
@@ -21,6 +21,7 @@ import com.yahoo.search.result.ErrorMessage;
import com.yahoo.search.result.Hit;
import com.yahoo.slime.ArrayTraverser;
import com.yahoo.slime.BinaryFormat;
+import com.yahoo.slime.BinaryView;
import java.util.Iterator;
import java.util.List;
@@ -42,11 +43,14 @@ public class RpcProtobufFillInvoker extends FillInvoker {
private static final Logger log = Logger.getLogger(RpcProtobufFillInvoker.class.getName());
+ enum DecodePolicy {EAGER, ONDEMAND}
+
private final DocumentDatabase documentDb;
private final RpcConnectionPool resourcePool;
private final boolean summaryNeedsQuery;
private final String serverId;
private final CompressPayload compressor;
+ private final DecodePolicy decodePolicy;
private BlockingQueue<Pair<Client.ResponseOrError<ProtobufResponse>, List<FastHit>>> responses;
@@ -56,12 +60,14 @@ public class RpcProtobufFillInvoker extends FillInvoker {
/** The number of responses we should receive (and process) before this is complete */
private int outstandingResponses;
- RpcProtobufFillInvoker(RpcConnectionPool resourcePool, CompressPayload compressor, DocumentDatabase documentDb, String serverId, boolean summaryNeedsQuery) {
+ RpcProtobufFillInvoker(RpcConnectionPool resourcePool, CompressPayload compressor, DocumentDatabase documentDb,
+ String serverId, DecodePolicy decodePolicy, boolean summaryNeedsQuery) {
this.documentDb = documentDb;
this.resourcePool = resourcePool;
this.serverId = serverId;
this.summaryNeedsQuery = summaryNeedsQuery;
this.compressor = compressor;
+ this.decodePolicy = decodePolicy;
}
@Override
@@ -211,7 +217,9 @@ public class RpcProtobufFillInvoker extends FillInvoker {
private int fill(Result result, List<FastHit> hits, String summaryClass, byte[] payload) {
try {
var protobuf = SearchProtocol.DocsumReply.parseFrom(payload);
- var root = BinaryFormat.decode(protobuf.getSlimeSummaries().toByteArray()).get();
+ var root = (decodePolicy == DecodePolicy.ONDEMAND)
+ ? BinaryView.inspect(protobuf.getSlimeSummaries().toByteArray())
+ : BinaryFormat.decode(protobuf.getSlimeSummaries().toByteArray()).get();
var errors = root.field("errors");
boolean hasErrors = errors.valid() && (errors.entries() > 0);
if (hasErrors) {
diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java
index 80a41ffdf22..b8be1dc210b 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java
@@ -70,8 +70,8 @@ public class FederationSearcher extends ForkingSearcher {
private static final Logger log = Logger.getLogger(FederationSearcher.class.getName());
/** The name of the query property containing the source name added to the query to each source by this */
- public final static CompoundName SOURCENAME = new CompoundName("sourceName");
- public final static CompoundName PROVIDERNAME = new CompoundName("providerName");
+ public static final CompoundName SOURCENAME = CompoundName.from("sourceName");
+ public static final CompoundName PROVIDERNAME = CompoundName.from("providerName");
public static final String FEDERATION = "Federation";
public static final String LOG_COUNT_PREFIX = "count_";
@@ -686,33 +686,6 @@ public class FederationSearcher extends ForkingSearcher {
}
- private static class CompoundKey {
-
- private final String sourceName;
- private final String propertyName;
-
- CompoundKey(String sourceName, String propertyName) {
- this.sourceName = sourceName;
- this.propertyName = propertyName;
- }
-
- @Override
- public int hashCode() {
- return sourceName.hashCode() ^ propertyName.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- CompoundKey rhs = (CompoundKey) o;
- return sourceName.equals(rhs.sourceName) && propertyName.equals(rhs.propertyName);
- }
-
- @Override
- public String toString() {
- return sourceName + '.' + propertyName;
- }
- }
-
private static class Window {
private final int hits;
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java
index 54194221958..180687d5274 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java
@@ -38,13 +38,13 @@ import java.util.TimeZone;
public class GroupingQueryParser extends Searcher {
public static final String SELECT_PARAMETER_PARSING = "SelectParameterParsing";
- public static final CompoundName PARAM_CONTINUE = new CompoundName("continue");
- public static final CompoundName PARAM_REQUEST = new CompoundName(Select.SELECT);
- public static final CompoundName PARAM_TIMEZONE = new CompoundName("timezone");
- @Beta public static final CompoundName PARAM_DEFAULT_MAX_HITS = new CompoundName("grouping.defaultMaxHits");
- @Beta public static final CompoundName PARAM_DEFAULT_MAX_GROUPS = new CompoundName("grouping.defaultMaxGroups");
- @Beta public static final CompoundName PARAM_DEFAULT_PRECISION_FACTOR = new CompoundName("grouping.defaultPrecisionFactor");
- @Beta public static final CompoundName GROUPING_GLOBAL_MAX_GROUPS = new CompoundName("grouping.globalMaxGroups");
+ public static final CompoundName PARAM_CONTINUE = CompoundName.from("continue");
+ public static final CompoundName PARAM_REQUEST = CompoundName.from(Select.SELECT);
+ public static final CompoundName PARAM_TIMEZONE = CompoundName.from("timezone");
+ @Beta public static final CompoundName PARAM_DEFAULT_MAX_HITS = CompoundName.from("grouping.defaultMaxHits");
+ @Beta public static final CompoundName PARAM_DEFAULT_MAX_GROUPS = CompoundName.from("grouping.defaultMaxGroups");
+ @Beta public static final CompoundName PARAM_DEFAULT_PRECISION_FACTOR = CompoundName.from("grouping.defaultPrecisionFactor");
+ @Beta public static final CompoundName GROUPING_GLOBAL_MAX_GROUPS = CompoundName.from("grouping.globalMaxGroups");
private static final ThreadLocal<ZoneCache> zoneCache = new ThreadLocal<>();
@Override
@@ -121,7 +121,6 @@ public class GroupingQueryParser extends Searcher {
return val != null ? OptionalDouble.of(val) : OptionalDouble.empty();
}
- @SuppressWarnings("serial")
private static class ZoneCache extends LinkedHashMap<String, TimeZone> {
ZoneCache() {
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java
index 35cf72d6124..878fbbb733d 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java
@@ -37,7 +37,7 @@ import static com.yahoo.search.grouping.GroupingQueryParser.SELECT_PARAMETER_PAR
public class GroupingValidator extends Searcher {
public static final String GROUPING_VALIDATED = "GroupingValidated";
- public static final CompoundName PARAM_ENABLED = new CompoundName("validate_" + GroupingQueryParser.PARAM_REQUEST);
+ public static final CompoundName PARAM_ENABLED = CompoundName.from("validate_" + GroupingQueryParser.PARAM_REQUEST);
private final HashMap<String, AttributesConfig.Attribute> attributes = new HashMap<>();
private final String clusterName;
private final boolean enabled;
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java
index a1446367f79..27d8a76511d 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java
@@ -42,7 +42,7 @@ import java.util.logging.Logger;
@Before(PhaseNames.TRANSFORMED_QUERY)
public class UniqueGroupingSearcher extends Searcher {
- public static final CompoundName PARAM_UNIQUE = new CompoundName("unique");
+ public static final CompoundName PARAM_UNIQUE = CompoundName.from("unique");
private static final Logger log = Logger.getLogger(UniqueGroupingSearcher.class.getName());
private static final HitOrderer NOP_ORDERER = new HitOrderer() {
@@ -144,17 +144,13 @@ public class UniqueGroupingSearcher extends Searcher {
for (Sorting.FieldOrder fieldOrder : sortingSpec.fieldOrders()) {
Sorting.Order sortOrder = fieldOrder.getSortOrder();
switch (sortOrder) {
- case ASCENDING:
- case UNDEFINED:
- // When we want ascending order, the hit with the smallest value should come first (and be surfaced).
- orderingClause.add(new MinAggregator(new AttributeValue(fieldOrder.getFieldName())));
- break;
- case DESCENDING:
- // When we sort in descending order, the hit with the largest value should come first (and be surfaced).
- orderingClause.add(new NegFunction(new MaxAggregator(new AttributeValue(fieldOrder.getFieldName()))));
- break;
- default:
- throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + ".");
+ case ASCENDING, UNDEFINED ->
+ // When we want ascending order, the hit with the smallest value should come first (and be surfaced).
+ orderingClause.add(new MinAggregator(new AttributeValue(fieldOrder.getFieldName())));
+ case DESCENDING ->
+ // When we sort in descending order, the hit with the largest value should come first (and be surfaced).
+ orderingClause.add(new NegFunction(new MaxAggregator(new AttributeValue(fieldOrder.getFieldName()))));
+ default -> throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + ".");
}
}
return orderingClause;
@@ -170,18 +166,13 @@ public class UniqueGroupingSearcher extends Searcher {
GroupingExpression groupingClause = null;
for (Sorting.FieldOrder fieldOrder : sortingSpec.fieldOrders()) {
Sorting.Order sortOrder = fieldOrder.getSortOrder();
- switch (sortOrder) {
- case ASCENDING:
- case UNDEFINED:
- groupingClause = new AttributeValue(fieldOrder.getFieldName());
- break;
- case DESCENDING:
- // To sort descending, just take the negative. This is the most common case
- groupingClause = new NegFunction(new AttributeValue(fieldOrder.getFieldName()));
- break;
- default:
- throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + ".");
- }
+ groupingClause = switch (sortOrder) {
+ case ASCENDING, UNDEFINED -> new AttributeValue(fieldOrder.getFieldName());
+ case DESCENDING ->
+ // To sort descending, just take the negative. This is the most common case
+ new NegFunction(new AttributeValue(fieldOrder.getFieldName()));
+ default -> throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + ".");
+ };
}
return groupingClause;
}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java
index e56b88c3d3e..027ba92f587 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java
@@ -22,7 +22,7 @@ import java.util.Iterator;
@Before(GroupingExecutor.COMPONENT_NAME)
public class FlatteningSearcher extends Searcher {
- private final CompoundName flatten = CompoundName.fromComponents("grouping", "flatten");
+ private final CompoundName flatten = CompoundName.from("grouping.flatten");
@Override
public Result search(Query query, Execution execution) {
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java
index 2ba33f60ea1..c85dbed83c8 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java
@@ -379,7 +379,7 @@ public class GroupingExecutor extends Searcher {
}
private static CompoundName newCompoundName(String name) {
- return new CompoundName(GroupingExecutor.class.getName() + "." + name);
+ return CompoundName.from(GroupingExecutor.class.getName() + "." + name);
}
private static class RequestContext {
diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
index fac6c598e4b..3b4d15f788f 100644
--- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
+++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
@@ -75,8 +75,8 @@ public class SearchHandler extends LoggingRequestHandler {
// max number of threads for the executor for this handler
private final int maxThreads;
- private static final CompoundName DETAILED_TIMING_LOGGING = new CompoundName("trace.timingDetails");
- private static final CompoundName FORCE_TIMESTAMPS = new CompoundName("trace.timestamps");
+ private static final CompoundName DETAILED_TIMING_LOGGING = CompoundName.from("trace.timingDetails");
+ private static final CompoundName FORCE_TIMESTAMPS = CompoundName.from("trace.timestamps");
/** Event name for number of connections to the search subsystem */
private static final String SEARCH_CONNECTIONS = "search_connections";
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java b/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java
index 9aafd2c4693..0c4b22fbbf2 100644
--- a/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java
@@ -5,7 +5,12 @@ import com.yahoo.search.Query;
import com.yahoo.processing.request.CompoundName;
import com.yahoo.text.interpretation.Interpretation;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* This is the root node of an intent model.
@@ -19,9 +24,9 @@ import java.util.*;
public class IntentModel extends ParentNode<InterpretationNode> {
/** The name of the property carrying the intent model string: intentModel */
- public static final CompoundName intentModelStringName=new CompoundName("intentModel");
+ public static final CompoundName intentModelStringName = CompoundName.from("intentModel");
/** The name of the property carrying the intent model object: IntentModel */
- public static final CompoundName intentModelObjectName=new CompoundName("IntentModel");
+ public static final CompoundName intentModelObjectName = CompoundName.from("IntentModel");
private static final InterpretationNodeComparator inodeComp = new InterpretationNodeComparator();
@@ -45,7 +50,7 @@ public class IntentModel extends ParentNode<InterpretationNode> {
/** Sort interpretations by descending score order */
public void sortChildren() {
- Collections.sort(children(), inodeComp);
+ children().sort(inodeComp);
}
/**
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java
index c23627accf4..5fef1821de2 100644
--- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java
@@ -72,13 +72,13 @@ import java.util.Map;
public class PageTemplateSearcher extends Searcher {
/** The name of the query property containing the resolved candidate page template list */
- public static final CompoundName pagePageTemplateListName=new CompoundName("page.PageTemplateList");
+ public static final CompoundName pagePageTemplateListName = CompoundName.from("page.PageTemplateList");
/** The name of the query property containing a list of candidate pages to consider */
- public static final CompoundName pageIdListName=new CompoundName("page.idList");
+ public static final CompoundName pageIdListName = CompoundName.from("page.idList");
/** The name of the query property containing the page id to use */
- public static final CompoundName pageIdName=new CompoundName("page.id");
+ public static final CompoundName pageIdName = CompoundName.from("page.id");
/** The name of the query property containing the resolver id to use */
- public static final CompoundName pageResolverName=new CompoundName("page.resolver");
+ public static final CompoundName pageResolverName = CompoundName.from("page.resolver");
private final ResolverRegistry resolverRegistry;
diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java
index 190ad675015..09b2f394f20 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Model.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Model.java
@@ -71,13 +71,13 @@ public class Model implements Cloneable {
argumentType.addField(new FieldDescription(SEARCH_PATH, "string", "searchpath"));
argumentType.addField(new FieldDescription(RESTRICT, "string", "restrict"));
argumentType.freeze();
- argumentTypeName = new CompoundName(argumentType.getId().getName());
+ argumentTypeName = CompoundName.from(argumentType.getId().getName());
}
public static QueryProfileType getArgumentType() { return argumentType; }
/** The name of the query property used for generating hit count estimate queries. */
- public static final CompoundName ESTIMATE = new CompoundName("hitcountestimate"); // TODO: Cleanup
+ public static final CompoundName ESTIMATE = CompoundName.from("hitcountestimate"); // TODO: Cleanup
private String encoding = null;
private String queryString = "";
diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java
index e8738a19412..5426268d173 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java
@@ -27,7 +27,7 @@ import java.util.Objects;
public class Ranking implements Cloneable {
/** An alias for listing features */
- public static final CompoundName RANKFEATURES = new CompoundName("rankfeatures");
+ public static final CompoundName RANKFEATURES = CompoundName.from("rankfeatures");
/** The type representing the property arguments consumed by this */
private static final QueryProfileType argumentType;
@@ -70,7 +70,7 @@ public class Ranking implements Cloneable {
argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature input")); // Repeated at the end of RankFeatures
argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty"));
argumentType.freeze();
- argumentTypeName = new CompoundName(argumentType.getId().getName());
+ argumentTypeName = CompoundName.from(argumentType.getId().getName());
}
public static QueryProfileType getArgumentType() { return argumentType; }
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java
index ad9d3f4c1a5..ae531c67dd1 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java
@@ -169,7 +169,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable
}
public final Object get(String name) {
- return get(name, Collections.emptyMap());
+ return get(name, Map.of());
}
public final Object get(String name, Map<String, String> context) {
return get(name, context, new QueryProfileProperties(this));
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
index 8fdbf8b2281..719a5a2c281 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
@@ -64,7 +64,7 @@ public class FieldDescription implements Comparable<FieldDescription> {
}
public FieldDescription(String name, FieldType type, String aliases, boolean mandatory, boolean overridable) {
- this(new CompoundName(name), type, aliases, mandatory, overridable);
+ this(CompoundName.from(name), type, aliases, mandatory, overridable);
}
/**
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java
index 3da2ad53f9a..9cbfe5b7112 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java
@@ -52,7 +52,7 @@ public class QueryProfileType extends FreezableSimpleComponent {
private QueryProfileType(ComponentId id, Map<String, FieldDescription> fields, List<QueryProfileType> inherited) {
super(id);
QueryProfile.validateName(id.getName());
- componentIdAsCompoundName = new CompoundName(getId().getName());
+ componentIdAsCompoundName = CompoundName.from(getId().getName());
this.fields = fields;
this.inherited = inherited;
}
@@ -318,10 +318,9 @@ public class QueryProfileType extends FreezableSimpleComponent {
QueryProfileType type = null;
FieldDescription fieldDescription = getField(name);
if (fieldDescription != null) {
- if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType))
+ if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType fieldType))
throw new IllegalArgumentException("Cannot use name '" + name + "' as a prefix because it is " +
"already a " + fieldDescription.getType());
- QueryProfileFieldType fieldType = (QueryProfileFieldType) fieldDescription.getType();
type = fieldType.getQueryProfileType();
}
@@ -399,8 +398,7 @@ public class QueryProfileType extends FreezableSimpleComponent {
@Override
public boolean equals(Object o) {
if (o == this) return true;
- if ( ! (o instanceof QueryProfileType)) return false;
- QueryProfileType other = (QueryProfileType)o;
+ if ( ! (o instanceof QueryProfileType other)) return false;
return other.getId().equals(this.getId());
}
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java
index 221368afeb6..6ccf4792efb 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java
@@ -16,9 +16,9 @@ import java.util.Map;
*/
public final class DefaultProperties extends Properties {
- public static final CompoundName MAX_OFFSET = new CompoundName("maxOffset");
- public static final CompoundName MAX_HITS = new CompoundName("maxHits");
- public static final CompoundName MAX_QUERY_ITEMS = new CompoundName("maxQueryItems");
+ public static final CompoundName MAX_OFFSET = CompoundName.from("maxOffset");
+ public static final CompoundName MAX_HITS = CompoundName.from("maxHits");
+ public static final CompoundName MAX_QUERY_ITEMS = CompoundName.from("maxQueryItems");
public static final QueryProfileType argumentType = new QueryProfileType("DefaultProperties");
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java
index 42fb4e44040..4fc9dfa03cd 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java
@@ -10,14 +10,16 @@ import java.util.Map;
* A wrapper around a chain of property objects that prefixes all gets/sets with a given path
*
* @author Arne Bergene Fossaa
+ * @deprecated Unused and will go away on vespa 9
*/
+@Deprecated (forRemoval = true)
public class SubProperties extends com.yahoo.search.query.Properties {
final private CompoundName pathPrefix;
final private Properties parent;
public SubProperties(String pathPrefix, Properties properties) {
- this(new CompoundName(pathPrefix),properties);
+ this(CompoundName.from(pathPrefix), properties);
}
public SubProperties(CompoundName pathPrefix, Properties properties) {
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java
index 30ee2f473b7..c49b14f0978 100644
--- a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java
@@ -24,8 +24,7 @@ public class SoftTimeout implements Cloneable {
public static final String TAILCOST = "tailcost";
/** The full property name for turning softtimeout on or off */
- public static final CompoundName enableProperty =
- CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, ENABLE);
+ public static final CompoundName enableProperty = CompoundName.from(Ranking.RANKING + "." + Ranking.SOFTTIMEOUT + "." + ENABLE);
static {
argumentType = new QueryProfileType(Ranking.SOFTTIMEOUT);
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java
index b0fce45de46..eeb8b675ae1 100644
--- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java
@@ -42,7 +42,7 @@ public class RewriterConstants {
public static final String REWRITER_CHAIN = "QRWChain";
/** Name for rewrite metadata retrieval from query properties */
- public static final CompoundName REWRITE_META = new CompoundName("RewriteMeta");
+ public static final CompoundName REWRITE_META = CompoundName.from("RewriteMeta");
/** Name for rewritten field retrieval from query properties */
public static final String REWRITTEN = "Rewritten";
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java
index f43be20e0ac..91315fe1bb3 100644
--- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java
@@ -28,9 +28,9 @@ import static com.yahoo.yolean.Exceptions.toMessageString;
@Provides(BooleanSearcher.PREDICATE)
public class BooleanSearcher extends Searcher {
- private static final CompoundName FIELD = new CompoundName("boolean.field");
- private static final CompoundName ATTRIBUTES = new CompoundName("boolean.attributes");
- private static final CompoundName RANGE_ATTRIBUTES = new CompoundName("boolean.rangeAttributes");
+ private static final CompoundName FIELD = CompoundName.from("boolean.field");
+ private static final CompoundName ATTRIBUTES = CompoundName.from("boolean.attributes");
+ private static final CompoundName RANGE_ATTRIBUTES = CompoundName.from("boolean.rangeAttributes");
public static final String PREDICATE = "predicate";
@Override
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java
index 75b06c29193..81881685a1d 100644
--- a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java
@@ -41,9 +41,9 @@ public class SortingDegrader extends Searcher {
/** Set this to false in query.properties to turn off degrading. Default: on */
// (this is not called ranking.sorting.degrading because it should not be part of the query object model
- public static final CompoundName DEGRADING = new CompoundName("sorting.degrading");
+ public static final CompoundName DEGRADING = CompoundName.from("sorting.degrading");
- public static final CompoundName PAGINATION = new CompoundName("to_be_removed_pagination");
+ public static final CompoundName PAGINATION = CompoundName.from("to_be_removed_pagination");
@Override
public Result search(Query query, Execution execution) {
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java
index 89245eaf137..bfac668b6c4 100644
--- a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java
@@ -63,12 +63,12 @@ public class WandSearcher extends Searcher {
*/
private static class InputResolver {
- private static final CompoundName WAND_FIELD = new CompoundName("wand.field");
- private static final CompoundName WAND_TOKENS = new CompoundName("wand.tokens");
- private static final CompoundName WAND_HEAP_SIZE = new CompoundName("wand.heapSize");
- private static final CompoundName WAND_TYPE = new CompoundName("wand.type");
- private static final CompoundName WAND_SCORE_THRESHOLD = new CompoundName("wand.scoreThreshold");
- private static final CompoundName WAND_THRESHOLD_BOOST_FACTOR = new CompoundName("wand.thresholdBoostFactor");
+ private static final CompoundName WAND_FIELD = CompoundName.from("wand.field");
+ private static final CompoundName WAND_TOKENS = CompoundName.from("wand.tokens");
+ private static final CompoundName WAND_HEAP_SIZE = CompoundName.from("wand.heapSize");
+ private static final CompoundName WAND_TYPE = CompoundName.from("wand.type");
+ private static final CompoundName WAND_SCORE_THRESHOLD = CompoundName.from("wand.scoreThreshold");
+ private static final CompoundName WAND_THRESHOLD_BOOST_FACTOR = CompoundName.from("wand.thresholdBoostFactor");
private final String fieldName;
private final WandType wandType;
private final Map<Object, Integer> tokens;
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java
index 2d6e059342e..9aa7a9d998d 100644
--- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java
@@ -21,8 +21,8 @@ import com.yahoo.yolean.chain.After;
*/
@After(MinimalQueryInserter.EXTERNAL_YQL)
public class WeakAndReplacementSearcher extends Searcher {
- static final CompoundName WEAKAND_REPLACE = new CompoundName("weakAnd.replace");
- static final CompoundName WAND_HITS = new CompoundName("wand.hits");
+ static final CompoundName WEAKAND_REPLACE = CompoundName.from("weakAnd.replace");
+ static final CompoundName WAND_HITS = CompoundName.from("wand.hits");
@Override public Result search(Query query, Execution execution) {
if (!query.properties().getBoolean(WEAKAND_REPLACE)) {
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
index 44620179c1d..b36c8788877 100644
--- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
+++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
@@ -76,10 +76,10 @@ import static com.fasterxml.jackson.databind.SerializationFeature.FLUSH_AFTER_WR
// NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc.
public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
- private static final CompoundName WRAP_DEEP_MAPS = new CompoundName("renderer.json.jsonMaps");
- private static final CompoundName WRAP_WSETS = new CompoundName("renderer.json.jsonWsets");
- private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug");
- private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback");
+ private static final CompoundName WRAP_DEEP_MAPS = CompoundName.from("renderer.json.jsonMaps");
+ private static final CompoundName WRAP_WSETS = CompoundName.from("renderer.json.jsonWsets");
+ private static final CompoundName DEBUG_RENDERING_KEY = CompoundName.from("renderer.json.debug");
+ private static final CompoundName JSON_CALLBACK = CompoundName.from("jsoncallback");
// if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString
private static final String BUCKET_LIMITS = "limits";
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java
index a4f03b088f2..29e8afa41ac 100644
--- a/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java
@@ -32,9 +32,9 @@ import com.yahoo.search.searchchain.Execution;
*/
public class CacheControlSearcher extends Searcher {
- private static final CompoundName cachecontrolNocache=new CompoundName("cachecontrol.nocache");
- private static final CompoundName cachecontrolMaxage=new CompoundName("cachecontrol.maxage");
- private static final CompoundName cachecontrolStaleage=new CompoundName("cachecontrol.staleage");
+ private static final CompoundName cachecontrolNocache=CompoundName.from("cachecontrol.nocache");
+ private static final CompoundName cachecontrolMaxage=CompoundName.from("cachecontrol.maxage");
+ private static final CompoundName cachecontrolStaleage=CompoundName.from("cachecontrol.staleage");
public static final String CACHE_CONTROL_HEADER = "Cache-Control";
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java
index 90cb05be1f8..18588b2026d 100644
--- a/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java
@@ -49,7 +49,7 @@ public class ConnectionControlSearcher extends Searcher {
private final LongSupplier clock;
- private static final CompoundName KEEPALIVE_MAXLIFETIMESECONDS = new CompoundName("connectioncontrol.maxlifetime");
+ private static final CompoundName KEEPALIVE_MAXLIFETIMESECONDS = CompoundName.from("connectioncontrol.maxlifetime");
private static final String HTTP_CONNECTION_HEADER_NAME = "Connection";
private static final String HTTP_CONNECTION_CLOSE_ARGUMENT = "Close";
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java
index cf81f62f64d..35a3c86f763 100755
--- a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java
@@ -54,11 +54,11 @@ public class RateLimitingSearcher extends Searcher {
/** Constant containing the name this Provides - "rateLimiting", for ordering constraints */
public static final String RATE_LIMITING = "rateLimiting";
- public static final CompoundName idKey = new CompoundName("rate.id");
- public static final CompoundName costKey = new CompoundName("rate.cost");
- public static final CompoundName quotaKey = new CompoundName("rate.quota");
- public static final CompoundName idDimensionKey = new CompoundName("rate.idDimension");
- public static final CompoundName dryRunKey = new CompoundName("rate.dryRun");
+ public static final CompoundName idKey = CompoundName.from("rate.id");
+ public static final CompoundName costKey = CompoundName.from("rate.cost");
+ public static final CompoundName quotaKey = CompoundName.from("rate.quota");
+ public static final CompoundName idDimensionKey = CompoundName.from("rate.idDimension");
+ public static final CompoundName dryRunKey = CompoundName.from("rate.dryRun");
private static final String requestsOverQuotaMetricName = "requestsOverQuota";
diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java
index 833c1251a7b..6961a189d22 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java
@@ -27,7 +27,7 @@ public class FieldFiller extends Searcher {
private final Set<String> intersectionOfAttributes;
private final SchemaInfo schemaInfo;
- public static final CompoundName FIELD_FILLER_DISABLE = new CompoundName("FieldFiller.disable");
+ public static final CompoundName FIELD_FILLER_DISABLE = CompoundName.from("FieldFiller.disable");
public FieldFiller(SchemaInfo schemaInfo) {
this.schemaInfo = schemaInfo;
diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java
index 8cc6a1b42b9..df9722d3214 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java
@@ -2,13 +2,11 @@
package com.yahoo.search.yql;
import java.util.Iterator;
-import java.util.Map.Entry;
import java.util.Set;
import com.yahoo.api.annotations.Beta;
import com.yahoo.component.chain.dependencies.After;
import com.yahoo.component.chain.dependencies.Before;
-import com.yahoo.prelude.fastsearch.FastHit;
import com.yahoo.processing.request.CompoundName;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
@@ -27,7 +25,7 @@ import com.yahoo.search.searchchain.Execution;
@Before("com.yahoo.search.yql.FieldFiller")
public class FieldFilter extends Searcher {
- public static final CompoundName FIELD_FILTER_DISABLE = new CompoundName("FieldFilter.disable");
+ public static final CompoundName FIELD_FILTER_DISABLE = CompoundName.from("FieldFilter.disable");
/** Fields that should be kept even if not explicitly requested */
private static final Set<String> syntheticFields = Set.of("matchfeatures", "rankfeatures", "summaryfeatures");
diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
index e844bac21e8..ee4f931d532 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
@@ -39,10 +39,10 @@ public class MinimalQueryInserter extends Searcher {
public static final String EXTERNAL_YQL = "ExternalYql";
- public static final CompoundName YQL = new CompoundName("yql");
+ public static final CompoundName YQL = CompoundName.from("yql");
- private static final CompoundName MAX_HITS = new CompoundName("maxHits");
- private static final CompoundName MAX_OFFSET = new CompoundName("maxOffset");
+ private static final CompoundName MAX_HITS = CompoundName.from("maxHits");
+ private static final CompoundName MAX_OFFSET = CompoundName.from("maxOffset");
private static final Logger log = Logger.getLogger(MinimalQueryInserter.class.getName());
@Inject
diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java
index ab9da8ccee5..536355ab62d 100644
--- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java
+++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java
@@ -22,8 +22,8 @@ import static com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher.STREAMING_S
*/
public class MetricsSearcher extends Searcher {
- private static final CompoundName metricsearcherId=new CompoundName("metricsearcher.id");
- private static final CompoundName streamingLoadtype=new CompoundName("streaming.loadtype");
+ private static final CompoundName metricsearcherId = CompoundName.from("metricsearcher.id");
+ private static final CompoundName streamingLoadtype = CompoundName.from("streaming.loadtype");
private static final Logger log = Logger.getLogger(MetricsSearcher.class.getName());
@@ -43,7 +43,8 @@ public class MetricsSearcher extends Searcher {
public Result search(Query query, Execution execution) {
long timeMs = System.currentTimeMillis();
- /** Backwards compatibility - convert metricsearcher.id to streaming.loadtype */
+ // Backwards compatibility - convert metricsearcher.id to streaming.loadtype
+ // TODO Cleanup at some point
String metricName = query.properties().getString(metricsearcherId);
if (metricName != null) {
query.properties().set(streamingLoadtype, metricName);
diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java
index ca199f37dd7..524001748c5 100644
--- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java
@@ -45,9 +45,9 @@ import java.util.logging.Logger;
*/
public class VdsStreamingSearcher extends VespaBackEndSearcher {
- private static final CompoundName streamingUserid = new CompoundName("streaming.userid");
- private static final CompoundName streamingGroupname = new CompoundName("streaming.groupname");
- private static final CompoundName streamingSelection = new CompoundName("streaming.selection");
+ private static final CompoundName streamingUserid = CompoundName.from("streaming.userid");
+ private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname");
+ private static final CompoundName streamingSelection = CompoundName.from("streaming.selection");
static final String STREAMING_STATISTICS = "streaming.statistics";
private final VisitorFactory visitorFactory;
diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java
index 85ef10b29f6..70dff6730ff 100644
--- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java
+++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java
@@ -51,13 +51,13 @@ import java.util.logging.Level;
*/
class VdsVisitor extends VisitorDataHandler implements Visitor {
- private static final CompoundName streamingUserid=new CompoundName("streaming.userid");
- private static final CompoundName streamingGroupname=new CompoundName("streaming.groupname");
- private static final CompoundName streamingSelection=new CompoundName("streaming.selection");
- private static final CompoundName streamingFromtimestamp=new CompoundName("streaming.fromtimestamp");
- private static final CompoundName streamingTotimestamp=new CompoundName("streaming.totimestamp");
- private static final CompoundName streamingPriority=new CompoundName("streaming.priority");
- private static final CompoundName streamingMaxbucketspervisitor=new CompoundName("streaming.maxbucketspervisitor");
+ private static final CompoundName streamingUserid = CompoundName.from("streaming.userid");
+ private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname");
+ private static final CompoundName streamingSelection = CompoundName.from("streaming.selection");
+ private static final CompoundName streamingFromtimestamp = CompoundName.from("streaming.fromtimestamp");
+ private static final CompoundName streamingTotimestamp = CompoundName.from("streaming.totimestamp");
+ private static final CompoundName streamingPriority = CompoundName.from("streaming.priority");
+ private static final CompoundName streamingMaxbucketspervisitor = CompoundName.from("streaming.maxbucketspervisitor");
protected static final int MAX_BUCKETS_PER_VISITOR = 1024;
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
index 3a6be1521e2..1ff5574ec03 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
@@ -13,6 +13,7 @@ import com.yahoo.prelude.query.parser.Tokenizer;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import static com.yahoo.prelude.query.parser.Token.Kind.COLON;
@@ -28,9 +29,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE;
import static com.yahoo.prelude.query.parser.Token.Kind.STAR;
import static com.yahoo.prelude.query.parser.Token.Kind.UNDERSCORE;
import static com.yahoo.prelude.query.parser.Token.Kind.WORD;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
/**
* Tests the tokenizer
@@ -284,7 +283,7 @@ public class TokenizerTestCase {
sd.addIndex(index2);
IndexFacts facts = new IndexFacts(new IndexModel(sd));
- IndexFacts.Session session = facts.newSession();
+ IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet());
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*& b:c", "default", session);
// tokenizer.print();
@@ -329,7 +328,7 @@ public class TokenizerTestCase {
IndexFacts facts = new IndexFacts(new IndexModel(sd));
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
- IndexFacts.Session session = facts.newSession();
+ IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet());
List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&", session);
assertEquals(new Token(WORD, "normal"), tokens.get(0));
assertEquals(new Token(SPACE, " "), tokens.get(1));
@@ -366,7 +365,7 @@ public class TokenizerTestCase {
IndexFacts facts = new IndexFacts(new IndexModel(sd));
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
- IndexFacts.Session session = facts.newSession();
+ IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet());
List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*", session);
assertEquals(new Token(WORD, "normal"), tokens.get(0));
assertEquals(new Token(SPACE, " "), tokens.get(1));
@@ -403,7 +402,7 @@ public class TokenizerTestCase {
IndexFacts facts = new IndexFacts(new IndexModel(sd));
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
- IndexFacts.Session session = facts.newSession();
+ IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet());
List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:!/%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&b:", session);
assertEquals(new Token(WORD, "normal"), tokens.get(0));
assertEquals(new Token(SPACE, " "), tokens.get(1));
@@ -440,7 +439,7 @@ public class TokenizerTestCase {
sd.addIndex(index2);
IndexFacts indexFacts = new IndexFacts(new IndexModel(sd));
- IndexFacts.Session facts = indexFacts.newSession();
+ IndexFacts.Session facts = indexFacts.newSession(Collections.emptySet(), Collections.emptySet());
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:foo) testexact2:bar", facts);
diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java
index dbcb393c922..e6c5a18c9da 100644
--- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java
@@ -15,12 +15,8 @@ import org.junit.jupiter.api.Test;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
/**
* Tests using synthetic index names for IndexFacts class.
@@ -184,7 +180,7 @@ public class IndexFactsTestCase {
query.getModel().getSources().add("one");
query.getModel().getRestrict().add("two");
- IndexFacts.Session indexFacts = createIndexFacts().newSession(List.of("clusterOne"), Set.of());
+ IndexFacts.Session indexFacts = createIndexFacts().newSession(List.of("clusterOne"), List.of());
assertTrue(indexFacts.isIndex("a"));
assertFalse(indexFacts.isIndex("b"));
assertTrue(indexFacts.isIndex("d"));
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java
index dcb41dc5e31..326c7985a5f 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java
@@ -3,10 +3,8 @@ package com.yahoo.search.query.profile.config.test;
import com.yahoo.jdisc.http.HttpRequest.Method;
import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.language.Language;
import com.yahoo.language.process.Embedder;
import com.yahoo.processing.request.CompoundName;
-import com.yahoo.search.query.profile.types.test.QueryProfileTypeTestCase;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
import com.yahoo.yolean.Exceptions;
@@ -25,7 +23,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
/**
* @author bratseth
@@ -77,8 +79,8 @@ public class XmlReadingTestCase {
CompiledQueryProfile defaultProfile = cRegistry.getComponent("default");
assertNull(defaultProfile.getType());
assertEquals("20", defaultProfile.get("hits"));
- assertFalse(defaultProfile.isOverridable(new CompoundName("hits"), null));
- assertFalse(defaultProfile.isOverridable(new CompoundName("user.trusted"), null));
+ assertFalse(defaultProfile.isOverridable(CompoundName.from("hits"), null));
+ assertFalse(defaultProfile.isOverridable(CompoundName.from("user.trusted"), null));
assertEquals("false", defaultProfile.get("user.trusted"));
CompiledQueryProfile referencingProfile = cRegistry.getComponent("referencingModelSettings");
@@ -97,7 +99,7 @@ public class XmlReadingTestCase {
assertEquals("rootType", rootProfile.getType().getId().getName());
assertEquals(30, rootProfile.get("hits"));
//assertEquals(3, rootProfile.get("traceLevel"));
- assertTrue(rootProfile.isOverridable(new CompoundName("hits"), null));
+ assertTrue(rootProfile.isOverridable(CompoundName.from("hits"), null));
query = new Query(request, rootProfile);
assertEquals(3, query.getTrace().getLevel());
@@ -231,8 +233,8 @@ public class XmlReadingTestCase {
assertEquals("a.b.c-value", new Query("?d1=d1v", profile).properties().get("a.b.c"));
assertEquals("a.b.c-variant-value", new Query("?d1=d1v&d2=d2v", profile).properties().get("a.b.c"));
- assertTrue(profile.isOverridable(new CompoundName("a.b.c"), Map.of("d1", "d1v")));
- assertFalse(profile.isOverridable(new CompoundName("a.b.c"), Map.of("d1", "d1v", "d2", "d2v")));
+ assertTrue(profile.isOverridable(CompoundName.from("a.b.c"), Map.of("d1", "d1v")));
+ assertFalse(profile.isOverridable(CompoundName.from("a.b.c"), Map.of("d1", "d1v", "d2", "d2v")));
}
@Test
@@ -479,18 +481,18 @@ public class XmlReadingTestCase {
QueryProfileType type1 = registry.getTypeRegistry().getComponent("type1");
assertEquals(TensorType.fromSpec("tensor<float>(x[1])"),
- type1.getFieldType(new CompoundName("ranking.features.query(tensor_1)")).asTensorType());
- assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_2)")));
- assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_3)")));
+ type1.getFieldType(CompoundName.from("ranking.features.query(tensor_1)")).asTensorType());
+ assertNull(type1.getFieldType(CompoundName.from("ranking.features.query(tensor_2)")));
+ assertNull(type1.getFieldType(CompoundName.from("ranking.features.query(tensor_3)")));
assertEquals(TensorType.fromSpec("tensor(key{})"),
- type1.getFieldType(new CompoundName("ranking.features.query(tensor_4)")).asTensorType());
+ type1.getFieldType(CompoundName.from("ranking.features.query(tensor_4)")).asTensorType());
QueryProfileType type2 = registry.getTypeRegistry().getComponent("type2");
- assertNull(type2.getFieldType(new CompoundName("ranking.features.query(tensor_1)")));
+ assertNull(type2.getFieldType(CompoundName.from("ranking.features.query(tensor_1)")));
assertEquals(TensorType.fromSpec("tensor<float>(x[2])"),
- type2.getFieldType(new CompoundName("ranking.features.query(tensor_2)")).asTensorType());
+ type2.getFieldType(CompoundName.from("ranking.features.query(tensor_2)")).asTensorType());
assertEquals(TensorType.fromSpec("tensor<float>(x[3])"),
- type2.getFieldType(new CompoundName("ranking.features.query(tensor_3)")).asTensorType());
+ type2.getFieldType(CompoundName.from("ranking.features.query(tensor_3)")).asTensorType());
Query queryProfile1 = new Query.Builder().setQueryProfile(registry.getComponent("profile1"))
.setRequest("?query=test&ranking.features.query(tensor_1)=[1.200]")
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java
index cd5a8e08aa9..080aef25c5f 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java
@@ -65,8 +65,8 @@ public class QueryProfileGetInComplexStructureMicroBenchmark {
Map<String,String> dimensionValues=createDimensionValueMap();
String prefix=generatePrefix();
final int dotInterval=1000000;
- final CompoundName found = new CompoundName(prefix + "a");
- final CompoundName notFound = new CompoundName(prefix + "nonexisting");
+ final CompoundName found = CompoundName.from(prefix + "a");
+ final CompoundName notFound = CompoundName.from(prefix + "nonexisting");
for (int i=0; i<count; i++) {
if (count>dotInterval && i%(dotInterval)==0)
System.out.print(".");
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java
index d38a1d64910..1123feb1b01 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java
@@ -57,8 +57,8 @@ public class QueryProfileGetMicroBenchmark {
private void getValues(int count,Query query) {
final int dotInterval=10000000;
- CompoundName found = new CompoundName(propertyPrefix + "property1");
- CompoundName notFound = new CompoundName(propertyPrefix + "nonExisting");
+ CompoundName found = CompoundName.from(propertyPrefix + "property1");
+ CompoundName notFound = CompoundName.from(propertyPrefix + "nonExisting");
for (int i=0; i<count; i++) {
if (count>dotInterval && i%(count/dotInterval)==0)
System.out.print(".");
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
index cd4cb32df2e..7d5a2137770 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
@@ -534,7 +534,7 @@ public class QueryProfileTestCase {
p.set("a.b", "a.b-value", null);
{
- Map<String, ValueWithSource> values = p.compile(null).listValuesWithSources(new CompoundName(""), new HashMap<>(), null);
+ Map<String, ValueWithSource> values = p.compile(null).listValuesWithSources(CompoundName.empty, new HashMap<>(), null);
assertEquals(2, values.size());
assertEquals("a-value", values.get("a").value());
assertEquals("test", values.get("a").source());
@@ -543,7 +543,7 @@ public class QueryProfileTestCase {
}
{
- Map<String, ValueWithSource> values = p.compile(null).listValuesWithSources(new CompoundName("a"), new HashMap<>(), null);
+ Map<String, ValueWithSource> values = p.compile(null).listValuesWithSources(CompoundName.from("a"), new HashMap<>(), null);
assertEquals(1, values.size());
assertEquals("a.b-value", values.get("b").value());
assertEquals("test", values.get("b").source());
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java
index 1a6cfee75a5..36ff33ffd97 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java
@@ -5,7 +5,6 @@ import ai.vespa.cloud.ApplicationId;
import ai.vespa.cloud.Environment;
import ai.vespa.cloud.Zone;
import ai.vespa.cloud.ZoneInfo;
-import com.yahoo.jdisc.application.Application;
import com.yahoo.jdisc.http.HttpRequest.Method;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.processing.request.CompoundName;
@@ -28,7 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* @author bratseth
@@ -133,7 +134,7 @@ public class QueryProfileVariantsTestCase {
base.set("a.b", 1, new String[]{null, null, "d3-val"}, registry);
QueryProfileVariant aVariants = base.getVariants().getVariants().get(0);
assertEquals("[d1, d2, d3]",
- ((QueryProfile) base.getVariants().getVariants().get(0).values().get("a")).getDimensions().toString(),
+ ((QueryProfile) aVariants.values().get("a")).getDimensions().toString(),
"Variant dimensions are not overridden by the referenced dimensions");
}
@@ -184,7 +185,7 @@ public class QueryProfileVariantsTestCase {
{
Map<String, ValueWithSource> values = cRegistry.findQueryProfile("test")
- .listValuesWithSources(new CompoundName(""),
+ .listValuesWithSources(CompoundName.empty,
new HashMap<>(),
null);
assertEquals(1, values.size());
@@ -195,7 +196,7 @@ public class QueryProfileVariantsTestCase {
{
Map<String, ValueWithSource> values = cRegistry.findQueryProfile("test")
- .listValuesWithSources(new CompoundName(""),
+ .listValuesWithSources(CompoundName.empty,
toMap("x=x1", "y=y1", "z=z1"),
null);
assertEquals(2, values.size());
@@ -1456,7 +1457,7 @@ public class QueryProfileVariantsTestCase {
return context;
}
- public static final Map<String, String> toMap(String... bindings) {
+ public static Map<String, String> toMap(String... bindings) {
Map<String, String> context = new HashMap<>();
for (String binding : bindings) {
String[] entry = binding.split("=");
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java
index 37991f7f14f..982ebd80fbd 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java
@@ -55,7 +55,7 @@ public class NameTestCase {
@Test
void testComponentIdAsCompoundName() {
String name = "a/b";
- assertEquals(new CompoundName(name), new QueryProfileType(name).getComponentIdAsCompoundName());
+ assertEquals(CompoundName.from(name), new QueryProfileType(name).getComponentIdAsCompoundName());
}
private void assertLegalName(String name) {
@@ -89,10 +89,6 @@ public class NameTestCase {
}
}
- private void assertIllegalFieldName(String name) {
- assertIllegalFieldName(name,"Could not set '" + name + "' to 'anyValue'","Illegal name '" + name + "'");
- }
-
/** Checks that this is illegal both for profiles and types */
private void assertIllegalFieldName(String name, String expectedHighError, String expectedLowError) {
try {
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
index 21a9b2fe399..4000c48bfed 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
@@ -761,7 +761,7 @@ public class QueryProfileTypeTestCase {
}
private void assertNotPermitted(QueryProfile profile,String name,Object value) {
- String localName = new CompoundName(name).last();
+ String localName = CompoundName.from(name).last();
try {
profile.set(name, value, registry);
fail("Should fail setting " + name + " to " + value);
diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java
index 0e9bf8d41ec..6b25dc526d9 100644
--- a/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.query.properties.test;
+package com.yahoo.search.query.properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -9,11 +9,12 @@ import java.util.HashSet;
import com.yahoo.processing.request.properties.PropertyMap;
import org.junit.jupiter.api.Test;
-import com.yahoo.search.query.properties.SubProperties;
/**
* @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
*/
+@SuppressWarnings({"removal"})
+
public class SubPropertiesTestCase {
@Test
diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
index 748a20801d1..63655da0784 100644
--- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
@@ -2,7 +2,6 @@
package com.yahoo.search.test;
import com.yahoo.component.chain.Chain;
-import com.yahoo.data.JsonProducer;
import com.yahoo.language.Language;
import com.yahoo.language.Linguistics;
import com.yahoo.language.detect.Detection;
@@ -16,7 +15,6 @@ import com.yahoo.prelude.Index;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.IndexModel;
import com.yahoo.prelude.SearchDefinition;
-import com.yahoo.prelude.fastsearch.FastHit;
import com.yahoo.prelude.query.AndItem;
import com.yahoo.prelude.query.AndSegmentItem;
import com.yahoo.prelude.query.CompositeItem;
@@ -145,7 +143,7 @@ public class QueryTestCase {
@Test
void testCloneWithConnectivity() {
List<String> l = List.of("a", "b", "c", "a");
- printIt(l.stream().filter(i -> isA(i)).toList());
+ printIt(l.stream().filter(this::isA).toList());
printIt(l.stream().filter(i -> !isA(i)).toList());
Query q = new Query();
@@ -314,7 +312,7 @@ public class QueryTestCase {
profile.set("myField", "Profile: %{queryProfile}", null);
Query query = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null));
- String source = query.properties().getInstance(com.yahoo.search.query.profile.QueryProfileProperties.class).getQueryProfile().listValuesWithSources(new CompoundName(""), query.getHttpRequest().propertyMap(), query.properties()).get("myField").source();
+ String source = query.properties().getInstance(com.yahoo.search.query.profile.QueryProfileProperties.class).getQueryProfile().listValuesWithSources(CompoundName.empty, query.getHttpRequest().propertyMap(), query.properties()).get("myField").source();
assertEquals("myProfile", source);
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java
index ecce9b7ccfa..a389f771db6 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java
@@ -42,11 +42,15 @@ public class PlanRegistryMock implements PlanRegistry {
private final boolean supported;
public MockPlan(String planId, boolean billed, boolean supported, double cpuPrice, double memPrice, double dgbPrice, int quota, String description) {
- this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> Quota.unlimited().withBudget(quota), description);
+ this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> createQuota(quota), description);
+ }
+
+ private static Quota createQuota(int quota) {
+ return quota == 0 ? Quota.zero() : Quota.unlimited().withBudget(quota);
}
public MockPlan(String planId, boolean billed, boolean supported, String cpuPrice, String memPrice, String dgbPrice, int quota, String description) {
- this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> Quota.unlimited().withBudget(quota), description);
+ this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> createQuota(quota), description);
}
public MockPlan(PlanId planId, boolean billed, boolean supported, MockCostCalculator calculator, QuotaCalculator quota, String description) {
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java
index ecea1ce6913..cff61f1a50a 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java
@@ -69,7 +69,9 @@ public class EndpointCertificateValidatorImpl implements EndpointCertificateVali
// Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner
throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store");
} catch (EndpointCertificateException e) {
- log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e);
+ if (!e.type().equals(EndpointCertificateException.Type.CERT_NOT_AVAILABLE)) { // such failures are normal and will be retried, it takes some time to show up in the secret store
+ log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e);
+ }
throw e;
} catch (Exception e) {
log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e);
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
index 945e0730fe6..34a7c1a6f7c 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
@@ -113,6 +113,8 @@ public class NodeRepositoryNode {
private String cloudAccount;
@JsonProperty("wireguardPubKey")
private String wireguardPubKey;
+ @JsonProperty("archiveUri")
+ private String archiveUri;
public String getUrl() {
return url;
@@ -460,6 +462,10 @@ public class NodeRepositoryNode {
public void setWireguardPubKey(String wireguardPubKey) { this.wireguardPubKey = wireguardPubKey; }
+ public String getArchiveUri() { return archiveUri; }
+
+ public void setArchiveUri(String archiveUri) { this.archiveUri = archiveUri; }
+
// --- Helper methods for code that (wrongly) consume this directly
public boolean hasType(NodeType type) {
@@ -521,6 +527,7 @@ public class NodeRepositoryNode {
", switchHostname='" + switchHostname + '\'' +
", cloudAccount='" + cloudAccount + '\'' +
", wireguardPubKey='" + wireguardPubKey + '\'' +
+ ", archiveUri='" + archiveUri + '\'' +
'}';
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index a1534ebc533..08a8440fbe2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan;
import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing;
@@ -47,6 +48,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter;
import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore;
+import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics.Warning;
@@ -60,6 +62,7 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValid
import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade;
import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates;
import com.yahoo.vespa.hosted.controller.concurrent.Once;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
import com.yahoo.vespa.hosted.controller.deployment.Run;
@@ -76,7 +79,6 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
import com.yahoo.yolean.Exceptions;
-
import java.io.ByteArrayInputStream;
import java.security.Principal;
import java.security.cert.X509Certificate;
@@ -586,7 +588,16 @@ public class ApplicationController {
}
// Validate new deployment spec thoroughly before storing it.
- controller.jobController().deploymentStatus(application.get());
+ DeploymentStatus status = controller.jobController().deploymentStatus(application.get());
+ Change dummyChange = Change.of(RevisionId.forProduction(Long.MAX_VALUE)); // Should always run everywhere.
+ for (var jobs : status.jobsToRun(applicationPackage.deploymentSpec().instanceNames().stream()
+ .collect(toMap(name -> name, __ -> dummyChange)))
+ .entrySet()) {
+ for (var job : jobs.getValue()) {
+ decideCloudAccountOf(new DeploymentId(jobs.getKey().application(), job.type().zone()),
+ applicationPackage.deploymentSpec());
+ }
+ }
for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) {
if ( notification.source().instance().isPresent()
@@ -696,7 +707,7 @@ public class ApplicationController {
throw new IllegalArgumentException("Requested cloud account '" + requestedAccount.get().value() +
"' is not valid for tenant '" + tenant + "'");
}
- if (!controller.zoneRegistry().hasZone(zoneId, requestedAccount.get())) {
+ if ( ! controller.zoneRegistry().hasZone(zoneId, requestedAccount.get())) {
throw new IllegalArgumentException("Zone " + zoneId + " is not configured in requested cloud account '" +
requestedAccount.get().value() + "'");
}
@@ -1049,4 +1060,14 @@ public class ApplicationController {
collectingAndThen(counting(), Long::intValue)));
}
+ public void verifyPlan(TenantName tenantName) {
+ var planId = controller.serviceRegistry().billingController().getPlan(tenantName);
+ Optional<Plan> plan = controller.serviceRegistry().planRegistry().plan(planId);
+ if (plan.isEmpty())
+ throw new IllegalArgumentException("Tenant '" + tenantName.value() + "' has no plan, not allowed to deploy. See https://cloud.vespa.ai/support");
+ if (plan.get().quota().calculate().equals(Quota.zero()))
+ throw new IllegalArgumentException("Tenant '" + tenantName.value() + "' has a plan '" +
+ plan.get().displayName() + "' with zero quota, not allowed to deploy. See https://cloud.vespa.ai/support");
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index 1eb68c14353..80b52e0c7a4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -31,7 +31,8 @@ import static java.util.Comparator.comparing;
*/
public class Endpoint {
- private static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud";
+ private static final String MAIN_OATH_DNS_SUFFIX = ".vespa.oath.cloud";
+ private static final String CD_OATH_DNS_SUFFIX = ".cd.vespa.oath.cloud";
private static final String PUBLIC_DNS_SUFFIX = ".vespa-app.cloud";
private static final String PUBLIC_CD_DNS_SUFFIX = ".cd.vespa-app.cloud";
@@ -243,18 +244,13 @@ public class Endpoint {
/** Returns the DNS suffix used for endpoints in given system */
private static String dnsSuffix(SystemName system) {
- switch (system) {
- case cd, main -> {
- return OATH_DNS_SUFFIX;
- }
- case Public -> {
- return PUBLIC_DNS_SUFFIX;
- }
- case PublicCd -> {
- return PUBLIC_CD_DNS_SUFFIX;
- }
+ return switch (system) {
+ case cd -> CD_OATH_DNS_SUFFIX;
+ case main -> MAIN_OATH_DNS_SUFFIX;
+ case Public -> PUBLIC_DNS_SUFFIX;
+ case PublicCd -> PUBLIC_CD_DNS_SUFFIX;
default -> throw new IllegalArgumentException("No DNS suffix declared for system " + system);
- }
+ };
}
/** Returns the DNS suffix used for internal names (i.e. names not exposed to tenants) in given system */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
index 2441da19b90..c1bf083b26c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
@@ -21,7 +21,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.OptionalInt;
import java.util.function.Function;
import static java.util.Comparator.comparing;
@@ -81,8 +80,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL
/** Returns the subset of instances that are allowed to upgrade to the given version at the given time */
public InstanceList canUpgradeAt(Version version, Instant instant) {
return matching(id -> instances.get(id).instanceSteps().get(id.instance())
- .readyAt(Change.of(version))
- .map(readyAt -> ! readyAt.isAfter(instant)).orElse(false));
+ .readiness(Change.of(version)).okAt(instant));
}
/** Returns the subset of instances which have at least one production deployment */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java
index f86073cfb25..3880b028eb0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java
@@ -5,6 +5,7 @@ import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
+import java.nio.file.attribute.FileTime;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
@@ -32,6 +33,7 @@ public class ApplicationPackageStream {
private final Supplier<Predicate<String>> filter;
private final Supplier<InputStream> in;
private final AtomicReference<ApplicationPackage> truncatedPackage = new AtomicReference<>();
+ private final FileTime createdAt = FileTime.fromMillis(System.currentTimeMillis());
/** Stream that effectively copies the input stream to its {@link #truncatedPackage()} when exhausted. */
public ApplicationPackageStream(Supplier<InputStream> in) {
@@ -60,7 +62,7 @@ public class ApplicationPackageStream {
* and the first to be exhausted will populate the truncated application package.
*/
public InputStream zipStream() {
- return new Stream(in.get(), replacer.get(), filter.get(), truncatedPackage);
+ return new Stream(in.get(), replacer.get(), filter.get(), createdAt, truncatedPackage);
}
/**
@@ -85,6 +87,7 @@ public class ApplicationPackageStream {
private final ZipInputStream inZip;
private final Replacer replacer;
private final Predicate<String> filter;
+ private final FileTime createdAt;
private byte[] currentOut = new byte[0];
private InputStream currentIn = InputStream.nullInputStream();
private boolean includeCurrent = false;
@@ -92,11 +95,12 @@ public class ApplicationPackageStream {
private boolean closed = false;
private boolean done = false;
- private Stream(InputStream in, Replacer replacer, Predicate<String> filter, AtomicReference<ApplicationPackage> truncatedPackage) {
+ private Stream(InputStream in, Replacer replacer, Predicate<String> filter, FileTime createdAt, AtomicReference<ApplicationPackage> truncatedPackage) {
this.in = in;
this.inZip = new ZipInputStream(in);
this.replacer = replacer;
this.filter = filter;
+ this.createdAt = createdAt;
this.truncatedPackage = truncatedPackage;
}
@@ -129,10 +133,12 @@ public class ApplicationPackageStream {
ZipEntry next = inZip.getNextEntry();
String name;
+ FileTime modifiedAt;
InputStream content = null;
if (next == null) {
// We may still have replacements to fill in, but if we don't, we're done filling, forever!
name = replacer.next();
+ modifiedAt = createdAt;
if (name == null) {
outZip.close(); // This typically makes new output available, so must check for that after this.
teeZip.close();
@@ -144,6 +150,7 @@ public class ApplicationPackageStream {
}
else {
name = next.getName();
+ modifiedAt = next.getLastModifiedTime();
content = new FilterInputStream(inZip) { @Override public void close() { } }; // Protect inZip from replacements closing it.
}
@@ -153,8 +160,8 @@ public class ApplicationPackageStream {
currentIn = InputStream.nullInputStream();
}
else {
- if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name));
- outZip.putNextEntry(new ZipEntry(name));
+ if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name) {{ setLastModifiedTime(modifiedAt); }});
+ outZip.putNextEntry(new ZipEntry(name) {{ setLastModifiedTime(modifiedAt); }});
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java
index aa6e3b0c44d..cbd0f685d80 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java
@@ -49,7 +49,7 @@ public record AuditLog(List<Entry> entries) {
public record Entry(Instant at, String principal, Method method, String resource, Optional<String> data,
Client client) implements Comparable<Entry> {
- private final static int maxDataLength = 1024;
+ final static int maxDataLength = 1024;
private final static Comparator<Entry> comparator = Comparator.comparing(Entry::at).reversed();
public Entry(Instant at, Client client, String principal, Method method, String resource, byte[] data) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java
index 033cd0a52c9..13b3d9d170f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java
@@ -4,11 +4,12 @@ package com.yahoo.vespa.hosted.controller.auditlog;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.jdisc.http.HttpHeaders;
import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.hosted.controller.auditlog.AuditLog.Entry;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.UncheckedIOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
import java.net.URI;
import java.security.Principal;
import java.time.Clock;
@@ -17,6 +18,9 @@ import java.time.Instant;
import java.util.Objects;
import java.util.Optional;
+import static com.yahoo.yolean.Exceptions.uncheck;
+import static java.util.Objects.requireNonNullElse;
+
/**
* This provides read and write operations for the audit log.
*
@@ -58,14 +62,8 @@ public class AuditLogger {
"misconfiguration and should not happen");
}
- byte[] data = new byte[0];
- try {
- if (request.getData() != null) {
- data = request.getData().readAllBytes();
- }
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
+ InputStream requestData = requireNonNullElse(request.getData(), InputStream.nullInputStream());
+ byte[] data = uncheck(() -> requestData.readNBytes(Entry.maxDataLength));
AuditLog.Entry.Client client = parseClient(request);
Instant now = clock.instant();
@@ -80,7 +78,9 @@ public class AuditLogger {
}
// Create a new input stream to allow callers to consume request body
- return new HttpRequest(request.getJDiscRequest(), new ByteArrayInputStream(data), request.propertyMap());
+ return new HttpRequest(request.getJDiscRequest(),
+ new SequenceInputStream(new ByteArrayInputStream(data), requestData),
+ request.propertyMap());
}
private static AuditLog.Entry.Client parseClient(HttpRequest request) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index a76e76611c2..00da34fe2e4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -39,11 +39,11 @@ import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@@ -303,7 +303,11 @@ public class DeploymentStatus {
fallbackPlatform(change, job));
if (step.completedAt(change, firstProductionJobWithDeploymentInCloud).isEmpty()) {
JobType typeWithZone = job.type().isSystemTest() ? JobType.systemTest(zones, cloud) : JobType.stagingTest(zones, cloud);
- jobs.merge(job, List.of(new Job(typeWithZone, versions, step.readyAt(change), change)), DeploymentStatus::union);
+ Readiness readiness = step.readiness(change, firstProductionJobWithDeploymentInCloud);
+ jobs.merge(job, List.of(new Job(typeWithZone,
+ versions,
+ readiness.okAt(now) && jobs().get(job).get().isRunning() ? readiness.running() : readiness,
+ change)), DeploymentStatus::union);
}
});
});
@@ -390,7 +394,7 @@ public class DeploymentStatus {
}
/** The set of jobs that need to run for the given changes to be considered complete. */
- private Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) {
+ public Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) {
return jobsToRun(changes, false);
}
@@ -498,21 +502,23 @@ public class DeploymentStatus {
}
/** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */
- public Optional<Instant> verifiedAt(JobId job, Versions versions) {
- Optional<Instant> triggeredAt = allJobs.get(job)
- .flatMap(status -> status.runs().values().stream()
- .filter(run -> run.versions().equals(versions))
- .findFirst())
- .map(Run::start);
- Optional<Instant> systemTestedAt = testedAt(job, systemTest(job.type()), versions);
- Optional<Instant> stagingTestedAt = testedAt(job, stagingTest(job.type()), versions);
- if (systemTestedAt.isEmpty() || stagingTestedAt.isEmpty()) return triggeredAt;
- Optional<Instant> testedAt = systemTestedAt.get().isAfter(stagingTestedAt.get()) ? systemTestedAt : stagingTestedAt;
- return triggeredAt.isPresent() && triggeredAt.get().isBefore(testedAt.get()) ? triggeredAt : testedAt;
+ public Readiness verifiedAt(JobId job, Versions versions) {
+ Readiness triggered = allJobs.get(job)
+ .flatMap(status -> status.runs().values().stream()
+ .filter(run -> run.versions().equals(versions))
+ .findFirst())
+ .map(Run::start)
+ .map(Readiness::new)
+ .orElse(Readiness.unverified);
+ Readiness systemTested = testedAt(job, systemTest(job.type()), versions);
+ Readiness stagingTested = testedAt(job, stagingTest(job.type()), versions);
+ if (! systemTested.ok() || ! stagingTested.ok()) return triggered;
+ Readiness tested = min(systemTested, stagingTested);
+ return triggered.ok() && triggered.at().isBefore(tested.at) ? triggered : tested;
}
/** Earliest instant when versions were tested for the given instance. */
- private Optional<Instant> testedAt(JobId job, JobType type, Versions versions) {
+ private Readiness testedAt(JobId job, JobType type, Versions versions) {
return prerequisiteTests(job, type).stream()
.map(test -> allJobs.get(test).stream()
.flatMap(status -> RunList.from(status)
@@ -522,19 +528,21 @@ public class DeploymentStatus {
.asList().stream()
.map(run -> run.end().get()))
.min(naturalOrder()))
- .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : o.get().isBefore(n.get()) ? n : o)
- .orElse(Optional.empty());
+ .map(testedAt -> testedAt.map(Readiness::new).orElse(Readiness.unverified))
+ .reduce(Readiness.empty, DeploymentStatus::max);
}
private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) {
Map<JobId, List<Job>> jobs = new LinkedHashMap<>();
- jobSteps.forEach((job, step) -> {
+ for (Entry<JobId, StepStatus> entry : reversed(List.copyOf(jobSteps.entrySet()))) {
+ JobId job = entry.getKey();
+ StepStatus step = entry.getValue();
if ( ! job.application().instance().equals(instance) || ! job.type().isProduction())
- return;
+ continue;
// Signal strict completion criterion by depending on job itself.
if (step.completedAt(change, Optional.of(job)).isPresent())
- return;
+ continue;
// When computing eager test jobs for outstanding changes, assume current change completes successfully.
Optional<Deployment> deployment = deploymentFor(job);
@@ -544,7 +552,7 @@ public class DeploymentStatus {
|| areIncompatible(change.platform(), existingRevision, job);
if (assumeUpgradesSucceed) {
if (deployingCompatibilityChange) // No eager tests for this.
- return;
+ continue;
Change currentChange = application.require(instance).change();
Versions target = Versions.from(currentChange, application, deployment, fallbackPlatform(currentChange, job));
@@ -554,21 +562,28 @@ public class DeploymentStatus {
List<Job> toRun = new ArrayList<>();
List<Change> changes = deployingCompatibilityChange
|| allJobs.get(job).flatMap(status -> status.lastCompleted()).isEmpty()
- ? List.of(change)
- : changes(job, step, change);
+ ? List.of(change)
+ : changes(job, step, change);
for (Change partial : changes) {
- Job jobToRun = new Job(job.type(),
- Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job)),
- step.readyAt(partial, Optional.of(job)),
- partial);
- toRun.add(jobToRun);
+ Versions versions = Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job));
+ Readiness readiness = step.readiness(partial, Optional.of(job));
+ // This job is blocked if it is already running ...
+ readiness = jobs().get(job).get().isRunning() && readiness.okAt(now) ? readiness.running() : readiness;
+ // ... or if it is a deployment, and a test job for the current state is not yet complete,
+ // which is the case when the next versions to run that test with is not the same as we want to deploy here.
+ List<Job> tests = job.type().isTest() ? null : jobs.get(new JobId(job.application(), JobType.productionTestOf(job.type().zone())));
+ readiness = tests != null && ! versions.targetsMatch(tests.get(0).versions) && readiness.okAt(now) ? readiness.blocked() : readiness;
+ toRun.add(new Job(job.type(), versions, readiness, partial));
// Assume first partial change is applied before the second.
- existingPlatform = Optional.of(jobToRun.versions.targetPlatform());
- existingRevision = Optional.of(jobToRun.versions.targetRevision());
+ existingPlatform = Optional.of(versions.targetPlatform());
+ existingRevision = Optional.of(versions.targetRevision());
}
jobs.put(job, toRun);
- });
- return jobs;
+ }
+ Map<JobId, List<Job>> jobsInOrder = new LinkedHashMap<>();
+ for (Entry<JobId, List<Job>> entry : reversed(List.copyOf(jobs.entrySet())))
+ jobsInOrder.put(entry.getKey(), entry.getValue());
+ return jobsInOrder;
}
private boolean areIncompatible(Optional<Version> platform, Optional<RevisionId> revision, JobId job) {
@@ -606,8 +621,7 @@ public class DeploymentStatus {
// the revision is now blocked by waiting for the production test to verify the upgrade.
// In this case we must abandon the production test on the pure upgrade, so the revision can be deployed.
if (platformDeployedAt.isPresent() && revisionDeployedAt.isEmpty()) {
- if (jobSteps.get(deployment).readyAt(change, Optional.of(deployment))
- .map(ready -> ! now.isBefore(ready)).orElse(false)) {
+ if (jobSteps.get(deployment).readiness(change, Optional.of(deployment)).okAt(now)) {
return switch (rollout) {
// If separate rollout, this test should keep blocking the revision, unless there are failures.
case separate -> hasFailures(jobSteps.get(deployment), jobSteps.get(job)) ? List.of(change) : List.of(change.withoutApplication(), change);
@@ -676,12 +690,15 @@ public class DeploymentStatus {
for (Job productionJob : versionsList)
if (allJobs.successOn(testType, productionJob.versions())
.instance(testJob.application().instance())
- .asList().isEmpty())
+ .asList().isEmpty()) {
+ Readiness readiness = jobSteps().get(testJob).readiness(productionJob.change, Optional.of(job));
testJobs.merge(testJob, List.of(new Job(testJob.type(),
productionJob.versions(),
- jobSteps().get(testJob).readyAt(productionJob.change),
+ readiness.okAt(now) && jobs().get(testJob).get().isRunning() ? readiness.running() : readiness,
productionJob.change)),
DeploymentStatus::union);
+
+ }
});
}
}
@@ -894,16 +911,17 @@ public class DeploymentStatus {
abstract Optional<Instant> completedAt(Change change, Optional<JobId> dependent);
/** The time at which this step is ready to run the specified change and / or versions. */
- public Optional<Instant> readyAt(Change change) { return readyAt(change, Optional.empty()); }
+ public Readiness readiness(Change change) { return readiness(change, Optional.empty()); }
/** The time at which this step is ready to run the specified change and / or versions. */
- Optional<Instant> readyAt(Change change, Optional<JobId> dependent) {
+ Readiness readiness(Change change, Optional<JobId> dependent) {
return dependenciesCompletedAt(change, dependent)
+ .map(Readiness::new)
.map(ready -> Stream.of(blockedUntil(change),
pausedUntil(),
coolingDownUntil(change, dependent))
- .flatMap(Optional::stream)
- .reduce(ready, maxBy(naturalOrder())));
+ .reduce(ready, maxBy(naturalOrder())))
+ .orElse(Readiness.notReady);
}
/** The time at which all dependencies completed on the given change and / or versions. */
@@ -918,13 +936,13 @@ public class DeploymentStatus {
}
/** The time until which this step is blocked by a change blocker. */
- public Optional<Instant> blockedUntil(Change change) { return Optional.empty(); }
+ public Readiness blockedUntil(Change change) { return Readiness.empty; }
/** The time until which this step is paused by user intervention. */
- public Optional<Instant> pausedUntil() { return Optional.empty(); }
+ public Readiness pausedUntil() { return Readiness.empty; }
/** The time until which this step is cooling down, due to consecutive failures. */
- public Optional<Instant> coolingDownUntil(Change change, Optional<JobId> dependent) { return Optional.empty(); }
+ public Readiness coolingDownUntil(Change change, Optional<JobId> dependent) { return Readiness.empty; }
/** Whether this step is declared in the deployment spec, or is an implicit step. */
public boolean isDeclared() { return true; }
@@ -940,7 +958,8 @@ public class DeploymentStatus {
@Override
Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
- return readyAt(change, dependent).map(completion -> completion.plus(step().delay()));
+ return Optional.ofNullable(readiness(change, dependent).at())
+ .map(completion -> completion.plus(step().delay()));
}
}
@@ -964,12 +983,12 @@ public class DeploymentStatus {
/** The time at which this step is ready to run the specified change and / or versions. */
@Override
- public Optional<Instant> readyAt(Change change) {
+ public Readiness readiness(Change change) {
return status.jobSteps.keySet().stream()
.filter(job -> job.type().isProduction() && job.application().instance().equals(instance.name()))
- .map(job -> super.readyAt(change, Optional.of(job)))
- .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : n.get().isBefore(o.get()) ? n : o)
- .orElseGet(() -> super.readyAt(change, Optional.empty()));
+ .map(job -> super.readiness(change, Optional.of(job)))
+ .reduce((a, b) -> ! a.ok() ? a : ! b.ok() ? b : min(a, b))
+ .orElseGet(() -> super.readiness(change, Optional.empty()));
}
/**
@@ -986,7 +1005,7 @@ public class DeploymentStatus {
}
@Override
- public Optional<Instant> blockedUntil(Change change) {
+ public Readiness blockedUntil(Change change) {
for (Instant current = now; now.plus(Duration.ofDays(7)).isAfter(current); ) {
boolean blocked = false;
for (DeploymentSpec.ChangeBlocker blocker : spec.changeBlocker()) {
@@ -999,9 +1018,9 @@ public class DeploymentStatus {
}
}
if ( ! blocked)
- return current == now ? Optional.empty() : Optional.of(current);
+ return current == now ? Readiness.empty : new Readiness(current, DelayCause.changeBlocked);
}
- return Optional.of(now.plusSeconds(1 << 30)); // Some time in the future that doesn't look like anything you'd expect.
+ return new Readiness(now.plusSeconds(1 << 30), DelayCause.changeBlocked); // Some time in the future that doesn't look like anything you'd expect.
}
}
@@ -1023,31 +1042,34 @@ public class DeploymentStatus {
public Optional<JobId> job() { return Optional.of(job.id()); }
@Override
- public Optional<Instant> pausedUntil() {
- return status.application().require(job.id().application().instance()).jobPause(job.id().type());
+ public Readiness pausedUntil() {
+ return status.application().require(job.id().application().instance()).jobPause(job.id().type())
+ .map(pause -> new Readiness(pause, DelayCause.paused))
+ .orElse(Readiness.empty);
}
@Override
- public Optional<Instant> coolingDownUntil(Change change, Optional<JobId> dependent) {
- if (job.lastTriggered().isEmpty()) return Optional.empty();
- if (job.lastCompleted().isEmpty()) return Optional.empty();
- if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Optional.empty();
+ public Readiness coolingDownUntil(Change change, Optional<JobId> dependent) {
+ if (job.lastTriggered().isEmpty()) return Readiness.empty;
+ if (job.lastCompleted().isEmpty()) return Readiness.empty;
+ if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Readiness.empty;
Versions lastVersions = job.lastCompleted().get().versions();
Versions toRun = Versions.from(change, status.application, dependent.flatMap(status::deploymentFor), status.fallbackPlatform(change, job.id()));
- if ( ! toRun.targetsMatch(lastVersions)) return Optional.empty();
+ if ( ! toRun.targetsMatch(lastVersions)) return Readiness.empty;
if ( job.id().type().environment().isTest()
&& ! dependent.map(JobId::type).map(status::findCloud).map(List.of(CloudName.AWS, CloudName.GCP)::contains).orElse(true)
- && job.isNodeAllocationFailure()) return Optional.empty();
+ && job.isNodeAllocationFailure()) return Readiness.empty;
- if (job.lastStatus().get() == invalidApplication) return Optional.of(status.now.plus(Duration.ofDays(36524))); // 100 years
+ if (job.lastStatus().get() == invalidApplication) return new Readiness(status.now.plus(Duration.ofSeconds(1 << 30)), DelayCause.invalidPackage);
Instant firstFailing = job.firstFailing().get().end().get();
Instant lastCompleted = job.lastCompleted().get().end().get();
- return firstFailing.equals(lastCompleted) ? Optional.of(lastCompleted)
- : Optional.of(lastCompleted.plus(Duration.ofMinutes(10))
- .plus(Duration.between(firstFailing, lastCompleted)
- .dividedBy(2)))
- .filter(status.now::isBefore);
+ Duration penalty = firstFailing.equals(lastCompleted) ? Duration.ZERO
+ : Duration.ofMinutes(10)
+ .plus(Duration.between(firstFailing, lastCompleted)
+ .dividedBy(2));
+ return lastCompleted.plus(penalty).isAfter(status.now) ? new Readiness(lastCompleted.plus(penalty), DelayCause.coolingDown)
+ : Readiness.empty;
}
private static JobStepStatus ofProductionDeployment(DeclaredZone step, List<StepStatus> dependencies,
@@ -1059,11 +1081,10 @@ public class DeploymentStatus {
return new JobStepStatus(StepType.deployment, step, dependencies, job, status) {
@Override
- public Optional<Instant> readyAt(Change change, Optional<JobId> dependent) {
- Optional<Instant> readyAt = super.readyAt(change, dependent);
- Optional<Instant> testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.fallbackPlatform(change, job.id())));
- if (readyAt.isEmpty() || testedAt.isEmpty()) return Optional.empty();
- return readyAt.get().isAfter(testedAt.get()) ? readyAt : testedAt;
+ public Readiness readiness(Change change, Optional<JobId> dependent) {
+ Readiness readyAt = super.readiness(change, dependent);
+ Readiness testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.fallbackPlatform(change, job.id())));
+ return max(readyAt, testedAt);
}
/** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */
@@ -1103,11 +1124,11 @@ public class DeploymentStatus {
JobId prodId = new JobId(job.id().application(), JobType.deploymentTo(job.id().type().zone()));
return new JobStepStatus(StepType.test, step, dependencies, job, status) {
@Override
- Optional<Instant> readyAt(Change change, Optional<JobId> dependent) {
- Optional<Instant> readyAt = super.readyAt(change, dependent);
- Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId));
- if (readyAt.isEmpty() || deployedAt.isEmpty()) return Optional.empty();
- return readyAt.get().isAfter(deployedAt.get()) ? readyAt : deployedAt;
+ Readiness readiness(Change change, Optional<JobId> dependent) {
+ Readiness readyAt = super.readiness(change, dependent);
+ Readiness deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId))
+ .map(Readiness::new).orElse(Readiness.notReady);
+ return max(readyAt, deployedAt);
}
@Override
@@ -1163,13 +1184,13 @@ public class DeploymentStatus {
private final JobType type;
private final Versions versions;
- private final Optional<Instant> readyAt;
+ private final Readiness readiness;
private final Change change;
- public Job(JobType type, Versions versions, Optional<Instant> readyAt, Change change) {
+ public Job(JobType type, Versions versions, Readiness readiness, Change change) {
this.type = type;
this.versions = type.isSystemTest() ? versions.withoutSources() : versions;
- this.readyAt = readyAt;
+ this.readiness = readiness;
this.change = change;
}
@@ -1181,8 +1202,8 @@ public class DeploymentStatus {
return versions;
}
- public Optional<Instant> readyAt() {
- return readyAt;
+ public Readiness readiness() {
+ return readiness;
}
@Override
@@ -1190,19 +1211,60 @@ public class DeploymentStatus {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Job job = (Job) o;
- return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change);
+ return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readiness.equals(job.readiness) && change.equals(job.change);
}
@Override
public int hashCode() {
- return Objects.hash(type.zone(), versions, readyAt, change);
+ return Objects.hash(type.zone(), versions, readiness, change);
}
@Override
public String toString() {
- return change + " with versions " + versions + ", ready at " + readyAt;
+ return change + " with versions " + versions + ", " + readiness;
}
}
+ public enum DelayCause { none, unverified, notReady, blocked, running, coolingDown, invalidPackage, changeBlocked, paused }
+ public record Readiness(Instant at, DelayCause cause) implements Comparable<Readiness> {
+ public static final Readiness unverified = new Readiness(null, DelayCause.unverified);
+ public static final Readiness notReady = new Readiness(null, DelayCause.notReady);
+ public static final Readiness empty = new Readiness(Instant.EPOCH, DelayCause.none);
+ public Readiness(Instant at) { this(at, DelayCause.none); }
+ public Readiness blocked() { return new Readiness(at, DelayCause.blocked); }
+ public Readiness running() { return new Readiness(at, DelayCause.running); }
+ public boolean ok() { return at != null; }
+ public boolean okAt(Instant at) { return ok() && cause != DelayCause.running && cause != DelayCause.blocked && ! at.isBefore(this.at); }
+ @Override public int compareTo(Readiness o) {
+ return at == null ? o.at == null ? 0 : 1
+ : o.at == null ? -1 : at.compareTo(o.at);
+ }
+ @Override public String toString() {
+ return ok() ? "ready at " + at + switch (cause) {
+ case none -> "";
+ case coolingDown -> ": cooling down after repeated failures";
+ case blocked -> ": waiting for verification test to complete";
+ case running -> ": waiting for current run to complete";
+ case invalidPackage -> ": invalid application package, must resubmit";
+ case changeBlocked -> ": deployment configuration blocks changes";
+ case paused -> ": manually paused";
+ default -> throw new IllegalStateException(cause + " should not have an instant at which it is ready");
+ }
+ : "not ready" + switch (cause) {
+ case unverified -> ": waiting for verification test to complete";
+ case notReady -> ": waiting for dependencies to complete";
+ default -> throw new IllegalStateException(cause + " should have an instant at which it is ready");
+ };
+ }
+ }
+
+ static <T extends Comparable<T>> T min(T a, T b) {
+ return a.compareTo(b) > 0 ? b : a;
+ }
+
+ static <T extends Comparable<T>> T max(T a, T b) {
+ return a.compareTo(b) < 0 ? b : a;
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index a5cb839e9c9..00a0e22f87d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -20,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList;
import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness;
import java.math.BigDecimal;
import java.time.Clock;
@@ -39,6 +41,7 @@ import java.util.logging.Logger;
import java.util.stream.Collectors;
import static java.util.Comparator.comparing;
+import static java.util.Comparator.comparingDouble;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toMap;
@@ -80,8 +83,7 @@ public class DeploymentTrigger {
Change outstanding = status.outstandingChange(instanceName);
boolean deployOutstanding = outstanding.hasTargets()
&& status.instanceSteps().get(instanceName)
- .readyAt(outstanding)
- .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false)
+ .readiness(outstanding).okAt(clock.instant())
&& acceptNewRevision(status, instanceName, outstanding.revision().get());
application = application.with(instanceName,
instance -> withRemainingChange(instance,
@@ -235,7 +237,7 @@ public class DeploymentTrigger {
if ( ! upgradeRevision && change.revision().isPresent()) change = change.withoutApplication();
if ( ! upgradePlatform && change.platform().isPresent()) change = change.withoutPlatform();
Versions versions = Versions.from(change, application, status.deploymentFor(job), status.fallbackPlatform(change, job));
- DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, Optional.of(controller.clock().instant()), instance.change());
+ DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, new Readiness(controller.clock().instant()), instance.change());
Map<JobId, List<DeploymentStatus.Job>> testJobs = status.testJobs(Map.of(job, List.of(toTrigger)));
Map<JobId, List<DeploymentStatus.Job>> jobs = testJobs.isEmpty() || ! requireTests
@@ -374,17 +376,17 @@ public class DeploymentTrigger {
List<Job> jobs = new ArrayList<>();
Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun();
jobsToRun.forEach((jobId, jobsList) -> {
+ abortIfOutdated(status, jobsToRun, jobId);
DeploymentStatus.Job job = jobsList.get(0);
- if ( job.readyAt().isPresent()
- && ! clock.instant().isBefore(job.readyAt().get())
+ if ( job.readiness().okAt(clock.instant())
&& ! controller.jobController().isDisabled(new JobId(jobId.application(), job.type()))
- && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId))
- && abortIfRunning(status, jobsToRun, jobId)) // Abort and trigger this later if running with outdated parameters.
+ && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId))) {
jobs.add(deploymentJob(status.application().require(jobId.application().instance()),
job.versions(),
job.type(),
status.instanceJobs(jobId.application().instance()).get(jobId.type()).isNodeAllocationFailure(),
- job.readyAt().get()));
+ job.readiness().at()));
+ }
});
return Collections.unmodifiableList(jobs);
}
@@ -403,41 +405,29 @@ public class DeploymentTrigger {
return false;
}
- private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) {
- status.jobs().get(job)
- .flatMap(JobStatus::lastTriggered)
- .filter(last -> ! last.hasEnded() && last.reason().isEmpty())
- .ifPresent(last -> {
- if (jobs.get(job).stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions())
- && versions.versions().sourcesMatchIfPresent(last.versions()))) {
- String blocked = jobs.get(job).stream()
- .map(scheduled -> scheduled.versions().toString())
- .collect(Collectors.joining(", "));
- log.log(Level.INFO, "Aborting outdated run " + last + ", which is blocking runs: " + blocked);
- controller.jobController().abort(last.id(), "run no longer scheduled, and is blocking scheduled runs: " + blocked);
- }
- });
+ private void abortIfOutdated(JobStatus job, List<DeploymentStatus.Job> jobs) {
+ job.lastTriggered()
+ .filter(last -> ! last.hasEnded() && last.reason().isEmpty())
+ .ifPresent(last -> {
+ if (jobs.stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions())
+ && versions.versions().sourcesMatchIfPresent(last.versions()))) {
+ String blocked = jobs.stream()
+ .map(scheduled -> scheduled.versions().toString())
+ .collect(Collectors.joining(", "));
+ log.log(Level.INFO, "Aborting outdated run " + last + ", which is blocking runs: " + blocked);
+ controller.jobController().abort(last.id(), "run no longer scheduled, and is blocking scheduled runs: " + blocked);
+ }
+ });
}
/** Returns whether the job is free to start, and also aborts it if it's running with outdated versions. */
- private boolean abortIfRunning(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) {
- abortIfOutdated(status, jobs, job);
- boolean blocked = status.jobs().get(job).get().isRunning();
-
- if ( ! job.type().isTest()) {
- Optional<JobStatus> productionTest = status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone())));
- if (productionTest.isPresent()) {
- abortIfOutdated(status, jobs, productionTest.get().id());
- // Production deployments are also blocked by their declared tests, if the next versions to run
- // for those are not the same as the versions we're considering running in the deployment job now.
- if (productionTest.map(JobStatus::id).map(jobs::get)
- .map(versions -> ! versions.get(0).versions().targetsMatch(jobs.get(job).get(0).versions()))
- .orElse(false))
- blocked = true;
- }
- }
-
- return ! blocked;
+ private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) {
+ Readiness readiness = jobs.get(job).get(0).readiness();
+ if (readiness.cause() == DelayCause.running)
+ abortIfOutdated(status.jobs().get(job).get(), jobs.get(job));
+ if (readiness.cause() == DelayCause.blocked && ! job.type().isTest())
+ status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone())))
+ .ifPresent(jobStatus -> abortIfOutdated(jobStatus, jobs.get(jobStatus.id())));
}
// ---------- Change management o_O ----------
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 451f5555eb2..7b1a1e879d6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -279,7 +279,7 @@ public class InternalStepRunner implements StepRunner {
switch (e.type()) {
case CERT_NOT_AVAILABLE:
// Same as CERTIFICATE_NOT_READY above, only from the controller
- logger.log("Creating a CA signed certificate for the application. " +
+ logger.log("Retrieving CA signed certificate for the application. " +
"This may take up to " + timeouts.endpointCertificate() + " on first deployment.");
if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) {
logger.log(WARNING, "CA signed certificate for app not available within " +
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 73c64be3e47..10e4052f067 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -713,6 +713,7 @@ public class JobController {
// TODO(mpolden): Enable for public CD once all tests have been updated
if (controller.system() != SystemName.PublicCd) {
controller.applications().validatePackage(applicationPackage, application.get());
+ controller.applications().decideCloudAccountOf(new DeploymentId(id, type.zone()), applicationPackage.deploymentSpec());
}
controller.applications().store(application);
});
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
index 02e1818932e..cbdfcf70123 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
@@ -79,7 +79,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer {
log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e));
}
});
- return asSuccessFactor(attempts.get(), failures.get());
+ return asSuccessFactorDeviation(attempts.get(), failures.get());
}
private boolean isInCurrentShard(TenantAndApplicationId id) {
@@ -122,7 +122,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer {
log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e));
}
});
- return asSuccessFactor(attempts.get(), failures.get());
+ return asSuccessFactorDeviation(attempts.get(), failures.get());
}
private double updateConfirmedApplicationOwners() {
@@ -149,7 +149,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer {
log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e));
}
});
- return asSuccessFactor(attempts.get(), failures.get());
+ return asSuccessFactorDeviation(attempts.get(), failures.get());
}
private ApplicationList applications() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java
index 518027f8099..c4f3c611cc5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java
@@ -98,7 +98,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer {
}
}
- return asSuccessFactor(tenantsByZone.size(), failures);
+ return asSuccessFactorDeviation(tenantsByZone.size(), failures);
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java
index 25c4121c271..32d06286820 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance;
import com.yahoo.component.Version;
import com.yahoo.config.provision.CloudName;
-import com.yahoo.config.provision.SystemName;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact;
import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry;
@@ -13,12 +12,9 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
import java.time.Duration;
import java.time.Instant;
import java.util.Comparator;
-import java.util.EnumSet;
import java.util.List;
-import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
-import java.util.stream.Collectors;
/**
* Periodically expire unused artifacts, e.g. container images and RPMs.
@@ -32,7 +28,7 @@ public class ArtifactExpirer extends ControllerMaintainer {
private static final Duration MIN_AGE = Duration.ofDays(14);
public ArtifactExpirer(Controller controller, Duration interval) {
- super(controller, interval, null, expiringSystems());
+ super(controller, interval);
}
@Override
@@ -56,10 +52,10 @@ public class ArtifactExpirer extends ControllerMaintainer {
log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts in " + cloudName + ": " + artifactsToExpire);
artifactRegistry.deleteAll(artifactsToExpire);
}
- return 1;
+ return 0;
} catch (RuntimeException e) {
log.log(Level.WARNING, "Failed to expire artifacts in " + cloudName + ". Will retry in " + interval(), e);
- return 0;
+ return 1;
}
}
@@ -77,11 +73,4 @@ public class ArtifactExpirer extends ControllerMaintainer {
return true;
}
- /** Returns systems where artifacts can be expired */
- private static Set<SystemName> expiringSystems() {
- // Run only in public and main. Public systems have distinct container registries, while main and CD have
- // shared registries.
- return EnumSet.of(SystemName.Public, SystemName.PublicCd, SystemName.main);
- }
-
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java
index 32e6ad0d557..fe1930a19ea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java
@@ -42,11 +42,13 @@ public class BcpGroupUpdater extends ControllerMaintainer {
private final ApplicationController applications;
private final NodeRepository nodeRepository;
+ private final Double successFactorBaseline;
- public BcpGroupUpdater(Controller controller, Duration duration) {
- super(controller, duration);
+ public BcpGroupUpdater(Controller controller, Duration duration, Double successFactorBaseline) {
+ super(controller, duration, successFactorBaseline);
this.applications = controller.applications();
this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
+ this.successFactorBaseline = successFactorBaseline;
}
@Override
@@ -58,7 +60,7 @@ public class BcpGroupUpdater extends ControllerMaintainer {
for (var application : applications.asList()) {
for (var instance : application.instances().values()) {
for (var deployment : instance.productionDeployments().values()) {
- if (shuttingDown()) return 1.0;
+ if (shuttingDown()) return 0.0;
try {
attempts++;
var bcpGroups = BcpGroup.groupsFrom(instance, application.deploymentSpec());
@@ -75,12 +77,12 @@ public class BcpGroupUpdater extends ControllerMaintainer {
}
}
}
- double successFactor = asSuccessFactor(attempts, failures);
- if ( successFactor == 0 )
+ double successFactorDeviation = asSuccessFactorDeviation(attempts, failures);
+ if ( successFactorDeviation == -successFactorBaseline )
log.log(Level.WARNING, "Could not update traffic share on any applications", lastException);
- else if ( successFactor < 0.9 )
+ else if ( successFactorDeviation < -0.1 )
log.log(Level.FINE, "Could not update traffic share on all applications", lastException);
- return successFactor;
+ return successFactorDeviation;
}
/** Adds deployment traffic share to the given patch. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java
index a7ebaec7c09..b40078eef51 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java
@@ -19,6 +19,6 @@ public class BillingDatabaseMaintainer extends ControllerMaintainer {
@Override
protected double maintain() {
controller().serviceRegistry().billingDatabase().maintain();
- return 1;
+ return 0.0;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java
index 914707aa318..68fd5c8bafe 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java
@@ -19,8 +19,8 @@ public class CloudDatabaseMaintainer extends ControllerMaintainer {
controller().serviceRegistry().billingController().updateCache(tenants);
} catch (Exception e) {
log.warning("Could not update cloud database cache: " + Exceptions.toMessageString(e));
- return 0.0;
+ return 1.0;
}
- return 1.0;
+ return 0.0;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
index 6ecad482cd2..f9c93a87c44 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
@@ -64,7 +64,7 @@ public class ContactInformationMaintainer extends ControllerMaintainer {
interval());
}
}
- return asSuccessFactor(attempts, failures);
+ return asSuccessFactorDeviation(attempts, failures);
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java
index c861d522818..f21803283eb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java
@@ -12,7 +12,6 @@ import java.util.EnumSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.logging.Logger;
/**
* A maintainer is some job which runs at a fixed interval to perform some maintenance task in the controller.
@@ -26,13 +25,22 @@ public abstract class ControllerMaintainer extends Maintainer {
/** The systems in which this maintainer should run */
private final Set<SystemName> activeSystems;
+
public ControllerMaintainer(Controller controller, Duration interval) {
- this(controller, interval, null, EnumSet.allOf(SystemName.class));
+ this(controller, interval, null, EnumSet.allOf(SystemName.class), 1.0);
+ }
+
+ public ControllerMaintainer(Controller controller, Duration interval, Double successFactorBaseline) {
+ this(controller, interval, null, EnumSet.allOf(SystemName.class), successFactorBaseline);
}
public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems) {
+ this(controller, interval, name, activeSystems, 1.0);
+ }
+
+ public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems, Double successFactorBaseline) {
super(name, interval, controller.clock(), controller.jobControl(),
- new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true);
+ new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true, successFactorBaseline);
this.controller = controller;
this.activeSystems = Set.copyOf(Objects.requireNonNull(activeSystems));
}
@@ -54,8 +62,8 @@ public abstract class ControllerMaintainer extends Maintainer {
}
@Override
- public void completed(String job, double successFactor, long durationMs) {
- metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job)));
+ public void completed(String job, double successFactorDeviation, long durationMs) {
+ metric.set("maintenance.successFactorDeviation", successFactorDeviation, metric.createContext(Map.of("job", job)));
metric.set("maintenance.duration", durationMs, metric.createContext(Map.of("job", job)));
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 05159b38ec6..7f48a1115c5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -40,6 +40,7 @@ public class ControllerMaintenance extends AbstractComponent {
@SuppressWarnings("unused") // instantiated by Dependency Injection
public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) {
Intervals intervals = new Intervals(controller.system());
+ SuccessFactorBaseline successFactorBaseline = new SuccessFactorBaseline(controller.system());
upgrader = new Upgrader(controller, intervals.defaultInterval);
osUpgradeScheduler = new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler);
maintainers.add(upgrader);
@@ -68,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent {
maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater));
maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer));
maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer));
- maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater));
+ maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater, successFactorBaseline.trafficFractionUpdater));
maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater));
maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer));
maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer));
@@ -189,4 +190,15 @@ public class ControllerMaintenance extends AbstractComponent {
}
+ private static class SuccessFactorBaseline {
+
+ private final Double defaultSuccessFactorBaseline;
+ private final Double trafficFractionUpdater;
+
+ public SuccessFactorBaseline(SystemName system) {
+ Objects.requireNonNull(system);
+ this.defaultSuccessFactorBaseline = 1.0;
+ this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.7;
+ }
+ }
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
index 403b5aed1ce..668893d5a7e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
@@ -34,7 +34,7 @@ public class CostReportMaintainer extends ControllerMaintainer {
protected double maintain() {
var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations());
consumer.consume(csv);
- return 1.0;
+ return 0.0;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
index 97f3f955a20..c22cb1efdb3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
@@ -5,8 +5,6 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.yolean.Exceptions;
@@ -47,7 +45,7 @@ public class DeploymentExpirer extends ControllerMaintainer {
}
}
}
- return asSuccessFactor(attempts, failures);
+ return asSuccessFactorDeviation(attempts, failures);
}
/** Returns whether given deployment has expired according to its TTL */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java
index f6029eade37..d8d89177a9e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java
@@ -38,7 +38,7 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer {
}
}
}
- return asSuccessFactor(attempts, failures);
+ return asSuccessFactorDeviation(attempts, failures);
}
private Collection<DeploymentId> instanceDeployments(Instance instance) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
index 6b058537c2d..c352fb053dc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
@@ -77,7 +77,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer {
fileDeploymentIssueFor(application);
else
store(application.id(), null);
- return 1.0;
+ return 0.0;
}
/**
@@ -87,24 +87,24 @@ public class DeploymentIssueReporter extends ControllerMaintainer {
*/
private double maintainPlatformIssue(List<Application> applications) {
if (controller().system() == SystemName.cd)
- return 1.0;
+ return 0.0;
VersionStatus versionStatus = controller().readVersionStatus();
Version systemVersion = controller().systemVersion(versionStatus);
if (versionStatus.version(systemVersion).confidence() != broken)
- return 1.0;
+ return 0.0;
DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications));
if (statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)).isEmpty())
- return 1.0;
+ return 0.0;
List<ApplicationId> failingApplications = statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant())
.mapToList(status -> status.application().id().defaultInstance());
// TODO jonmv: Send only tenant and application, here and elsewhere in this.
deploymentIssues.fileUnlessOpen(failingApplications, systemVersion);
- return 1.0;
+ return 0.0;
}
private Tenant ownerOf(TenantAndApplicationId applicationId) {
@@ -145,7 +145,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer {
log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e));
}
}));
- return asSuccessFactor(attempts.get(), failures.get());
+ return asSuccessFactorDeviation(attempts.get(), failures.get());
}
private void store(TenantAndApplicationId id, IssueId issueId) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index fa917d2eb4e..427ee7c4b06 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -96,7 +96,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer {
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
- return asSuccessFactor(attempts.get(), failures.get());
+ return asSuccessFactorDeviation(attempts.get(), failures.get());
}
static DeploymentMetrics updateDeploymentMetrics(DeploymentMetrics current, List<ClusterMetrics> metrics) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
index 934a1b4fa2f..8a3a2a11e09 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
@@ -78,7 +78,7 @@ public class DeploymentUpgrader extends ControllerMaintainer {
": " + Exceptions.toMessageString(e) + ". Retrying in " +
interval());
}
- return asSuccessFactor(attempts.get(), failures.get());
+ return asSuccessFactorDeviation(attempts.get(), failures.get());
}
/** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java
index 7af96d10f2f..5218da91c46 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java
@@ -26,7 +26,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer {
return controller().serviceRegistry().enclaveAccessService().allowAccessFor(externalAccounts());
} catch (RuntimeException e) {
logger.log(WARNING, "Failed sharing resources with enclave", e);
- return 0;
+ return 1.0;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
index 0b96d8adc1a..713782eb7b9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
@@ -75,10 +75,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
deleteOrReportUnmanagedCertificates();
} catch (Exception e) {
log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e);
- return 0.0;
+ return 1.0;
}
- return 1.0;
+ return 0.0;
}
private void updateRefreshedCertificates() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java
index 1f21c688540..31236f4fcda 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java
@@ -73,7 +73,7 @@ public class HostInfoUpdater extends ControllerMaintainer {
LOG.info("Updated information for " + hostsUpdated + " hosts(s)");
}
}
- return 1.0;
+ return 0.0;
}
private static Optional<String> modelNameOf(NodeEntity nodeEntity) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
index b051590ac5a..68c79fdca7e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
@@ -50,7 +50,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten
@Override
protected double maintain() {
return target().map(target -> upgradeAll(target, managedApplications))
- .orElse(1.0);
+ .orElse(0.0);
}
/** Deploy a list of system applications until they converge on the given version */
@@ -81,7 +81,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten
break;
}
}
- return asSuccessFactor(attempts, failures);
+ return asSuccessFactorDeviation(attempts, failures);
}
/** Returns whether all applications have converged to the target version in zone */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index 294d5bad42d..67188eb5e3a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -90,7 +90,7 @@ public class MetricsReporter extends ControllerMaintainer {
reportBrokenSystemVersion(versionStatus);
reportTenantMetrics();
reportZmsQuotaMetrics();
- return 1.0;
+ return 0.0;
}
private void reportBrokenSystemVersion(VersionStatus versionStatus) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
index f930e64fc5a..f0d218ae6cf 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
@@ -40,7 +40,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer {
if (!change.get().scheduleAt(now)) continue;
controller().upgradeOsIn(cloud, change.get().version(), false);
}
- return 1.0;
+ return 0.0;
}
/** Returns the wanted change for cloud at given instant, if any */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java
index 119540eaa68..c643df6af68 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java
@@ -22,12 +22,12 @@ public class OsVersionStatusUpdater extends ControllerMaintainer {
try {
OsVersionStatus newStatus = OsVersionStatus.compute(controller());
controller().updateOsVersionStatus(newStatus);
- return 1.0;
+ return 0.0;
} catch (Exception e) {
log.log(Level.WARNING, "Failed to compute OS version status: " + Exceptions.toMessageString(e) +
". Retrying in " + interval());
}
- return 0.0;
+ return 1.0;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java
index 400673bfd0c..945b6d32a30 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java
@@ -53,11 +53,11 @@ public class ReindexingTriggerer extends ControllerMaintainer {
controller().applications().reindex(id, deployment.zone(), List.of(), List.of(), true, speed,
"bakground reindexing, to account for changes in built-in linguistics components");
});
- return 1.0;
+ return 0.0;
}
catch (RuntimeException e) {
log.log(Level.WARNING, "Failed to trigger reindexing: " + Exceptions.toMessageString(e));
- return 0.0;
+ return 1.0;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
index 3f20c2eac8f..52206d41c00 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
@@ -98,13 +98,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer {
} catch (Exception e) {
log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " +
Exceptions.toMessageString(e));
- return 0.0;
+ return 1.0;
}
if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots);
if (systemName.isPublic()) reportAllScalingEvents();
updateDeploymentCost(resourceSnapshots);
- return 1.0;
+ return 0.0;
}
void updateDeploymentCost(Collection<ResourceSnapshot> resourceSnapshots) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java
index 18ed154fcf1..59871f716e0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java
@@ -43,7 +43,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer {
if (taggedResources > 0)
log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId());
});
- return 1.0;
+ return 0.0;
}
private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java
index 74bb89e4105..aaf730cc158 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java
@@ -47,9 +47,9 @@ public class RetriggerMaintainer extends ControllerMaintainer {
controller().curator().writeRetriggerEntries(remaining);
} catch (Exception e) {
logger.log(Level.WARNING, "Exception while triggering jobs", e);
- return 0.0;
+ return 1.0;
}
- return 1.0;
+ return 0.0;
}
/** Returns true if a job is ready to run, i.e. is currently not running */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java
index f29df1bc0d5..e3a3415e170 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java
@@ -26,6 +26,6 @@ public class TenantRoleCleanupMaintainer extends ControllerMaintainer {
controller().serviceRegistry().tenantSecretService().cleanupSecretStores(deletedTenants);
}
- return 1.0;
+ return 0.0;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
index bd121871c7c..c7b236880fd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java
@@ -37,7 +37,7 @@ public class TenantRoleMaintainer extends ControllerMaintainer {
controller().tenants().updateLastTenantRolesMaintained(t.name(), updated);
});
- return 1.0;
+ return 0.0;
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index 82b3141e503..edcfcc317a7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -64,7 +64,7 @@ public class Upgrader extends ControllerMaintainer {
for (UpgradePolicy policy : UpgradePolicy.values())
updateTargets(versionStatus, deploymentStatuses, policy);
- return 1.0;
+ return 0.0;
}
private DeploymentStatusList deploymentStatuses(VersionStatus versionStatus) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java
index 03987efab8b..7c4645a6e48 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java
@@ -11,6 +11,7 @@ import java.util.Optional;
import java.util.logging.Logger;
import java.util.stream.Collectors;
+
/**
* Maintains user management resources.
* For now, ensures there's no discrepnacy between expected tenant/application roles and auth0/athenz roles
@@ -46,7 +47,7 @@ public class UserManagementMaintainer extends ControllerMaintainer {
});
}
- return 1.0;
+ return 0.0;
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java
index da0fa890960..9271f870390 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java
@@ -90,7 +90,7 @@ public class VcmrMaintainer extends ControllerMaintainer {
}
});
updateMetrics();
- return 1.0;
+ return 0.0;
}
/**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java
index 154455c5198..1c4d13aa16d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java
@@ -39,12 +39,12 @@ public class VersionStatusUpdater extends ControllerMaintainer {
controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(),
convert(version.confidence()));
});
- return 1.0;
+ return 0.0;
} catch (Exception e) {
log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) +
". Retrying in " + interval());
}
- return 0.0;
+ return 1.0;
}
static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 8a0e2d01d8c..81988753621 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -131,7 +131,6 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfo;
import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
import com.yahoo.yolean.Exceptions;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -1897,7 +1896,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
JobControllerApiHandlerHelper.toSlime(response.setObject("applicationVersion"), application.revisions().get(deployment.revision()));
if ( ! status.jobsToRun().containsKey(stepStatus.job().get()))
response.setString("status", "complete");
- else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true))
+ else if ( ! stepStatus.readiness(instance.change()).okAt(controller.clock().instant()))
response.setString("status", "pending");
else
response.setString("status", "running");
@@ -3004,7 +3003,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
byte[] testPackage = dataParts.getOrDefault(EnvironmentResource.APPLICATION_TEST_ZIP, new byte[0]);
Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, risk);
- controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant),
+ TenantName tenantName = TenantName.from(tenant);
+ controller.applications().verifyPlan(tenantName);
+ controller.applications().verifyApplicationIdentityConfiguration(tenantName,
Optional.empty(),
Optional.empty(),
applicationPackage,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index ce60e0054c4..804ae7b7805 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -15,7 +15,6 @@ import com.yahoo.text.Text;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.NotExistsException;
-import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
@@ -26,6 +25,8 @@ import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness;
import com.yahoo.vespa.hosted.controller.deployment.JobController;
import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
import com.yahoo.vespa.hosted.controller.deployment.Run;
@@ -269,17 +270,39 @@ class JobControllerApiHandlerHelper {
stepObject.setString("instance", stepStatus.instance().value());
// TODO: recursively search dependents for what is the relevant partial change when this is a delay step ...
- Optional<Instant> readyAt = stepStatus.job().map(jobsToRun::get).map(jobs -> jobs.get(0).readyAt())
- .orElse(stepStatus.readyAt(change));
- readyAt.ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli()));
- readyAt.filter(controller.clock().instant()::isBefore)
- .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli()));
- stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli()));
- stepStatus.coolingDownUntil(change, Optional.empty()).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli()));
- stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform.
- .ifPresent(until -> stepObject.setLong("platformBlockedUntil", until.toEpochMilli()));
- stepStatus.blockedUntil(Change.of(RevisionId.forProduction(1))) // Dummy version — just anything with an application.
- .ifPresent(until -> stepObject.setLong("applicationBlockedUntil", until.toEpochMilli()));
+ Readiness readiness = stepStatus.job().map(jobsToRun::get).map(job -> job.get(0).readiness())
+ .orElse(stepStatus.readiness(change));
+ if (readiness.ok()) {
+ stepObject.setLong("readyAt", readiness.at().toEpochMilli());
+ if ( ! readiness.okAt(controller.clock().instant())) {
+ Instant until = readiness.at();
+ stepObject.setLong("delayedUntil", readiness.at().toEpochMilli());
+ switch (readiness.cause()) {
+ case paused -> stepObject.setLong("pausedUntil", until.toEpochMilli());
+ case coolingDown -> stepObject.setLong("coolingDownUntil", until.toEpochMilli());
+ case changeBlocked -> {
+ Readiness platformReadiness = stepStatus.readiness(Change.of(controller.systemVersion(versionStatus))); // Dummy version — just anything with a platform.
+ if (platformReadiness.cause() == DelayCause.changeBlocked)
+ stepObject.setLong("platformBlockedUntil", platformReadiness.at().toEpochMilli());
+ Readiness applicationReadiness = stepStatus.readiness(Change.of(RevisionId.forProduction(1))); // Dummy version — just anything with an application.
+ if (applicationReadiness.cause() == DelayCause.changeBlocked)
+ stepObject.setLong("applicationBlockedUntil", applicationReadiness.at().toEpochMilli());
+ }
+ }
+ }
+ }
+ stepObject.setString("delayCause",
+ switch (readiness.cause()) {
+ case none -> null;
+ case invalidPackage -> "invalidPackage";
+ case paused -> "paused";
+ case coolingDown -> "coolingDown";
+ case changeBlocked -> "changeBlocked";
+ case blocked -> "blocked";
+ case running -> "running";
+ case notReady -> "notReady";
+ case unverified -> "unverified";
+ });
if (stepStatus.type() == DeploymentStatus.StepType.delay)
stepStatus.completedAt(change).ifPresent(completed -> stepObject.setLong("completedAt", completed.toEpochMilli()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index 759b2366229..069ee58e9c5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -22,6 +22,8 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList;
import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness;
import com.yahoo.vespa.hosted.controller.deployment.Run;
import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
import com.yahoo.vespa.hosted.controller.deployment.Versions;
@@ -172,8 +174,9 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler {
instanceObject.setBool("pinned", status.application().require(instance.instance()).change().isPinned());
DeploymentStatus.StepStatus stepStatus = status.instanceSteps().get(instance.instance());
if (stepStatus != null) { // Instance may not have any steps, i.e. an empty deployment spec has been submitted
- stepStatus.blockedUntil(Change.of(statistics.version()))
- .ifPresent(until -> instanceObject.setLong("blockedUntil", until.toEpochMilli()));
+ Readiness platformReadiness = stepStatus.blockedUntil(Change.of(statistics.version()));
+ if (platformReadiness.cause() == DelayCause.changeBlocked)
+ instanceObject.setLong("blockedUntil", platformReadiness.at().toEpochMilli());
}
instanceObject.setString("upgradePolicy", toString(status.application().deploymentSpec().instance(instance.instance())
.map(DeploymentInstanceSpec::upgradePolicy)
@@ -185,10 +188,12 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler {
if ( ! job.application().equals(instance)) return;
Cursor jobObject = jobsArray.addObject();
jobObject.setString("name", job.type().jobName());
- jobStatus.pausedUntil().ifPresent(until -> jobObject.setLong("pausedUntil", until.toEpochMilli()));
- jobStatus.coolingDownUntil(status.application().require(instance.instance()).change(), Optional.empty())
- .ifPresent(until -> jobObject.setLong("coolingDownUntil", until.toEpochMilli()));
if (jobsToRun.containsKey(job)) {
+ Readiness readiness = jobsToRun.get(job).get(0).readiness();
+ switch (readiness.cause()) {
+ case paused -> jobObject.setLong("pausedUntil", readiness.at().toEpochMilli());
+ case coolingDown -> jobObject.setLong("coolingDownUntil", readiness.at().toEpochMilli());
+ }
List<Versions> versionsOnThisPlatform = jobsToRun.get(job).stream()
.map(DeploymentStatus.Job::versions)
.filter(versions -> versions.targetPlatform().equals(statistics.version()))
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index e9947f3d565..45c00848407 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -41,8 +41,8 @@ public record VespaVersion(Version version,
.not().upgradingTo(statistics.version());
InstanceList failingOnThis = all.matching(instance -> statistics.failingUpgrades().stream().anyMatch(run -> run.id().application().equals(instance)));
- // 'broken' if any canary fails
- if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty())
+ // 'broken' if any canary fails, and no non-canary is upgraded
+ if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty() && productionOnThis.not().with(UpgradePolicy.canary).isEmpty())
return Confidence.broken;
// 'broken' if 6 non-canary was broken by this, and that is at least 5% of all
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 8f0536480f5..a9a6fe602b6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -22,6 +22,7 @@ import com.yahoo.path.Path;
import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock;
import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
@@ -54,7 +55,6 @@ import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
import org.junit.jupiter.api.Test;
-
import java.io.InputStream;
import java.time.Duration;
import java.time.Instant;
@@ -1429,9 +1429,14 @@ public class ControllerTest {
// Deployment fails because zone is not configured in requested cloud account
tester.controllerTester().flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of(cloudAccount), String.class);
- context.submit(applicationPackage)
- .runJobExpectingFailure(systemTest, "Zone test.us-east-1 is not configured in requested cloud account '012345678912'")
- .abortJob(stagingTest);
+ assertEquals("Zone test.us-east-1 is not configured in requested cloud account '012345678912'",
+ assertThrows(IllegalArgumentException.class,
+ () -> context.submit(applicationPackage))
+ .getMessage());
+ assertEquals("Zone dev.us-east-1 is not configured in requested cloud account '012345678912'",
+ assertThrows(IllegalArgumentException.class,
+ () -> context.runJob(devUsEast1, applicationPackage))
+ .getMessage());
// Deployment to prod succeeds once all zones are configured in requested account
tester.controllerTester().zoneRegistry().configureCloudAccount(CloudAccount.from(cloudAccount),
@@ -1508,4 +1513,19 @@ public class ControllerTest {
assertFalse(tester.configServer().application(deployment.applicationId(), deployment.zoneId()).isPresent());
}
+ @Test
+ void testVerifyPlan() {
+ DeploymentId deployment = tester.newDeploymentContext().deploymentIdIn(ZoneId.from("prod", "us-west-1"));
+ TenantName tenant = deployment.applicationId().tenant();
+
+ tester.controller().serviceRegistry().billingController().setPlan(tenant, PlanRegistryMock.nonePlan.id(), false, false);
+ try {
+ tester.controller().applications().verifyPlan(tenant);
+ fail("should have thrown an exception");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Tenant 'tenant' has a plan 'None Plan - for testing purposes' with zero quota, not allowed to deploy. " +
+ "See https://cloud.vespa.ai/support", e.getMessage());
+ }
+ }
+
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
index a76d2eca521..ca31ceebc17 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
@@ -126,7 +126,7 @@ public class EndpointTest {
Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).in(SystemName.main),
// Prod endpoint in CD
- "https://cd.a1.t1.us-north-1.vespa.oath.cloud/",
+ "https://cd.a1.t1.us-north-1.cd.vespa.oath.cloud/",
Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).in(SystemName.cd),
// Test endpoint in main
@@ -300,7 +300,7 @@ public class EndpointTest {
.routingMethod(RoutingMethod.exclusive)
.on(Port.tls())
.in(SystemName.main),
- "cd.a2.t2.us-east-3-r.vespa.oath.cloud",
+ "cd.a2.t2.us-east-3-r.cd.vespa.oath.cloud",
Endpoint.of(app2)
.targetApplication(EndpointId.defaultId(), ClusterSpec.Id.from("qrs"),
Map.of(new DeploymentId(app2.instance("i1"), ZoneId.from("prod", "us-east-3")), 1))
@@ -335,7 +335,7 @@ public class EndpointTest {
.routingMethod(RoutingMethod.exclusive)
.on(Port.tls())
.in(SystemName.main),
- "https://cd.a2.t2.a.vespa.oath.cloud/",
+ "https://cd.a2.t2.a.cd.vespa.oath.cloud/",
Endpoint.of(app2)
.targetApplication(EndpointId.defaultId(), ClusterSpec.Id.from("qrs"),
Map.of(new DeploymentId(app2.instance("i1"), ZoneId.from("prod", "us-east-3")), 1))
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 226fb785bf6..afb92d84f3b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -756,8 +756,8 @@ public class DeploymentTriggerTest {
// Last job has a different deployment target, so tests need to run again.
app1.runJob(productionEuWest1) // Upgrade completes, and revision is the only change.
- .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure.
- .runJob(productionEuWest1); // Finally, west changes revision.
+ .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure.
+ .runJob(productionEuWest1); // Finally, west changes revision.
assertEquals(Change.empty(), app1.instance().change());
assertEquals(Optional.of(RunStatus.success), app1.instanceJobs().get(productionUsCentral1).lastStatus());
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java
index 5deba19c5ea..b8b05bfcfc1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java
@@ -45,7 +45,7 @@ public class BcpGroupUpdaterTest {
tester.controllerTester().upgradeSystem(Version.fromString("7.1"));
var context = tester.newDeploymentContext();
var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1));
- var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1));
+ var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0);
ZoneId prod1 = ZoneId.from("prod", "ap-northeast-1");
ZoneId prod2 = ZoneId.from("prod", "us-east-3");
ZoneId prod3 = ZoneId.from("prod", "us-west-1");
@@ -57,7 +57,7 @@ public class BcpGroupUpdaterTest {
setQpsMetric(50.0, context.application().id().defaultInstance(), prod1, tester);
setBcpMetrics(1.5, 0.1, 0.45, context.instanceId(), prod1, "cluster1", tester);
deploymentMetricsMaintainer.maintain();
- assertEquals(1.0, updater.maintain(), 0.0000001);
+ assertEquals(0.0, updater.maintain(), 0.0000001);
assertTrafficFraction(1.0, 1.0, context.instanceId(), prod1, tester);
assertNoBcpGroupInfo(context.instanceId(), prod1, "cluster1", tester, "No other regions in group");
@@ -67,7 +67,7 @@ public class BcpGroupUpdaterTest {
setQpsMetric(20.0, context.application().id().defaultInstance(), prod2, tester);
setBcpMetrics(100.0, 0.1, 0.45, context.instanceId(), prod1, "cluster1", tester);
deploymentMetricsMaintainer.maintain();
- assertEquals(1.0, updater.maintain(), 0.0000001);
+ assertEquals(0.0, updater.maintain(), 0.0000001);
assertTrafficFraction(0.75, 1.0, context.instanceId(), prod1, tester);
assertTrafficFraction(0.25, 1.0, context.instanceId(), prod2, tester);
assertNoBcpGroupInfo(context.instanceId(), prod1, "cluster1", tester,
@@ -75,7 +75,7 @@ public class BcpGroupUpdaterTest {
assertBcpGroupInfo(100.0, 0.1, 0.45,
context.instanceId(), prod2, "cluster1", tester);
setBcpMetrics(50.0, 0.2, 0.5, context.instanceId(), prod2, "cluster1", tester);
- assertEquals(1.0, updater.maintain(), 0.0000001);
+ assertEquals(0.0, updater.maintain(), 0.0000001);
assertBcpGroupInfo(50.0, 0.2, 0.5,
context.instanceId(), prod1, "cluster1", tester);
@@ -85,7 +85,7 @@ public class BcpGroupUpdaterTest {
setQpsMetric(45.0, context.application().id().defaultInstance(), prod2, tester);
setQpsMetric(02.0, context.application().id().defaultInstance(), prod3, tester);
deploymentMetricsMaintainer.maintain();
- assertEquals(1.0, updater.maintain(), 0.0000001);
+ assertEquals(0.0, updater.maintain(), 0.0000001);
assertTrafficFraction(0.53, 0.53 + (double)45/2 / 100, context.instanceId(), prod1, tester);
assertTrafficFraction(0.45, 0.45 + (double)53/2 / 100, context.instanceId(), prod2, tester);
assertTrafficFraction(0.02, 0.02 + (double)53/2 / 100, context.instanceId(), prod3, tester);
@@ -129,7 +129,7 @@ public class BcpGroupUpdaterTest {
locked -> tester.controller().applications().store(locked.with(deploymentSpec)));
var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1));
- var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1));
+ var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0);
ZoneId ap1 = ZoneId.from("prod", "ap-northeast-1");
ZoneId ap2 = ZoneId.from("prod", "ap-southeast-1");
@@ -150,7 +150,7 @@ public class BcpGroupUpdaterTest {
setQpsMetric(40.0, context.application().id().defaultInstance(), eu1, tester);
deploymentMetricsMaintainer.maintain();
- assertEquals(1.0, updater.maintain(), 0.0000001);
+ assertEquals(0.0, updater.maintain(), 0.0000001);
assertTrafficFraction(0.5, 0.5, context.instanceId(), ap1, tester);
assertTrafficFraction(0.0, 0.5, context.instanceId(), ap2, tester);
assertTrafficFraction(0.1, 0.1, context.instanceId(), us1, tester);
@@ -197,7 +197,7 @@ public class BcpGroupUpdaterTest {
locked -> tester.controller().applications().store(locked.with(deploymentSpec)));
var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1));
- var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1));
+ var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0);
ZoneId ap1 = ZoneId.from("prod", "ap-northeast-1");
ZoneId ap2 = ZoneId.from("prod", "ap-southeast-1");
@@ -221,7 +221,7 @@ public class BcpGroupUpdaterTest {
setQpsMetric(60.0, context.application().id().defaultInstance(), eu1, tester);
deploymentMetricsMaintainer.maintain();
- assertEquals(1.0, updater.maintain(), 0.0000001);
+ assertEquals(0.0, updater.maintain(), 0.0000001);
assertTrafficFraction(0.10, 0.10 + 50 / 200.0 / 1.5, context.instanceId(), ap1, tester);
assertTrafficFraction(0.25, 0.25 + 30 / 200.0 / 1.5, context.instanceId(), ap2, tester);
assertTrafficFraction(0.00, 0.00 + 40 / 200.0 / 2.5, context.instanceId(), us1, tester);
@@ -242,7 +242,7 @@ public class BcpGroupUpdaterTest {
setBcpMetrics(300, 0.3, 0.3, context.instanceId(), us3, "cluster2", tester);
setBcpMetrics(100, 0.1, 0.1, context.instanceId(), eu1, "cluster2", tester);
- assertEquals(1.0, updater.maintain(), 0.0000001);
+ assertEquals(0.0, updater.maintain(), 0.0000001);
assertNoBcpGroupInfo(context.instanceId(), ap1, "cluster1", tester, "No info in ap");
assertNoBcpGroupInfo(context.instanceId(), ap2, "cluster1", tester, "No info in ap");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java
index 63e2c99cb6e..6452edc9e61 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java
@@ -38,14 +38,14 @@ public class ControllerMaintainerTest {
void records_metric() {
TestControllerMaintainer maintainer = new TestControllerMaintainer(tester.controller(), SystemName.main, new AtomicInteger());
maintainer.run();
- assertEquals(1.0, successFactorMetric(), 0.0000001);
+ assertEquals(0.0, successFactorDeviationMetric(), 0.0000001);
maintainer.success = false;
maintainer.run();
maintainer.run();
- assertEquals(0.0, successFactorMetric(), 0.0000001);
+ assertEquals(1.0, successFactorDeviationMetric(), 0.0000001);
maintainer.success = true;
maintainer.run();
- assertEquals(1.0, successFactorMetric(), 0.0000001);
+ assertEquals(0.0, successFactorDeviationMetric(), 0.0000001);
}
private long consecutiveFailuresMetric() {
@@ -54,10 +54,10 @@ public class ControllerMaintainerTest {
"maintenance.consecutiveFailures").get().longValue();
}
- private long successFactorMetric() {
+ private long successFactorDeviationMetric() {
MetricsMock metrics = (MetricsMock) tester.controller().metric();
return metrics.getMetric((context) -> "TestControllerMaintainer".equals(context.get("job")),
- "maintenance.successFactor").get().longValue();
+ "maintenance.successFactorDeviation").get().longValue();
}
private static class TestControllerMaintainer extends ControllerMaintainer {
@@ -73,7 +73,7 @@ public class ControllerMaintainerTest {
@Override
protected double maintain() {
executions.incrementAndGet();
- return success ? 1.0 : 0.0;
+ return success ? 0.0 : 1.0;
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
index 934e15ad623..49cf8c634ba 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
@@ -49,7 +49,7 @@ public class EndpointCertificateMaintainerTest {
@Test
void old_and_unused_cert_is_deleted() {
tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata);
- assertEquals(1.0, maintainer.maintain(), 0.0000001);
+ assertEquals(0.0, maintainer.maintain(), 0.0000001);
assertTrue(tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()).isEmpty());
}
@@ -57,7 +57,7 @@ public class EndpointCertificateMaintainerTest {
void unused_but_recently_used_cert_is_not_deleted() {
EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond());
tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert);
- assertEquals(1.0, maintainer.maintain(), 0.0000001);
+ assertEquals(0.0, maintainer.maintain(), 0.0000001);
assertEquals(Optional.of(recentlyRequestedCert), tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()));
}
@@ -69,7 +69,7 @@ public class EndpointCertificateMaintainerTest {
secretStore.setSecret(exampleMetadata.keyName(), "foo", 1);
secretStore.setSecret(exampleMetadata.certName(), "bar", 1);
- assertEquals(1.0, maintainer.maintain(), 0.0000001);
+ assertEquals(0.0, maintainer.maintain(), 0.0000001);
var updatedCert = Optional.of(recentlyRequestedCert.withLastRefreshed(tester.clock().instant().getEpochSecond()).withVersion(1));
@@ -90,7 +90,7 @@ public class EndpointCertificateMaintainerTest {
deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1);
- assertEquals(1.0, maintainer.maintain(), 0.0000001);
+ assertEquals(0.0, maintainer.maintain(), 0.0000001);
var metadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow();
tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(metadata.rootRequestId()); // cert should not be deleted, the app is deployed!
}
@@ -110,7 +110,7 @@ public class EndpointCertificateMaintainerTest {
var originalMetadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow();
// cert should not be deleted, the app is deployed!
- assertEquals(1.0, maintainer.maintain(), 0.0000001);
+ assertEquals(0.0, maintainer.maintain(), 0.0000001);
assertEquals(tester.curator().readEndpointCertificateMetadata(appId), Optional.of(originalMetadata));
tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(originalMetadata.rootRequestId());
@@ -121,7 +121,7 @@ public class EndpointCertificateMaintainerTest {
tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate(appId, originalMetadata.requestedDnsSans(), Optional.of(originalMetadata));
// We should now pick up the new key and cert version + uuid, but not force trigger deployment yet
- assertEquals(1.0, maintainer.maintain(), 0.0000001);
+ assertEquals(0.0, maintainer.maintain(), 0.0000001);
deploymentContext.assertNotRunning(productionUsWest1);
var updatedMetadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow();
assertNotEquals(originalMetadata.leafRequestId().orElseThrow(), updatedMetadata.leafRequestId().orElseThrow());
@@ -130,7 +130,7 @@ public class EndpointCertificateMaintainerTest {
// after another 4 days, we should force trigger deployment if it hasn't already happened
tester.clock().advance(Duration.ofDays(4).plusSeconds(1));
deploymentContext.assertNotRunning(productionUsWest1);
- assertEquals(1.0, maintainer.maintain(), 0.0000001);
+ assertEquals(0.0, maintainer.maintain(), 0.0000001);
deploymentContext.assertRunning(productionUsWest1);
}
@@ -156,7 +156,7 @@ public class EndpointCertificateMaintainerTest {
ApplicationId unknown = ApplicationId.fromSerializedForm("applicationid:is:unknown");
endpointCertificateProvider.requestCaSignedCertificate(unknown, List.of("a", "b", "c"), Optional.empty()); // Unknown to controller!
- assertEquals(1.0, maintainer.maintain(), 0.0000001);
+ assertEquals(0.0, maintainer.maintain(), 0.0000001);
assertTrue(endpointCertificateProvider.dnsNamesOf(unknown).isEmpty());
assertTrue(endpointCertificateProvider.listCertificates().isEmpty());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
index a02fb1fb375..6793553faca 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
@@ -9,6 +9,7 @@
"declared": true,
"instance": "default",
"readyAt": 0,
+ "delayCause": null,
"deploying": {
"application": {
"build": 3,
@@ -155,6 +156,7 @@
"declared": false,
"instance": "default",
"readyAt": 0,
+ "delayCause": null,
"jobName": "system-test",
"url": "https://some.url:43/instance/default/job/system-test",
"environment": "test",
@@ -344,6 +346,7 @@
"readyAt": 15153000,
"delayedUntil": 15153000,
"coolingDownUntil": 15153000,
+ "delayCause": "coolingDown",
"jobName": "staging-test",
"url": "https://some.url:43/instance/default/job/staging-test",
"environment": "staging",
@@ -777,6 +780,8 @@
"declared": true,
"instance": "default",
"readyAt": 14403000,
+ "delayedUntil": 14403000,
+ "delayCause": "running",
"jobName": "production-us-central-1",
"url": "https://some.url:43/instance/default/job/production-us-central-1",
"environment": "prod",
@@ -902,6 +907,7 @@
],
"declared": true,
"instance": "default",
+ "delayCause": "notReady",
"jobName": "test-us-central-1",
"url": "https://some.url:43/instance/default/job/test-us-central-1",
"environment": "prod",
@@ -1042,6 +1048,7 @@
],
"declared": true,
"instance": "default",
+ "delayCause": "notReady",
"jobName": "production-us-west-1",
"url": "https://some.url:43/instance/default/job/production-us-west-1",
"environment": "prod",
@@ -1150,6 +1157,7 @@
],
"declared": true,
"instance": "default",
+ "delayCause": "notReady",
"jobName": "production-us-east-3",
"url": "https://some.url:43/instance/default/job/production-us-east-3",
"environment": "prod",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
index 35dd6fc5398..ec6ccf3ecf2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
@@ -9,6 +9,7 @@
"declared": true,
"instance": "instance1",
"readyAt": 0,
+ "delayCause": null,
"deploying": {
"application": {
"build": 4,
@@ -59,6 +60,8 @@
"declared": false,
"instance": "instance1",
"readyAt": 0,
+ "delayedUntil": 0,
+ "delayCause": "running",
"jobName": "system-test",
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test",
"environment": "test",
@@ -187,6 +190,8 @@
"declared": false,
"instance": "instance1",
"readyAt": 0,
+ "delayedUntil": 0,
+ "delayCause": "running",
"jobName": "staging-test",
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test",
"environment": "staging",
@@ -348,6 +353,7 @@
],
"declared": true,
"instance": "instance1",
+ "delayCause": "unverified",
"jobName": "production-us-central-1",
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1",
"environment": "prod",
@@ -405,6 +411,7 @@
],
"declared": true,
"instance": "instance1",
+ "delayCause": "notReady",
"jobName": "production-us-west-1",
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1",
"environment": "prod",
@@ -462,6 +469,7 @@
],
"declared": true,
"instance": "instance1",
+ "delayCause": "notReady",
"jobName": "production-us-east-3",
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3",
"environment": "prod",
@@ -547,6 +555,7 @@
],
"declared": true,
"instance": "instance2",
+ "delayCause": "notReady",
"deploying": {
"application": {
"build": 4,
@@ -598,6 +607,7 @@
],
"declared": true,
"instance": "instance2",
+ "delayCause": "unverified",
"jobName": "production-us-central-1",
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-central-1",
"environment": "prod",
@@ -624,6 +634,7 @@
],
"declared": true,
"instance": "instance2",
+ "delayCause": "notReady",
"jobName": "production-us-west-1",
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-west-1",
"environment": "prod",
@@ -650,6 +661,7 @@
],
"declared": true,
"instance": "instance2",
+ "delayCause": "notReady",
"jobName": "production-us-east-3",
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-east-3",
"environment": "prod",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
index 7ee5f6db9b9..c942a7ad63d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
import org.junit.jupiter.api.Test;
import java.io.File;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -76,7 +77,7 @@ public class DeploymentApiTest extends ControllerContainerTest {
deploymentTester.upgrader().maintain();
deploymentTester.triggerJobs();
productionApp.runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsWest1);
- failingApp.failDeployment(DeploymentContext.systemTest).failDeployment(DeploymentContext.stagingTest);
+ failingApp.failDeployment(DeploymentContext.systemTest).failDeployment(DeploymentContext.stagingTest).timeOutConvergence(DeploymentContext.stagingTest);
deploymentTester.upgrader().maintain();
deploymentTester.triggerJobs();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
index 51398daa1d4..a1f386d51a7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
@@ -41,12 +41,11 @@
"compileVersion": "6.1.0",
"jobs": [
{
- "name": "system-test",
- "coolingDownUntil": 1600000000000
+ "name": "system-test"
},
{
"name": "staging-test",
- "coolingDownUntil": 1600000000000
+ "coolingDownUntil": 1600022201500
},
{
"name": "production-us-west-1"
@@ -141,7 +140,7 @@
"url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
"upgradePolicy": "default",
"failing": "staging-test",
- "status": "error"
+ "status": "installationFailed"
}
],
"productionApplications": [
@@ -165,14 +164,6 @@
"running": "system-test"
},
{
- "tenant": "tenant1",
- "application": "application1",
- "instance": "default",
- "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
- "upgradePolicy": "default",
- "running": "staging-test"
- },
- {
"tenant": "tenant2",
"application": "application2",
"instance": "i2",
@@ -193,12 +184,11 @@
"jobs": [
{
"name": "system-test",
- "coolingDownUntil": 1600000000000,
"pending": "application"
},
{
"name": "staging-test",
- "coolingDownUntil": 1600000000000,
+ "coolingDownUntil": 1600022201500,
"pending": "platform"
},
{
@@ -222,15 +212,10 @@
},
"staging-test": {
"failing": {
- "number": 2,
- "start": 1600000000000,
- "end": 1600000000000,
- "status": "error"
- },
- "running": {
"number": 3,
"start": 1600000000000,
- "status": "running"
+ "end": 1600014401000,
+ "status": "installationFailed"
}
}
},
@@ -250,15 +235,10 @@
},
"staging-test": {
"failing": {
- "number": 2,
- "start": 1600000000000,
- "end": 1600000000000,
- "status": "error"
- },
- "running": {
"number": 3,
"start": 1600000000000,
- "status": "running"
+ "end": 1600014401000,
+ "status": "installationFailed"
}
}
}
@@ -289,15 +269,15 @@
"system-test": {
"failing": {
"number": 3,
- "start": 1600000000000,
- "end": 1600000000000,
+ "start": 1600014401000,
+ "end": 1600014401000,
"status": "error"
}
},
"staging-test": {
"running": {
"number": 3,
- "start": 1600000000000,
+ "start": 1600014401000,
"status": "running"
}
},
@@ -341,7 +321,7 @@
"production-us-west-1": {
"running": {
"number": 2,
- "start": 1600000000000,
+ "start": 1600014401000,
"status": "running"
}
}
@@ -350,7 +330,7 @@
"production-us-west-1": {
"running": {
"number": 2,
- "start": 1600000000000,
+ "start": 1600014401000,
"status": "running"
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java
index 43ad01fc5c2..168a1345c39 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java
@@ -18,7 +18,6 @@ import org.junit.jupiter.api.Test;
import java.net.URI;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -155,7 +154,7 @@ public class RotationRepositoryTest {
var application2 = tester.newDeploymentContext("tenant2", "app2", "default");
application2.submit(applicationPackage).deploy();
assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations()));
- assertEquals("https://cd.app2.tenant2.global.vespa.oath.cloud/",
+ assertEquals("https://cd.app2.tenant2.global.cd.vespa.oath.cloud/",
tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString());
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index f08e92a515d..7afa5c7f44a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -349,6 +349,13 @@ public class VersionStatusTest {
assertEquals(Confidence.high, confidence(tester.controller(), version0), "Confidence remains unchanged for version0: High");
assertEquals(VespaVersion.Confidence.high, confidence(tester.controller(), version2), "90% of defaults deployed successfully: High");
+ // Canary failing a new revision does not affect confidence
+ canary0.submit(canaryPolicy).failDeployment(systemTest);
+ tester.controllerTester().computeVersionStatus();
+ assertEquals(Confidence.high, confidence(tester.controller(), version0), "Confidence remains unchanged for version0: High");
+ assertEquals(VespaVersion.Confidence.high, confidence(tester.controller(), version2), "90% of defaults deployed successfully: High");
+ canary0.deploy();
+
// A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version
// as broken
Version version3 = new Version("6.5");
diff --git a/default_build_settings.cmake b/default_build_settings.cmake
index c82bb91accc..2785a98a396 100644
--- a/default_build_settings.cmake
+++ b/default_build_settings.cmake
@@ -4,11 +4,11 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL)
function(setup_vespa_default_build_settings_darwin)
message("-- Setting up default build settings for darwin")
- set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS_PREFIX}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib")
- list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib")
+ set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS_PREFIX}/lib" "${VESPA_HOMEBREW_PREFIX}/opt/bison/lib" "${VESPA_HOMEBREW_PREFIX}/opt/flex/lib" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c/lib" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1/lib" "${VESPA_HOMEBREW_PREFIX}/opt/openblas/lib")
+ list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_HOMEBREW_PREFIX}/lib")
set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE)
- set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS_PREFIX}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include" "/usr/local/opt/openblas/include")
- list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "/usr/local/include")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS_PREFIX}/include" "${VESPA_HOMEBREW_PREFIX}/opt/flex/include" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c/include" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1/include" "${VESPA_HOMEBREW_PREFIX}/opt/openblas/include")
+ list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_HOMEBREW_PREFIX}/include")
set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE)
endfunction()
@@ -84,7 +84,7 @@ endfunction()
function(vespa_use_default_cmake_prefix_path)
set(DEFAULT_CMAKE_PREFIX_PATH ${VESPA_DEPS_PREFIX})
if (APPLE)
- list(APPEND DEFAULT_CMAKE_PREFIX_PATH "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" "/usr/local/opt/icu4c")
+ list(APPEND DEFAULT_CMAKE_PREFIX_PATH "${VESPA_HOMEBREW_PREFIX}/opt/bison" "${VESPA_HOMEBREW_PREFIX}/opt/flex" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1" "${VESPA_HOMEBREW_PREFIX}/opt/openblas" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c")
endif()
message("-- DEFAULT_CMAKE_PREFIX_PATH is ${DEFAULT_CMAKE_PREFIX_PATH}")
if(NOT DEFINED CMAKE_PREFIX_PATH)
@@ -203,8 +203,8 @@ function(vespa_use_default_cxx_compiler)
unset(DEFAULT_CMAKE_CXX_COMPILER)
if(NOT DEFINED VESPA_COMPILER_VARIANT OR VESPA_COMPILER_VARIANT STREQUAL "gcc")
if(APPLE)
- set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-12")
- set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-12")
+ set(DEFAULT_CMAKE_C_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/gcc-12")
+ set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/g++-12")
elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2")
set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/gcc10-gcc")
set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/gcc10-g++")
@@ -215,8 +215,8 @@ function(vespa_use_default_cxx_compiler)
endif()
elseif(VESPA_COMPILER_VARIANT STREQUAL "clang")
if(APPLE)
- set(DEFAULT_CMAKE_C_COMPILER, "/usr/local/opt/llvm/bin/clang")
- set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++")
+ set(DEFAULT_CMAKE_C_COMPILER, "${VESPA_HOMEBREW_PREFIX}/opt/llvm/bin/clang")
+ set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/opt/llvm/bin/clang++")
elseif(EXISTS "/usr/bin/clang" AND EXISTS "/usr/bin/clang++")
set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/clang")
set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/clang++")
diff --git a/document/src/vespa/document/select/parse_utils.cpp b/document/src/vespa/document/select/parse_utils.cpp
index 95461442349..4c116d5bff4 100644
--- a/document/src/vespa/document/select/parse_utils.cpp
+++ b/document/src/vespa/document/select/parse_utils.cpp
@@ -24,7 +24,7 @@ parse_i64(const char* str, size_t len, int64_t& out) {
}
bool
parse_double(const char* str, size_t len, double& out) {
-#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000
+#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000
// Temporary workaround that also handles underflow (cf. issue 3081)
// until libc++ supports std::from_chars for double
char *str_end = const_cast<char*>(str) + len;
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 76702d3791d..d55b58a1728 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -57,7 +57,7 @@ public class Flags {
public static final UnboundBooleanFlag DROP_CACHES = defineFeatureFlag(
"drop-caches", false,
- List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-04-05",
+ List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-06-05",
"Drop caches on tenant hosts",
"Takes effect on next tick",
ZONE_ID,
@@ -79,6 +79,12 @@ public class Flags {
" latency-amortized-over-requests, latency-amortized-over-time",
"Takes effect at redeployment (requires restart)",
ZONE_ID, APPLICATION_ID);
+ public static final UnboundStringFlag SUMMARY_DECODE_POLICY = defineStringFlag(
+ "summary-decode-policy", "eager",
+ List.of("baldersheim"), "2023-03-30", "2023-12-31",
+ "Select summary decoding policy, valid values are eager and on-demand/ondemand.",
+ "Takes effect at redeployment (requires restart)",
+ ZONE_ID, APPLICATION_ID);
public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag(
"feed-sequencer-type", "THROUGHPUT",
@@ -303,7 +309,7 @@ public class Flags {
public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag(
"enable-proxy-protocol-mixed-mode", true,
- List.of("tokle"), "2022-05-09", "2023-03-31",
+ List.of("tokle"), "2022-05-09", "2023-04-30",
"Enable or disable proxy protocol mixed mode",
"Takes effect on redeployment",
APPLICATION_ID);
@@ -379,6 +385,19 @@ public class Flags {
"Takes effect on next config server container start",
ZONE_ID);
+ public static final UnboundBooleanFlag NODE_ADMIN_TENANT_SERVICE_REGISTRY = defineFeatureFlag(
+ "node-admin-tenant-service-registry", false,
+ List.of("olaa"), "2023-04-12", "2023-06-12",
+ "Whether AthenzCredentialsMaintainer in node-admin should create tenant service identity certificate",
+ "Takes effect on next tick",
+ ZONE_ID, HOSTNAME
+ );
+
+ public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag(
+ "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-06-13",
+ "Whether to enable CrowdStrike.", "Takes effect on next host admin tick",
+ HOSTNAME);
+
/** WARNING: public for testing: All flags should be defined in {@link Flags}. */
public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
String createdAt, String expiresAt, String description,
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
index 336edf74e8f..dea0dbd3623 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
@@ -347,6 +347,12 @@ public class PermanentFlags {
"Takes effect immediately",
TENANT_ID);
+ public static final UnboundIntFlag KEEP_FILE_REFERENCES_ON_TENANT_NODES = defineIntFlag(
+ "keep-file-references-on-tenant-nodes", 14,
+ "How many days to keep file references on tenant nodes (based on last modification time)",
+ "Takes effect on restart of Docker container",
+ ZONE_ID, APPLICATION_ID
+ );
private PermanentFlags() {}
diff --git a/functions.cmake b/functions.cmake
index 7fa0b0db954..7f217867314 100644
--- a/functions.cmake
+++ b/functions.cmake
@@ -746,6 +746,14 @@ function(vespa_detect_build_platform)
elseif(APPLE)
set(OS_DISTRO "darwin")
set(OS_DISTRO_VERSION ${CMAKE_SYSTEM_VERSION})
+ if(EXISTS "/opt/homebrew/bin/brew")
+ set(VESPA_HOMEBREW_PREFIX "/opt/homebrew")
+ elseif(EXISTS "/usr/local/bin/brew")
+ set(VESPA_HOMEBREW_PREFIX "/usr/local")
+ else()
+ message(FATAL_ERROR "-- Cannot determine homebrew prefix")
+ endif()
+ set(VESPA_HOMEBREW_PREFIX ${VESPA_HOMEBREW_PREFIX} PARENT_SCOPE)
endif()
if(OS_DISTRO)
set(VESPA_OS_DISTRO ${OS_DISTRO} PARENT_SCOPE)
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java
index 45d8637aa3e..231f6fb7598 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java
@@ -1,122 +1,22 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.indexinglanguage;
-import com.yahoo.collections.Pair;
-import com.yahoo.vespa.indexinglanguage.expressions.*;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
/**
* @author Simon Thoresen Hult
*/
-@SuppressWarnings({ "UnusedDeclaration" })
public abstract class ExpressionConverter implements Cloneable {
- public final Expression convert(Expression exp) {
- if (exp == null) {
- return null;
- }
- if (shouldConvert(exp)) {
- return doConvert(exp);
- }
- if (!(exp instanceof CompositeExpression)) {
- return exp;
- }
- try {
- // The class.getMethod here takes 8% of the cpu time in reading the SSBE application package
- // TODO: Implement double dispatch through visitor instead?
- return (Expression)ExpressionConverter.class.getMethod("innerConvert", exp.getClass()).invoke(this, exp);
- } catch (IllegalAccessException | NoSuchMethodException e) {
- throw new UnsupportedOperationException(exp.getClass().getName(), e);
- } catch (InvocationTargetException e) {
- Throwable t = e.getTargetException();
- throw t instanceof RuntimeException ? (RuntimeException)t : new RuntimeException(t);
- }
- }
-
- public Expression innerConvert(ArithmeticExpression exp) {
- return new ArithmeticExpression(convert(exp.getLeftHandSide()),
- exp.getOperator(),
- convert(exp.getRightHandSide()));
- }
-
- public Expression innerConvert(CatExpression exp) {
- List<Expression> lst = new LinkedList<>();
- for (Expression innerExp : exp) {
- Expression next = convert(innerExp);
- if (next != null) {
- lst.add(next);
- }
- }
- return new CatExpression(lst);
- }
-
- public Expression innerConvert(ForEachExpression exp) {
- return new ForEachExpression(convert(exp.getInnerExpression()));
- }
-
- public Expression innerConvert(GuardExpression exp) {
- return new GuardExpression(convert(exp.getInnerExpression()));
- }
-
- public Expression innerConvert(IfThenExpression exp) {
- return new IfThenExpression(branch().convert(exp.getLeftHandSide()),
- exp.getComparator(),
- branch().convert(exp.getRightHandSide()),
- branch().convert(exp.getIfTrueExpression()),
- branch().convert(exp.getIfFalseExpression()));
- }
-
- public Expression innerConvert(ParenthesisExpression exp) {
- return new ParenthesisExpression(convert(exp.getInnerExpression()));
- }
-
- public Expression innerConvert(ScriptExpression exp) {
- List<StatementExpression> lst = new LinkedList<>();
- for (Expression innerExp : exp) {
- StatementExpression next = (StatementExpression)branch().convert(innerExp);
- if (next != null) {
- lst.add(next);
- }
- }
- return new ScriptExpression(lst);
- }
-
- public Expression innerConvert(SelectInputExpression exp) {
- List<Pair<String, Expression>> cases = new LinkedList<>();
- for (Pair<String, Expression> pair : exp.getCases()) {
- cases.add(new Pair<>(pair.getFirst(), branch().convert(pair.getSecond())));
- }
- return new SelectInputExpression(cases);
- }
-
- public Expression innerConvert(StatementExpression exp) {
- List<Expression> lst = new LinkedList<>();
- for (Expression innerExp : exp) {
- Expression next = convert(innerExp);
- if (next != null) {
- lst.add(next);
- }
- }
- return new StatementExpression(lst);
- }
-
- public Expression innerConvert(SwitchExpression exp) {
- Map<String, Expression> cases = new HashMap<>();
- for (Map.Entry<String, Expression> entry : exp.getCases().entrySet()) {
- Expression next = branch().convert(entry.getValue());
- if (next != null) {
- cases.put(entry.getKey(), next);
- }
- }
- return new SwitchExpression(cases, branch().convert(exp.getDefaultExpression()));
+ public final Expression convert(Expression expression) {
+ if (expression == null) return null;
+ if (shouldConvert(expression))
+ return doConvert(expression);
+ else
+ return expression.convertChildren(this);
}
- protected ExpressionConverter branch() {
+ public ExpressionConverter branch() {
return this;
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java
index bbd8c5ebcb8..623f940b06b 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java
@@ -20,7 +20,7 @@ public abstract class ValueTransformProvider extends ExpressionConverter {
}
@Override
- protected final ExpressionConverter branch() {
+ public final ExpressionConverter branch() {
return clone();
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java
index e4bc2dae965..b7ee444975f 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.NumericDataType;
import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -55,6 +56,12 @@ public final class ArithmeticExpression extends CompositeExpression {
this.rhs = rhs;
}
+ @Override
+ public ArithmeticExpression convertChildren(ExpressionConverter converter) {
+ // TODO: branch()?
+ return new ArithmeticExpression(converter.convert(lhs), op, converter.convert(rhs));
+ }
+
public Expression getLeftHandSide() {
return lhs;
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java
index 4c14c633fbf..564ab015e10 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java
@@ -9,6 +9,7 @@ import com.yahoo.document.datatypes.Array;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.StringFieldValue;
import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import java.util.*;
@@ -26,6 +27,11 @@ public final class CatExpression extends ExpressionList<Expression> {
}
@Override
+ public CatExpression convertChildren(ExpressionConverter converter) {
+ return new CatExpression(convertChildList(converter));
+ }
+
+ @Override
protected void doExecute(ExecutionContext context) {
FieldValue input = context.getValue();
DataType inputType = input != null ? input.getDataType() : null;
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java
index 4f83cbfdd8c..86826770828 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import java.util.Arrays;
import java.util.Collection;
@@ -32,6 +33,11 @@ public class ChoiceExpression extends ExpressionList<Expression> {
}
@Override
+ public ChoiceExpression convertChildren(ExpressionConverter converter) {
+ return new ChoiceExpression(asList().stream().map(choice -> converter.branch().convert(choice)).toList());
+ }
+
+ @Override
protected void doExecute(ExecutionContext context) {
FieldValue input = context.getValue();
for (Expression expression : this) {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java
index 27e5524f4ad..8c00aad6bb0 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java
@@ -2,12 +2,16 @@
package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
/**
* @author Simon Thoresen Hult
*/
public abstract class CompositeExpression extends Expression {
+ @Override
+ public abstract CompositeExpression convertChildren(ExpressionConverter converter);
+
protected CompositeExpression(DataType inputType) {
super(inputType);
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java
index bf8201ee7ee..f498b871096 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java
@@ -35,6 +35,12 @@ public abstract class Expression extends Selectable {
this.inputType = inputType;
}
+ /**
+ * Returns an expression where the children of this has been converted using the given converter.
+ * This default implementation returns this as it has no children.
+ */
+ public Expression convertChildren(ExpressionConverter converter) { return this; }
+
/** Sets the document type and field the statement this expression is part of will write to */
public void setStatementOutput(DocumentType documentType, Field field) {}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java
index e2ff1de7126..57de66f80a0 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.DocumentType;
import com.yahoo.document.Field;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -11,6 +12,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Objects;
/**
* @author Simon Thoresen Hult
@@ -26,6 +28,10 @@ public abstract class ExpressionList<T extends Expression> extends CompositeExpr
}
}
+ protected List<Expression> convertChildList(ExpressionConverter converter) {
+ return asList().stream().map(converter::convert).filter(Objects::nonNull).toList();
+ }
+
@Override
public void setStatementOutput(DocumentType documentType, Field field) {
for (Expression expression : expressions)
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java
index 0f3a445bcb9..3053a391823 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java
@@ -6,6 +6,7 @@ import com.yahoo.document.datatypes.Array;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.Struct;
import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.indexinglanguage.FieldValueConverter;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -27,6 +28,11 @@ public final class ForEachExpression extends CompositeExpression {
}
@Override
+ public ForEachExpression convertChildren(ExpressionConverter converter) {
+ return new ForEachExpression(converter.convert(exp));
+ }
+
+ @Override
public void setStatementOutput(DocumentType documentType, Field field) {
exp.setStatementOutput(documentType, field);
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java
index da7cfcdcaee..38a05c3056c 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.DocumentType;
import com.yahoo.document.Field;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
import com.yahoo.vespa.indexinglanguage.UpdateAdapter;
import com.yahoo.vespa.objects.ObjectOperation;
@@ -28,6 +29,11 @@ public final class GuardExpression extends CompositeExpression {
}
@Override
+ public GuardExpression convertChildren(ExpressionConverter converter) {
+ return new GuardExpression(converter.convert(exp));
+ }
+
+ @Override
public void setStatementOutput(DocumentType documentType, Field field) {
exp.setStatementOutput(documentType, field);
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java
index 8a29c8e8645..f05795aa234 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java
@@ -6,6 +6,7 @@ import com.yahoo.document.DocumentType;
import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.NumericFieldValue;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -57,6 +58,15 @@ public final class IfThenExpression extends CompositeExpression {
}
@Override
+ public IfThenExpression convertChildren(ExpressionConverter converter) {
+ return new IfThenExpression(converter.branch().convert(lhs),
+ cmp,
+ converter.branch().convert(rhs),
+ converter.branch().convert(ifTrue),
+ converter.branch().convert(ifFalse));
+ }
+
+ @Override
public void setStatementOutput(DocumentType documentType, Field field) {
lhs.setStatementOutput(documentType, field);
rhs.setStatementOutput(documentType, field);
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java
index 30c824d410d..bba1b09cda2 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java
@@ -21,7 +21,9 @@ public final class InputExpression extends Expression {
public InputExpression(String fieldName) {
super(null);
- this.fieldName = Objects.requireNonNull(fieldName);
+ if (fieldName == null)
+ throw new IllegalArgumentException("'input' must be given a field name as argument");
+ this.fieldName = fieldName;
}
public String getFieldName() {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java
index 60b059f3ef1..6e476f5f7e4 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.DocumentType;
import com.yahoo.document.Field;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -24,6 +25,11 @@ public class ParenthesisExpression extends CompositeExpression {
}
@Override
+ public ParenthesisExpression convertChildren(ExpressionConverter converter) {
+ return new ParenthesisExpression(converter.convert(innerExp));
+ }
+
+ @Override
public void setStatementOutput(DocumentType documentType, Field field) {
innerExp.setStatementOutput(documentType, field);
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java
index f0c37960a99..1a640c9924e 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java
@@ -6,6 +6,7 @@ import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.language.Linguistics;
import com.yahoo.language.process.Embedder;
import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.indexinglanguage.ScriptParser;
import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
import com.yahoo.vespa.indexinglanguage.parser.IndexingInput;
@@ -17,6 +18,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
/**
* @author Simon Thoresen Hult
@@ -36,6 +38,14 @@ public final class ScriptExpression extends ExpressionList<StatementExpression>
}
@Override
+ public ScriptExpression convertChildren(ExpressionConverter converter) {
+ return new ScriptExpression(asList().stream()
+ .map(child -> (StatementExpression)converter.branch().convert(child))
+ .filter(Objects::nonNull)
+ .toList());
+ }
+
+ @Override
protected void doExecute(ExecutionContext context) {
FieldValue input = context.getValue();
for (StatementExpression statement : this) {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java
index 212b60525f9..bb8111f358e 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java
@@ -6,10 +6,12 @@ import com.yahoo.document.DataType;
import com.yahoo.document.DocumentType;
import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -32,6 +34,13 @@ public final class SelectInputExpression extends CompositeExpression {
}
@Override
+ public SelectInputExpression convertChildren(ExpressionConverter converter) {
+ return new SelectInputExpression(cases.stream()
+ .map(c -> new Pair<>(c.getFirst(), converter.branch().convert(c.getSecond())))
+ .toList());
+ }
+
+ @Override
public void setStatementOutput(DocumentType documentType, Field field) {
for (var casePair : cases)
casePair.getSecond().setStatementOutput(documentType, field);
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java
index 8516ddb883d..75f206ef47d 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java
@@ -5,6 +5,7 @@ import com.yahoo.document.DataType;
import com.yahoo.language.Linguistics;
import com.yahoo.language.process.Embedder;
import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.indexinglanguage.ScriptParser;
import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
import com.yahoo.vespa.indexinglanguage.parser.IndexingInput;
@@ -15,6 +16,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
/**
@@ -45,6 +47,14 @@ public final class StatementExpression extends ExpressionList<Expression> {
public List<String> getInputFields() { return inputFields; }
@Override
+ public StatementExpression convertChildren(ExpressionConverter converter) {
+ return new StatementExpression(asList().stream()
+ .map(child -> converter.convert(child))
+ .filter(Objects::nonNull)
+ .toList());
+ }
+
+ @Override
protected void doExecute(ExecutionContext context) {
for (Expression expression : this) {
context.execute(expression);
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java
index 86913d8c1ba..c7cf7066483 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java
@@ -7,6 +7,7 @@ import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.StringFieldValue;
import com.yahoo.text.StringUtilities;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -32,6 +33,17 @@ public final class SwitchExpression extends CompositeExpression {
this.cases.putAll(cases);
}
+ @Override
+ public SwitchExpression convertChildren(ExpressionConverter converter) {
+ var convertedCases = new LinkedHashMap<String, Expression>();
+ for (var entry : cases.entrySet()) {
+ var converted = converter.branch().convert(entry.getValue());
+ if (converted != null)
+ convertedCases.put(entry.getKey(), converted);
+ }
+ return new SwitchExpression(convertedCases, converter.branch().convert(defaultExp));
+ }
+
public boolean isEmpty() {
return defaultExp == null && cases.isEmpty();
}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java
index f1e1be0ae41..8aeaa084e1b 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.indexinglanguage;
import com.yahoo.collections.Pair;
-import com.yahoo.document.DataType;
import com.yahoo.document.datatypes.IntegerFieldValue;
import com.yahoo.language.simple.SimpleLinguistics;
import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression;
@@ -11,9 +10,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression;
import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression;
import com.yahoo.vespa.indexinglanguage.expressions.CatExpression;
import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression;
-import com.yahoo.vespa.indexinglanguage.expressions.CompositeExpression;
import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression;
-import com.yahoo.vespa.indexinglanguage.expressions.ExecutionContext;
import com.yahoo.vespa.indexinglanguage.expressions.Expression;
import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
import com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression;
@@ -54,7 +51,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression;
import com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression;
import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression;
import com.yahoo.vespa.indexinglanguage.expressions.TrimExpression;
-import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext;
import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression;
import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
import org.junit.Test;
@@ -73,7 +69,6 @@ import static org.junit.Assert.fail;
*/
public class ExpressionConverterTestCase {
- @SuppressWarnings("unchecked")
@Test
public void requireThatAllExpressionTypesCanBeTraversed() {
assertConvertable(new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD,
@@ -167,16 +162,6 @@ public class ExpressionConverterTestCase {
}
@Test
- public void requireThatUnknownCompositeThrows() {
- try {
- new MyTraverser().convert(new MyComposite());
- fail();
- } catch (UnsupportedOperationException e) {
- assertEquals(NoSuchMethodException.class, e.getCause().getClass());
- }
- }
-
- @Test
public void requireThatConversionExceptionCanBeThrown() {
final RuntimeException expectedCause = new RuntimeException();
try {
@@ -254,24 +239,4 @@ public class ExpressionConverterTestCase {
}
}
- private static class MyComposite extends CompositeExpression {
-
- MyComposite() {
- super(null);
- }
- @Override
- protected void doExecute(ExecutionContext context) {
-
- }
-
- @Override
- protected void doVerify(VerificationContext context) {
-
- }
-
- @Override
- public DataType createdOutputType() {
- return null;
- }
- }
}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java
index 351c925ed56..7ece841e9b7 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java
@@ -4,7 +4,13 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.Field;
import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.Embedder;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.ExpressionSearcher;
import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import com.yahoo.yolean.Exceptions;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -44,4 +50,24 @@ public class ChoiceTestCase {
}
}
+ @Test
+ public void testIllegalChoiceExpression() throws ParseException {
+ try {
+ parse("input (foo || 99999999) | attribute");
+ }
+ catch (IllegalArgumentException e) {
+ assertEquals("'input' must be given a field name as argument", Exceptions.toMessageString(e));
+ }
+ }
+
+ @Test
+ public void testInnerConvert() throws ParseException {
+ var expression = parse("(input foo || 99999999) | attribute");
+ new ExpressionSearcher<>(AttributeExpression.class).searchIn(expression); // trigger innerConvert
+ }
+
+ private static Expression parse(String s) throws ParseException {
+ return Expression.fromString(s, new SimpleLinguistics(), Embedder.throwsOnUse.asMap());
+ }
+
}
diff --git a/model-integration/pom.xml b/model-integration/pom.xml
index 9bb60827a68..c27ed9d2c31 100644
--- a/model-integration/pom.xml
+++ b/model-integration/pom.xml
@@ -106,6 +106,11 @@
</dependency>
<dependency>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java
index 7cdc27b6d63..02fa7b68dc4 100644
--- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java
+++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java
@@ -7,6 +7,7 @@ import ai.onnxruntime.OnnxTensor;
import ai.onnxruntime.OnnxValue;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
+import ai.vespa.modelintegration.evaluator.OnnxRuntime.ModelPathOrData;
import ai.vespa.modelintegration.evaluator.OnnxRuntime.ReferencedOrtSession;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
@@ -28,7 +29,11 @@ public class OnnxEvaluator implements AutoCloseable {
private final ReferencedOrtSession session;
OnnxEvaluator(String modelPath, OnnxEvaluatorOptions options, OnnxRuntime runtime) {
- session = createSession(modelPath, runtime, options, true);
+ session = createSession(ModelPathOrData.of(modelPath), runtime, options, true);
+ }
+
+ OnnxEvaluator(byte[] data, OnnxEvaluatorOptions options, OnnxRuntime runtime) {
+ session = createSession(ModelPathOrData.of(data), runtime, options, true);
}
public Tensor evaluate(Map<String, Tensor> inputs, String output) {
@@ -125,19 +130,20 @@ public class OnnxEvaluator implements AutoCloseable {
}
}
- private static ReferencedOrtSession createSession(String modelPath, OnnxRuntime runtime, OnnxEvaluatorOptions options, boolean tryCuda) {
+ private static ReferencedOrtSession createSession(
+ ModelPathOrData model, OnnxRuntime runtime, OnnxEvaluatorOptions options, boolean tryCuda) {
if (options == null) {
options = new OnnxEvaluatorOptions();
}
try {
- return runtime.acquireSession(modelPath, options, tryCuda && options.requestingGpu());
+ return runtime.acquireSession(model, options, tryCuda && options.requestingGpu());
} catch (OrtException e) {
if (e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE) {
- throw new IllegalArgumentException("No such file: " + modelPath);
+ throw new IllegalArgumentException("No such file: " + model.path().get());
}
if (tryCuda && isCudaError(e) && !options.gpuDeviceRequired()) {
// Failed in CUDA native code, but GPU device is optional, so we can proceed without it
- return createSession(modelPath, runtime, options, false);
+ return createSession(model, runtime, options, false);
}
if (isCudaError(e)) {
throw new IllegalArgumentException("GPU device is required, but CUDA initialization failed", e);
@@ -146,6 +152,9 @@ public class OnnxEvaluator implements AutoCloseable {
}
}
+ // For unit testing
+ OrtSession ortSession() { return session.instance(); }
+
private String mapToInternalName(String outputName) throws OrtException {
var info = session.instance().getOutputInfo();
var internalNames = info.keySet();
diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java
index 42830041c02..ece1db55c1e 100644
--- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java
+++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java
@@ -10,9 +10,15 @@ import com.yahoo.component.annotation.Inject;
import com.yahoo.jdisc.ResourceReference;
import com.yahoo.jdisc.refcount.DebugReferencesWithStack;
import com.yahoo.jdisc.refcount.References;
+import net.jpountz.xxhash.XXHashFactory;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -26,14 +32,22 @@ import static com.yahoo.yolean.Exceptions.throwUnchecked;
public class OnnxRuntime extends AbstractComponent {
// For unit testing
- @FunctionalInterface interface OrtSessionFactory {
+ interface OrtSessionFactory {
OrtSession create(String path, OrtSession.SessionOptions opts) throws OrtException;
+ OrtSession create(byte[] data, OrtSession.SessionOptions opts) throws OrtException;
}
private static final Logger log = Logger.getLogger(OnnxRuntime.class.getName());
private static final OrtEnvironmentResult ortEnvironment = getOrtEnvironment();
- private static final OrtSessionFactory defaultFactory = (path, opts) -> ortEnvironment().createSession(path, opts);
+ private static final OrtSessionFactory defaultFactory = new OrtSessionFactory() {
+ @Override public OrtSession create(String path, OrtSession.SessionOptions opts) throws OrtException {
+ return ortEnvironment().createSession(path, opts);
+ }
+ @Override public OrtSession create(byte[] data, OrtSession.SessionOptions opts) throws OrtException {
+ return ortEnvironment().createSession(data, opts);
+ }
+ };
private final Object monitor = new Object();
private final Map<OrtSessionId, SharedOrtSession> sessions = new HashMap<>();
@@ -43,6 +57,14 @@ public class OnnxRuntime extends AbstractComponent {
OnnxRuntime(OrtSessionFactory factory) { this.factory = factory; }
+ public OnnxEvaluator evaluatorOf(byte[] model) {
+ return new OnnxEvaluator(model, null, this);
+ }
+
+ public OnnxEvaluator evaluatorOf(byte[] model, OnnxEvaluatorOptions options) {
+ return new OnnxEvaluator(model, options, this);
+ }
+
public OnnxEvaluator evaluatorOf(String modelPath) {
return new OnnxEvaluator(modelPath, null, this);
}
@@ -105,8 +127,8 @@ public class OnnxRuntime extends AbstractComponent {
};
}
- ReferencedOrtSession acquireSession(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) throws OrtException {
- var sessionId = new OrtSessionId(modelPath, options, loadCuda);
+ ReferencedOrtSession acquireSession(ModelPathOrData model, OnnxEvaluatorOptions options, boolean loadCuda) throws OrtException {
+ var sessionId = new OrtSessionId(calculateModelHash(model), options, loadCuda);
synchronized (monitor) {
var sharedSession = sessions.get(sessionId);
if (sharedSession != null) {
@@ -114,8 +136,9 @@ public class OnnxRuntime extends AbstractComponent {
}
}
+ var opts = options.getOptions(loadCuda);
// Note: identical models loaded simultaneously will result in duplicate session instances
- var session = factory.create(modelPath, options.getOptions(loadCuda));
+ var session = model.path().isPresent() ? factory.create(model.path().get(), opts) : factory.create(model.data().get(), opts);
log.fine(() -> "Created new session (%s)".formatted(System.identityHashCode(session)));
var sharedSession = new SharedOrtSession(sessionId, session);
@@ -125,25 +148,52 @@ public class OnnxRuntime extends AbstractComponent {
return referencedSession;
}
+ private static long calculateModelHash(ModelPathOrData model) {
+ if (model.path().isPresent()) {
+ try (var hasher = XXHashFactory.fastestInstance().newStreamingHash64(0);
+ var in = Files.newInputStream(Paths.get(model.path().get()))) {
+ byte[] buffer = new byte[8192];
+ int bytesRead;
+ while ((bytesRead = in.read(buffer)) != -1) {
+ hasher.update(buffer, 0, bytesRead);
+ }
+ return hasher.getValue();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ } else {
+ var data = model.data().get();
+ return XXHashFactory.fastestInstance().hash64().hash(data, 0, data.length, 0);
+ }
+ }
+
int sessionsCached() { synchronized(monitor) { return sessions.size(); } }
- public static class ReferencedOrtSession implements AutoCloseable {
+ static class ReferencedOrtSession implements AutoCloseable {
private final OrtSession instance;
private final ResourceReference ref;
- public ReferencedOrtSession(OrtSession instance, ResourceReference ref) {
+ ReferencedOrtSession(OrtSession instance, ResourceReference ref) {
this.instance = instance;
this.ref = ref;
}
- public OrtSession instance() { return instance; }
+ OrtSession instance() { return instance; }
@Override public void close() { ref.close(); }
}
+ record ModelPathOrData(Optional<String> path, Optional<byte[]> data) {
+ static ModelPathOrData of(String path) { return new ModelPathOrData(Optional.of(path), Optional.empty()); }
+ static ModelPathOrData of(byte[] data) { return new ModelPathOrData(Optional.empty(), Optional.of(data)); }
+ ModelPathOrData {
+ if (path.isEmpty() == data.isEmpty()) throw new IllegalArgumentException("Either path or data must be non-empty");
+ }
+ }
+
// Assumes options are never modified after being stored in `onnxSessions`
- record OrtSessionId(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) {}
+ private record OrtSessionId(long modelHash, OnnxEvaluatorOptions options, boolean loadCuda) {}
- record OrtEnvironmentResult(OrtEnvironment env, Throwable failure) {}
+ private record OrtEnvironmentResult(OrtEnvironment env, Throwable failure) {}
private class SharedOrtSession {
private final OrtSessionId id;
diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java
index 5aba54de11b..5a367ef83e4 100644
--- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java
+++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java
@@ -5,30 +5,26 @@ package ai.vespa.modelintegration.evaluator;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
import org.junit.Test;
-import org.junit.jupiter.api.BeforeAll;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeNotNull;
+import static org.junit.Assume.assumeTrue;
/**
* @author lesters
*/
public class OnnxEvaluatorTest {
- private static OnnxRuntime runtime;
-
- @BeforeAll
- public static void beforeAll() {
- if (OnnxRuntime.isRuntimeAvailable()) runtime = new OnnxRuntime();
- }
-
@Test
public void testSimpleModel() {
- assumeNotNull(runtime);
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ var runtime = new OnnxRuntime();
OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/simple/simple.onnx");
// Input types
@@ -53,7 +49,8 @@ public class OnnxEvaluatorTest {
@Test
public void testBatchDimension() {
- assumeNotNull(runtime);
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ var runtime = new OnnxRuntime();
OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/pytorch/one_layer.onnx");
// Input types
@@ -72,21 +69,23 @@ public class OnnxEvaluatorTest {
@Test
public void testMatMul() {
- assumeNotNull(runtime);
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ var runtime = new OnnxRuntime();
String expected = "tensor<float>(d0[2],d1[4]):[38,44,50,56,83,98,113,128]";
String input1 = "tensor<float>(d0[2],d1[3]):[1,2,3,4,5,6]";
String input2 = "tensor<float>(d0[3],d1[4]):[1,2,3,4,5,6,7,8,9,10,11,12]";
- assertEvaluate("simple/matmul.onnx", expected, input1, input2);
+ assertEvaluate(runtime, "simple/matmul.onnx", expected, input1, input2);
}
@Test
public void testTypes() {
- assumeNotNull(runtime);
- assertEvaluate("add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]");
- assertEvaluate("add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]");
- assertEvaluate("add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]");
- assertEvaluate("cast_int8_float.onnx", "tensor<float>(d0[1]):[-128]", "tensor<int8>(d0[1]):[128]");
- assertEvaluate("cast_float_int8.onnx", "tensor<int8>(d0[1]):[-1]", "tensor<float>(d0[1]):[255]");
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ var runtime = new OnnxRuntime();
+ assertEvaluate(runtime, "add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]");
+ assertEvaluate(runtime, "add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]");
+ assertEvaluate(runtime, "add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]");
+ assertEvaluate(runtime, "cast_int8_float.onnx", "tensor<float>(d0[1]):[-128]", "tensor<int8>(d0[1]):[128]");
+ assertEvaluate(runtime, "cast_float_int8.onnx", "tensor<int8>(d0[1]):[-1]", "tensor<float>(d0[1]):[255]");
// ONNX Runtime 1.8.0 does not support much of bfloat16 yet
// assertEvaluate("cast_bfloat16_float.onnx", "tensor<float>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]");
@@ -94,7 +93,8 @@ public class OnnxEvaluatorTest {
@Test
public void testNotIdentifiers() {
- assumeNotNull(runtime);
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ var runtime = new OnnxRuntime();
OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/badnames.onnx");
var inputInfo = evaluator.getInputInfo();
var outputInfo = evaluator.getOutputInfo();
@@ -159,7 +159,18 @@ public class OnnxEvaluatorTest {
assertEquals(3, allResults.size());
}
- private void assertEvaluate(String model, String output, String... input) {
+ @Test
+ public void testLoadModelFromBytes() throws IOException {
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ var runtime = new OnnxRuntime();
+ var model = Files.readAllBytes(Paths.get("src/test/models/onnx/simple/simple.onnx"));
+ var evaluator = runtime.evaluatorOf(model);
+ assertEquals(3, evaluator.getInputs().size());
+ assertEquals(1, evaluator.getOutputs().size());
+ evaluator.close();
+ }
+
+ private void assertEvaluate(OnnxRuntime runtime, String model, String output, String... input) {
OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/" + model);
Map<String, Tensor> inputs = new HashMap<>();
for (int i = 0; i < input.length; ++i) {
diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java
index 81b1237e770..fdbd4fa4e5c 100644
--- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java
+++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java
@@ -2,16 +2,18 @@
package ai.vespa.modelintegration.evaluator;
-import ai.onnxruntime.OrtException;
-import ai.onnxruntime.OrtSession;
import org.junit.jupiter.api.Test;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertSame;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
/**
* @author bjorncs
@@ -19,30 +21,81 @@ import static org.mockito.Mockito.verify;
class OnnxRuntimeTest {
@Test
- void reuses_sessions_while_active() throws OrtException {
- var runtime = new OnnxRuntime((__, ___) -> mock(OrtSession.class));
- var session1 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false);
- var session2 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false);
- var session3 = runtime.acquireSession("model2", new OnnxEvaluatorOptions(), false);
- assertSame(session1.instance(), session2.instance());
- assertNotSame(session1.instance(), session3.instance());
+ void reuses_sessions_while_active() {
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ OnnxRuntime runtime = new OnnxRuntime();
+ String model1 = "src/test/models/onnx/simple/simple.onnx";
+ var evaluator1 = runtime.evaluatorOf(model1);
+ var evaluator2 = runtime.evaluatorOf(model1);
+ String model2 = "src/test/models/onnx/simple/matmul.onnx";
+ var evaluator3 = runtime.evaluatorOf(model2);
+ assertSameSession(evaluator1, evaluator2);
+ assertNotSameSession(evaluator1, evaluator3);
assertEquals(2, runtime.sessionsCached());
- session1.close();
- session2.close();
+ evaluator1.close();
+ evaluator2.close();
assertEquals(1, runtime.sessionsCached());
- verify(session1.instance()).close();
- verify(session3.instance(), never()).close();
+ assertClosed(evaluator1);
+ assertNotClosed(evaluator3);
- session3.close();
+ evaluator3.close();
assertEquals(0, runtime.sessionsCached());
- verify(session3.instance()).close();
+ assertClosed(evaluator3);
- var session4 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false);
- assertNotSame(session1.instance(), session4.instance());
+ var session4 = runtime.evaluatorOf(model1);
+ assertNotSameSession(evaluator1, session4);
assertEquals(1, runtime.sessionsCached());
session4.close();
assertEquals(0, runtime.sessionsCached());
- verify(session4.instance()).close();
+ assertClosed(session4);
+ }
+
+ @Test
+ void loads_model_from_byte_array() throws IOException {
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ var runtime = new OnnxRuntime();
+ byte[] bytes = Files.readAllBytes(Paths.get("src/test/models/onnx/simple/simple.onnx"));
+ var evaluator1 = runtime.evaluatorOf(bytes);
+ var evaluator2 = runtime.evaluatorOf(bytes);
+ assertEquals(3, evaluator1.getInputs().size());
+ assertEquals(1, runtime.sessionsCached());
+ assertSameSession(evaluator1, evaluator2);
+ evaluator2.close();
+ evaluator1.close();
+ assertEquals(0, runtime.sessionsCached());
+ assertClosed(evaluator1);
+ }
+
+ @Test
+ void loading_same_model_from_bytes_and_file_resolve_to_same_instance() throws IOException {
+ assumeTrue(OnnxRuntime.isRuntimeAvailable());
+ var runtime = new OnnxRuntime();
+ String modelPath = "src/test/models/onnx/simple/simple.onnx";
+ byte[] bytes = Files.readAllBytes(Paths.get(modelPath));
+ try (var evaluator1 = runtime.evaluatorOf(bytes);
+ var evaluator2 = runtime.evaluatorOf(modelPath)) {
+ assertSameSession(evaluator1, evaluator2);
+ assertEquals(1, runtime.sessionsCached());
+ }
+ }
+
+ private static void assertClosed(OnnxEvaluator evaluator) { assertTrue(isClosed(evaluator), "Session is not closed"); }
+ private static void assertNotClosed(OnnxEvaluator evaluator) { assertFalse(isClosed(evaluator), "Session is closed"); }
+ private static void assertSameSession(OnnxEvaluator evaluator1, OnnxEvaluator evaluator2) {
+ assertSame(evaluator1.ortSession(), evaluator2.ortSession());
+ }
+ private static void assertNotSameSession(OnnxEvaluator evaluator1, OnnxEvaluator evaluator2) {
+ assertNotSame(evaluator1.ortSession(), evaluator2.ortSession());
+ }
+
+ private static boolean isClosed(OnnxEvaluator evaluator) {
+ try {
+ evaluator.getInputs();
+ return false;
+ } catch (IllegalStateException e) {
+ assertEquals("Asking for inputs from a closed OrtSession.", e.getMessage());
+ return true;
+ }
}
} \ No newline at end of file
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java
index 7367a254b4a..92cccf86ecb 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.cores;
import com.yahoo.config.provision.DockerImage;
import java.nio.file.Path;
+import java.time.Instant;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -12,7 +13,11 @@ import java.util.Optional;
* @author hakonhall
*/
public class CoreDumpMetadata {
+ public enum Type { CORE_DUMP, JVM_HEAP, OOM }
+
+ private Type type;
private String binPath;
+ private Instant created;
private List<String> backtrace;
private List<String> backtraceAllThreads;
private Path coreDumpPath;
@@ -24,30 +29,36 @@ public class CoreDumpMetadata {
public CoreDumpMetadata() {}
- public Optional<String> binPath() { return Optional.ofNullable(binPath); };
- public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); };
- public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); };
- public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); };
+ public Optional<Type> type() { return Optional.ofNullable(type); }
+ public Optional<String> binPath() { return Optional.ofNullable(binPath); }
+ public Optional<Instant> created() { return Optional.ofNullable(created); }
+ public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); }
+ public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); }
+ public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); }
public Optional<String> decryptionToken() { return Optional.ofNullable(decryptionToken); }
- public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); };
- public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); };
- public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); };
- public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); };
+ public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); }
+ public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); }
+ public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); }
+ public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); }
- public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; };
- public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; };
- public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; };
- public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; };
+ public CoreDumpMetadata setType(Type type) { this.type = type; return this; }
+ public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; }
+ public CoreDumpMetadata setCreated(Instant created) { this.created = created; return this; }
+ public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; }
+ public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; }
+ public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; }
public CoreDumpMetadata setDecryptionToken(String decryptionToken) { this.decryptionToken = decryptionToken; return this; }
- public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; };
- public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; };
- public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; };
- public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; };
+ public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; }
+ public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; }
+ public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; }
+ public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; }
@Override
public String toString() {
return "CoreDumpMetadata{" +
- "binPath=" + binPath +
+ "type=" + type +
+ ", binPath=" + binPath +
+ ", created=" + created +
", backtrace=" + backtrace +
", backtraceAllThreads=" + backtraceAllThreads +
", coreDumpPath=" + coreDumpPath +
@@ -64,7 +75,9 @@ public class CoreDumpMetadata {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
CoreDumpMetadata metadata = (CoreDumpMetadata) o;
- return Objects.equals(binPath, metadata.binPath) &&
+ return type == metadata.type &&
+ Objects.equals(binPath, metadata.binPath) &&
+ Objects.equals(created, metadata.created) &&
Objects.equals(backtrace, metadata.backtrace) &&
Objects.equals(backtraceAllThreads, metadata.backtraceAllThreads) &&
Objects.equals(coreDumpPath, metadata.coreDumpPath) &&
@@ -77,7 +90,7 @@ public class CoreDumpMetadata {
@Override
public int hashCode() {
- return Objects.hash(binPath, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion,
+ return Objects.hash(type, binPath, created, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion,
cpuMicrocodeVersion, dockerImage, vespaVersion);
}
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java
index 27cf28b8e1e..a9620ebabc2 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java
@@ -14,6 +14,7 @@ import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
+import java.time.Instant;
import java.util.List;
import java.util.Optional;
@@ -31,6 +32,8 @@ public class ReportCoreDumpRequest {
public List<String> backtrace;
public List<String> backtrace_all_threads;
+ public Long created;
+ public String type;
public String bin_path;
public String coredump_path;
public String cpu_microcode_version;
@@ -44,7 +47,9 @@ public class ReportCoreDumpRequest {
/** Fill this from metadata and return this. */
@JsonIgnore
public ReportCoreDumpRequest fillFrom(CoreDumpMetadata metadata) {
+ metadata.type().ifPresent(type -> this.type = type.name());
metadata.binPath().ifPresent(binPath -> this.bin_path = binPath);
+ metadata.created().ifPresent(created -> this.created = created.toEpochMilli());
metadata.backtrace().ifPresent(backtrace -> this.backtrace = List.copyOf(backtrace));
metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> this.backtrace_all_threads = List.copyOf(backtraceAllThreads));
metadata.coredumpPath().ifPresent(coredumpPath -> this.coredump_path = coredumpPath.toString());
@@ -58,7 +63,9 @@ public class ReportCoreDumpRequest {
@JsonIgnore
public void populateMetadata(CoreDumpMetadata metadata, FileSystem fileSystem) {
+ if (type != null) metadata.setType(CoreDumpMetadata.Type.valueOf(type));
if (bin_path != null) metadata.setBinPath(bin_path);
+ if (created != null) metadata.setCreated(Instant.ofEpochMilli(created));
if (backtrace != null) metadata.setBacktrace(backtrace);
if (backtrace_all_threads != null) metadata.setBacktraceAllThreads(backtrace_all_threads);
if (coredump_path != null) metadata.setCoreDumpPath(fileSystem.getPath(coredump_path));
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
index 28773767d24..5d4628b41b6 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java
@@ -5,6 +5,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
@@ -96,17 +97,19 @@ public class CoreCollector {
}
CoreDumpMetadata collect(NodeAgentContext context, ContainerPath coredumpPath) {
- var metadata = new CoreDumpMetadata();
+ var metadata = new CoreDumpMetadata()
+ .setCreated(new UnixPath(coredumpPath).getLastModifiedTime());
if (JAVA_HEAP_DUMP_PATTERN.matcher(coredumpPath.getFileName().toString()).find()) {
- metadata.setBinPath("java")
+ metadata.setType(CoreDumpMetadata.Type.JVM_HEAP)
+ .setBinPath("java")
.setBacktrace(List.of("Heap dump, no backtrace available"));
return metadata;
}
try {
String binPath = readBinPath(context, coredumpPath);
- metadata.setBinPath(binPath);
+ metadata.setType(CoreDumpMetadata.Type.CORE_DUMP).setBinPath(binPath);
if (Path.of(binPath).getFileName().toString().equals("java")) {
metadata.setBacktraceAllThreads(readJstack(context, coredumpPath, binPath));
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
index 15be7accb7d..d22fd667202 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
@@ -19,6 +19,10 @@ import com.yahoo.vespa.athenz.identityprovider.client.CsrGenerator;
import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient;
import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier;
import com.yahoo.vespa.athenz.utils.SiaUtils;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo;
import com.yahoo.vespa.hosted.node.admin.container.ContainerName;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
@@ -47,6 +51,9 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.NODE;
+import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.TENANT;
+
/**
* A maintainer that is responsible for providing and refreshing Athenz credentials for a container.
*
@@ -68,7 +75,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
private final String certificateDnsSuffix;
private final ServiceIdentityProvider hostIdentityProvider;
private final IdentityDocumentClient identityDocumentClient;
- private final boolean useInternalZts;
+ private final BooleanFlag tenantServiceIdentityFlag;
// Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts
private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>();
@@ -78,7 +85,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
ConfigServerInfo configServerInfo,
String certificateDnsSuffix,
ServiceIdentityProvider hostIdentityProvider,
- boolean useInternalZts,
+ FlagSource flagSource,
Clock clock) {
this.ztsEndpoint = ztsEndpoint;
this.ztsTrustStorePath = ztsTrustStorePath;
@@ -89,24 +96,33 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
hostIdentityProvider,
new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity())));
this.clock = clock;
- this.useInternalZts = useInternalZts;
+ this.tenantServiceIdentityFlag = Flags.NODE_ADMIN_TENANT_SERVICE_REGISTRY.bindTo(flagSource);
}
public boolean converge(NodeAgentContext context) {
+ var modified = false;
+ modified |= maintain(context, NODE);
+ if (shouldWriteTenantServiceIdentity(context))
+ modified |= maintain(context, TENANT);
+ return modified;
+ }
+
+ private boolean maintain(NodeAgentContext context, IdentityType identityType) {
if (context.isDisabled(NodeAgentTask.CredentialsMaintainer)) return false;
try {
context.log(logger, Level.FINE, "Checking certificate");
- ContainerPath containerSiaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa());
- ContainerPath privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(containerSiaDirectory, context.identity());
- ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(containerSiaDirectory, context.identity());
- ContainerPath identityDocumentFile = containerSiaDirectory.resolve("vespa-node-identity-document.json");
+ ContainerPath siaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa());
+ ContainerPath identityDocumentFile = siaDirectory.resolve(identityType.getIdentityDocument());
+ AthenzIdentity athenzIdentity = getAthenzIdentity(context, identityType, identityDocumentFile);
+ ContainerPath privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(siaDirectory, athenzIdentity);
+ ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(siaDirectory, athenzIdentity);
if (!Files.exists(privateKeyFile) || !Files.exists(certificateFile) || !Files.exists(identityDocumentFile)) {
context.log(logger, "Certificate/private key/identity document file does not exist");
Files.createDirectories(privateKeyFile.getParent());
Files.createDirectories(certificateFile.getParent());
Files.createDirectories(identityDocumentFile.getParent());
- registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile);
+ registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType);
return true;
}
@@ -116,11 +132,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
var doc = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile);
if (doc.outdated()) {
context.log(logger, "Identity document is outdated (version=%d)", doc.documentVersion());
- registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile);
+ registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType);
return true;
} else if (isCertificateExpired(expiry, now)) {
context.log(logger, "Certificate has expired (expiry=%s)", expiry.toString());
- registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile);
+ registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType);
return true;
}
@@ -134,7 +150,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
return false;
} else {
lastRefreshAttempt.put(context.containerName(), now);
- refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc);
+ refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc, identityType);
return true;
}
}
@@ -182,9 +198,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
now)) > 0;
}
- private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile) {
+ private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, IdentityType identityType) {
KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
- SignedIdentityDocument doc = identityDocumentClient.getNodeIdentityDocument(context.hostname().value());
+ SignedIdentityDocument doc = signedIdentityDocument(context, identityType);
CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName());
Pkcs10Csr csr = csrGenerator.generateInstanceCsr(
context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair);
@@ -214,7 +230,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
.orElse(ztsEndpoint);
}
private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile,
- ContainerPath identityDocumentFile, SignedIdentityDocument doc) {
+ ContainerPath identityDocumentFile, SignedIdentityDocument doc, IdentityType identityType) {
KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName());
Pkcs10Csr csr = csrGenerator.generateInstanceCsr(
@@ -239,7 +255,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
} catch (ZtsClientException e) {
if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) {
context.log(logger, Level.SEVERE, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e);
- registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile);
+ registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType);
} else {
throw e;
}
@@ -272,4 +288,46 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer {
private static boolean isCertificateExpired(Instant expiry, Instant now) {
return now.isAfter(expiry.minus(EXPIRY_MARGIN));
}
+
+ private SignedIdentityDocument signedIdentityDocument(NodeAgentContext context, IdentityType identityType) {
+ return switch (identityType) {
+ case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value());
+ case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value());
+ };
+ }
+
+ private AthenzIdentity getAthenzIdentity(NodeAgentContext context, IdentityType identityType, ContainerPath identityDocumentFile) {
+ return switch (identityType) {
+ case NODE -> context.identity();
+ case TENANT -> getTenantIdentity(context, identityDocumentFile);
+ };
+ }
+
+ private AthenzIdentity getTenantIdentity(NodeAgentContext context, ContainerPath identityDocumentFile) {
+ if (Files.exists(identityDocumentFile)) {
+ return EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).serviceIdentity();
+ } else {
+ return identityDocumentClient.getTenantIdentityDocument(context.hostname().value()).serviceIdentity();
+ }
+ }
+
+ private boolean shouldWriteTenantServiceIdentity(NodeAgentContext context) {
+ return tenantServiceIdentityFlag
+ .with(FetchVector.Dimension.HOSTNAME, context.hostname().value())
+ .value();
+ }
+
+ enum IdentityType {
+ NODE("vespa-node-identity-document.json"),
+ TENANT("vespa-tenant-identity-document.json");
+
+ private String identityDocument;
+ IdentityType(String identityDocument) {
+ this.identityDocument = identityDocument;
+ }
+
+ public String getIdentityDocument() {
+ return identityDocument;
+ }
+ }
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
index 3a12191a0de..c743f1c8c85 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
@@ -71,11 +71,14 @@ public class SyncFileInfo {
if (filename.startsWith("vespa.log")) {
dir = "logs/vespa/";
compression = Compression.ZSTD;
- minDurationBetweenSync = filename.length() == 9 ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null;
+ if (filename.length() == 9) {
+ if (!rotatedOnly) remoteFilename = "vespa.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime());
+ minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO;
+ }
} else if (filename.startsWith("zookeeper.") && filename.endsWith(".log")) {
compression = Compression.ZSTD;
dir = "logs/zookeeper/";
- remoteFilename = filename.endsWith(".0.log") ? "zookeeper.log" :
+ remoteFilename = rotatedOnly && filename.endsWith(".0.log") ? "zookeeper.log" :
"zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime());
minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null;
} else {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java
index c7d34a12f43..9662d4184df 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java
@@ -91,7 +91,7 @@ public class SystemCtl {
public String getServiceProperty(TaskContext context, String unit, String property) {
return newCommandLine(context)
.add("systemctl", "show", "--property", property, "--value", unit + ".service")
- .execute()
+ .executeSilently()
.getOutput();
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java
index f49dd2e705b..b35f4d6c790 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java
@@ -16,6 +16,7 @@ import org.mockito.ArgumentCaptor;
import java.nio.file.FileSystem;
import java.nio.file.Path;
+import java.time.Instant;
import java.util.List;
import java.util.Optional;
@@ -40,6 +41,8 @@ class CoresTest {
private final HostName hostname = HostName.of("foo.com");
private final String id = "5c987afb-347a-49ee-a0c5-bef56bbddeb0";
private final CoreDumpMetadata metadata = new CoreDumpMetadata()
+ .setType(CoreDumpMetadata.Type.OOM)
+ .setCreated(Instant.ofEpochMilli(12345678))
.setKernelVersion("4.18.0-372.26.1.el8_6.x86_64")
.setCpuMicrocodeVersion("0x1000065")
.setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813"))
@@ -83,9 +86,11 @@ class CoresTest {
"bin_path": "/usr/bin/java",
"coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813",
"cpu_microcode_version": "0x1000065",
+ "created": 12345678,
"decryption_token": "987def",
"docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8",
"kernel_version": "4.18.0-372.26.1.el8_6.x86_64",
+ "type": "OOM",
"vespa_version": "8.68.8"
}""",
JsonTestHelper.normalize(uncheck(() -> mapper.writeValueAsString(bodyJsonPojoCaptor.getValue()))));
@@ -128,9 +133,11 @@ class CoresTest {
"bin_path": "/usr/bin/java",
"coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813",
"cpu_microcode_version": "0x1000065",
+ "created": 12345678,
"decryption_token": "987def",
"docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8",
"kernel_version": "4.18.0-372.26.1.el8_6.x86_64",
+ "type": "OOM",
"vespa_version": "8.68.8"
}""",
JsonTestHelper.normalize(new UnixPath(path).readUtf8File()));
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
index 4fa18c71da0..b4a35d6012c 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java
@@ -5,11 +5,13 @@ import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata;
import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath;
import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
import com.yahoo.vespa.test.file.TestFileSystem;
import org.junit.jupiter.api.Test;
+import java.time.Instant;
import java.util.List;
import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8;
@@ -22,12 +24,18 @@ import static org.mockito.Mockito.when;
* @author freva
*/
public class CoreCollectorTest {
+ private static final Instant CORE_CREATED = Instant.ofEpochMilli(2233445566L);
+
private final ContainerOperations docker = mock(ContainerOperations.class);
private final CoreCollector coreCollector = new CoreCollector(docker);
private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld")
.fileSystem(TestFileSystem.create()).build();
- private final ContainerPath TEST_CORE_PATH = context.paths().of("/tmp/core.1234");
+ private final ContainerPath TEST_CORE_PATH = (ContainerPath) new UnixPath(context.paths().of("/tmp/core.1234"))
+ .createParents()
+ .createNewFile()
+ .setLastModifiedTime(CORE_CREATED)
+ .toPath();
private final String TEST_BIN_PATH = "/usr/bin/program";
private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]",
"Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.",
@@ -143,6 +151,8 @@ public class CoreCollectorTest {
String.join("\n", GDB_BACKTRACE));
var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH)
+ .setCreated(CORE_CREATED)
+ .setType(CoreDumpMetadata.Type.CORE_DUMP)
.setBacktrace(GDB_BACKTRACE)
.setBacktraceAllThreads(GDB_BACKTRACE);
assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH));
@@ -156,7 +166,7 @@ public class CoreCollectorTest {
mockExec(new String[]{GDB_PATH_RHEL8 + " -n -ex set print frame-arguments none -ex bt -batch /usr/bin/program /tmp/core.1234"},
"", "Failure");
- var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH);
+ var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH).setCreated(CORE_CREATED).setType(CoreDumpMetadata.Type.CORE_DUMP);
assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH));
}
@@ -174,6 +184,8 @@ public class CoreCollectorTest {
jstack);
var expected = new CoreDumpMetadata().setBinPath(jdkPath)
+ .setCreated(CORE_CREATED)
+ .setType(CoreDumpMetadata.Type.CORE_DUMP)
.setBacktraceAllThreads(List.of(jstack));
assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH));
}
@@ -181,9 +193,14 @@ public class CoreCollectorTest {
@Test
void metadata_for_java_heap_dump() {
var expected = new CoreDumpMetadata().setBinPath("java")
+ .setType(CoreDumpMetadata.Type.JVM_HEAP)
+ .setCreated(CORE_CREATED)
.setBacktrace(List.of("Heap dump, no backtrace available"));
- assertEquals(expected, coreCollector.collect(context, context.paths().of("/dump_java_pid123.hprof")));
+ assertEquals(expected, coreCollector.collect(context, (ContainerPath) new UnixPath(context.paths().of("/dump_java_pid123.hprof"))
+ .createNewFile()
+ .setLastModifiedTime(CORE_CREATED)
+ .toPath()));
}
private void mockExec(String[] cmd, String output) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
index 3c91a9f32d1..b7aee6706b1 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
@@ -74,8 +74,9 @@ public class SyncFileInfoTest {
@Test
void vespa_logs() {
+ new UnixPath(vespaLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z"));
assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true);
- assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ZERO, false);
+ assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2022-05-09.14-22-11.zst", ZSTD, Duration.ZERO, false);
assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true);
assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false);
@@ -83,8 +84,9 @@ public class SyncFileInfoTest {
@Test
void zookeeper_logs() {
+ new UnixPath(zkLogPath0).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-13T13:13:45Z"));
assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true);
- assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ZERO, false);
+ assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-13.13-13-45.zst", ZSTD, Duration.ZERO, false);
new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z"));
assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
index 4c9fab748d1..856d6e07156 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
@@ -59,7 +59,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
failures++;
}
}
- return asSuccessFactor(attempts, failures);
+ return asSuccessFactorDeviation(attempts, failures);
}
/**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java
index acd5cb61d81..6f2eb726e91 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java
@@ -51,6 +51,6 @@ public class DiskReplacer extends NodeRepositoryMaintainer {
log.log(Level.WARNING, "Failed to rebuild " + host.hostname() + ", will retry in " + interval(), e);
}
}
- return this.asSuccessFactor(nodes.size(), failures);
+ return this.asSuccessFactorDeviation(nodes.size(), failures);
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
index 83dadddf76c..a4bc3a1aea5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
@@ -113,7 +113,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer {
}
success++;
}
- return asSuccessFactor(attempts, attempts - success);
+ return asSuccessFactorDeviation(attempts, attempts - success);
}
/**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java
index 7ecfc8f7926..0fa16f22061 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java
@@ -53,7 +53,7 @@ public class HostDeprovisioner extends NodeRepositoryMaintainer {
log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e);
}
}
- return asSuccessFactor(hosts.size(), failures);
+ return asSuccessFactorDeviation(hosts.size(), failures);
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java
index 86c5a926900..3c77725298d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java
@@ -63,7 +63,7 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer {
}
}
}
- return asSuccessFactor(hosts.size(), failures);
+ return asSuccessFactorDeviation(hosts.size(), failures);
}
private void setIpConfig(Node host, NodeList children, HostIpConfig hostIpConfig) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
index f864ab18920..baa2e596b36 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -94,7 +94,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer {
.collect(Collectors.joining(", ")),
interval()));
}
- return asSuccessFactor(attempts.get(), failed.size());
+ return asSuccessFactorDeviation(attempts.get(), failed.size());
}
/** Remove reals from inactive load balancers */
@@ -131,7 +131,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer {
interval()),
lastException.get());
}
- return asSuccessFactor(attempts.get(), failed.size());
+ return asSuccessFactorDeviation(attempts.get(), failed.size());
}
/** Patch load balancers matching given filter, while holding lock */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index 5af74214648..e6cfe8ca6b5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -176,6 +176,9 @@ public class MetricsReporter extends NodeRepositoryMaintainer {
boolean converged = currentVersion.isPresent() &&
currentVersion.get().equals(wantedVersion);
metric.set("wantToChangeVespaVersion", converged ? 0 : 1, context);
+ if (node.cloudAccount().isEnclave(nodeRepository().zone())) {
+ metric.set("hasWireguardKey", node.wireguardPubKey().isPresent() ? 1 : 0, context);
+ }
} else {
context = getContext(Map.of("state", node.state().name(),
"host", node.hostname()));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index afea08711fa..f9ff2f08375 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -97,7 +97,7 @@ public class NodeFailer extends NodeRepositoryMaintainer {
metric.set(throttlingActiveMetric, throttlingActive, null);
metric.set(throttledHostFailuresMetric, throttledHostFailures, null);
metric.set(throttledNodeFailuresMetric, throttledNodeFailures, null);
- return asSuccessFactor(attempts, failures);
+ return asSuccessFactorDeviation(attempts, failures);
}
private Collection<FailingNode> findActiveFailingNodes() {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java
index 781debe26a0..979a2771082 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java
@@ -77,7 +77,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer {
failures.add(1);
}
});
- return asSuccessFactor(attempts.get(), failures.get());
+ return asSuccessFactorDeviation(attempts.get(), failures.get());
}
/**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
index b299369db1a..e28dac1c915 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
@@ -55,10 +55,10 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer {
nodeRepository().metricsDb().gc();
- return asSuccessFactor(attempts, failures.get());
+ return asSuccessFactorDeviation(attempts, failures.get());
}
catch (InterruptedException e) {
- return asSuccessFactor(attempts, failures.get());
+ return asSuccessFactorDeviation(attempts, failures.get());
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java
index 3c00e3b708d..b8d37c7eb5c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java
@@ -59,7 +59,7 @@ public abstract class NodeRepositoryMaintainer extends Maintainer {
@Override
public void completed(String job, double successFactor, long duration) {
var context = metric.createContext(Map.of("job", job));
- metric.set("maintenance.successFactor", successFactor, context);
+ metric.set("maintenance.successFactorDeviation", successFactor, context);
metric.set("maintenance.duration", duration, context);
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
index af368934188..4071559d841 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java
@@ -48,7 +48,7 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer {
failures++;
}
}
- return asSuccessFactor(attempts, failures);
+ return asSuccessFactorDeviation(attempts, failures);
}
private Applications applications() {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
index 178ea6ed514..35b2fef2c78 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
@@ -1,14 +1,14 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
-import com.yahoo.component.annotation.Inject;
+import ai.vespa.http.DomainName;
import com.yahoo.component.Version;
+import com.yahoo.component.annotation.Inject;
import com.yahoo.config.provision.ActivationContext;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ApplicationTransaction;
import com.yahoo.config.provision.Deployment;
import com.yahoo.config.provision.HostFilter;
-import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.InfraDeployer;
import com.yahoo.config.provision.NodeType;
@@ -116,7 +116,7 @@ public class InfraDeployerImpl implements InfraDeployer {
duperModel.infraApplicationActivated(
application.getApplicationId(),
- hostSpecs.stream().map(HostSpec::hostname).map(HostName::of).toList());
+ hostSpecs.stream().map(HostSpec::hostname).map(DomainName::of).toList());
logger.log(Level.FINE, () -> generateActivationLogMessage(hostSpecs, application.getApplicationId()));
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
index 04f64b070b3..890d190c24e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -109,7 +109,7 @@ public class LoadBalancerProvisioner {
public void activate(Set<ClusterSpec> clusters, NodeList newActive, ApplicationTransaction transaction) {
Map<ClusterSpec.Id, ZoneEndpoint> activatingClusters = clusters.stream()
// .collect(Collectors.toMap(ClusterSpec::id, ClusterSpec::zoneEndpoint));
- // TODO: this dies with combined clusters Ü
+ // TODO: this dies with combined clusters
.collect(groupingBy(LoadBalancerProvisioner::effectiveId,
reducing(ZoneEndpoint.defaultEndpoint,
ClusterSpec::zoneEndpoint,
@@ -193,14 +193,13 @@ public class LoadBalancerProvisioner {
Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id);
LoadBalancer newLoadBalancer;
LoadBalancer.State fromState = loadBalancer.map(LoadBalancer::state).orElse(null);
- if ( loadBalancer.isPresent()
- && ( ! inAccount(cloudAccount, loadBalancer.get())
- || ! hasCorrectVisibility(loadBalancer.get(), zoneEndpoint))) {
- // We have a load balancer, but with the wrong account or visibility.
- // Load balancer must be removed before we can provision a new one with the wanted visibility
- newLoadBalancer = loadBalancer.get().with(LoadBalancer.State.removable, now);
- }
- else {
+ boolean recreateLoadBalancer = loadBalancer.isPresent() && (!inAccount(cloudAccount, loadBalancer.get())
+ || !hasCorrectVisibility(loadBalancer.get(), zoneEndpoint));
+ if (recreateLoadBalancer) {
+ // We have a load balancer, but with the wrong account or visibility.
+ // Load balancer must be removed before we can provision a new one with the wanted visibility
+ newLoadBalancer = loadBalancer.get().with(LoadBalancer.State.removable, now);
+ } else {
Optional<LoadBalancerInstance> instance = provisionInstance(id, loadBalancer, zoneEndpoint, cloudAccount);
newLoadBalancer = loadBalancer.isEmpty() ? new LoadBalancer(id, instance, LoadBalancer.State.reserved, now)
: loadBalancer.get().with(instance);
@@ -211,8 +210,8 @@ public class LoadBalancerProvisioner {
}
private static boolean hasCorrectVisibility(LoadBalancer newLoadBalancer, ZoneEndpoint zoneEndpoint) {
- return newLoadBalancer.instance().isEmpty()
- || newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint();
+ return newLoadBalancer.instance().isEmpty() ||
+ newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint();
}
private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, ZoneEndpoint settings, NodeList nodes) {
@@ -320,14 +319,6 @@ public class LoadBalancerProvisioner {
return loadBalancer.instance().isEmpty() || loadBalancer.instance().get().cloudAccount().equals(cloudAccount);
}
- /** Returns whether load balancer has given reals, and settings if specified */
- private static boolean isUpToDate(LoadBalancer loadBalancer, Set<Real> reals, ZoneEndpoint zoneEndpoint) {
- if (loadBalancer.instance().isEmpty())
- throw new IllegalStateException("Expected a load balancer instance to be present, for " + loadBalancer.id());
- return loadBalancer.instance().get().reals().equals(reals)
- && loadBalancer.instance().get().settings().equals(zoneEndpoint);
- }
-
/** Find IP addresses reachable by the load balancer service */
private Set<String> reachableIpAddresses(Node node) {
Set<String> reachable = new LinkedHashSet<>(node.ipConfig().primary());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
index 15126ed5845..06ce08df8c3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
@@ -1,9 +1,9 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.testutils;
+import ai.vespa.http.DomainName;
import com.yahoo.component.annotation.Inject;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
import com.yahoo.vespa.service.monitor.InfraApplicationApi;
@@ -20,7 +20,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class MockDuperModel implements DuperModelInfraApi {
private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>();
- private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<ApplicationId, List<DomainName>> activeApps = new ConcurrentHashMap<>();
@Inject
public MockDuperModel() {
@@ -46,12 +46,12 @@ public class MockDuperModel implements DuperModelInfraApi {
return activeApps.containsKey(applicationId);
}
- public List<HostName> hostnamesOf(ApplicationId applicationId) {
+ public List<DomainName> hostnamesOf(ApplicationId applicationId) {
return activeApps.getOrDefault(applicationId, List.of());
}
@Override
- public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) {
+ public void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames) {
activeApps.put(applicationId, hostnames);
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java
index c7c6e770fe3..611594cc72e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java
@@ -44,7 +44,7 @@ public class NodeMetricsDbMaintainerTest {
fetcher,
Duration.ofHours(1),
new TestMetric());
- assertEquals(maintainer.maintain(), 1.0, 0.0000001);
+ assertEquals(maintainer.maintain(), 0.0, 0.0000001);
List<NodeTimeseries> timeseriesList = tester.nodeRepository().metricsDb().getNodeTimeseries(Duration.ofDays(1),
Set.of("host-1.yahoo.com", "host-2.yahoo.com"));
assertEquals(2, timeseriesList.size());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index 58f4be18992..aaa4bffc000 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.maintenance;
+import ai.vespa.http.DomainName;
import com.yahoo.component.Version;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ApplicationName;
@@ -268,7 +269,7 @@ public class RetiredExpirerTest {
private Set<String> configServerHostnames(MockDuperModel duperModel) {
return duperModel.hostnamesOf(new ConfigServerApplication().getApplicationId()).stream()
- .map(com.yahoo.config.provision.HostName::value)
+ .map(DomainName::value)
.collect(Collectors.toSet());
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
index 9cd5adef5f4..7763459dd92 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
@@ -1,11 +1,11 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
+import ai.vespa.http.DomainName;
import com.yahoo.component.Version;
import com.yahoo.config.provision.ApplicationTransaction;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.Provisioner;
@@ -133,7 +133,7 @@ public class InfraDeployerImplTest {
@SuppressWarnings("unchecked")
private void verifyActivated(String... hostnames) {
verify(duperModelInfraApi).infraApplicationActivated(
- eq(application.getApplicationId()), eq(Stream.of(hostnames).map(HostName::of).toList()));
+ eq(application.getApplicationId()), eq(Stream.of(hostnames).map(DomainName::of).toList()));
ArgumentMatcher<ApplicationTransaction> transactionMatcher = t -> {
assertEquals(application.getApplicationId(), t.application());
return true;
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
index 323ae678b0b..73774321ffb 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.orchestrator;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.SuperModel;
import com.yahoo.config.model.api.SuperModelListener;
import com.yahoo.config.model.api.SuperModelProvider;
@@ -87,9 +88,9 @@ public class OrchestratorTest {
// There is one config server application with 3 nodes
ApplicationId applicationId = new ConfigServerApplication().getApplicationId();
- var cfg1 = com.yahoo.config.provision.HostName.of("cfg1");
- var cfg2 = com.yahoo.config.provision.HostName.of("cfg2");
- var cfg3 = com.yahoo.config.provision.HostName.of("cfg3");
+ var cfg1 = DomainName.of("cfg1");
+ var cfg2 = DomainName.of("cfg2");
+ var cfg3 = DomainName.of("cfg3");
duperModelManager.infraApplicationActivated(applicationId, List.of(cfg1, cfg2, cfg3));
duperModelManager.infraApplicationsIsNowComplete();
@@ -135,7 +136,7 @@ public class OrchestratorTest {
// etc (should be the same as for cfg1)
}
- private HostName toApplicationModelHostName(com.yahoo.config.provision.HostName hostname) {
+ private HostName toApplicationModelHostName(DomainName hostname) {
return new HostName(hostname.value());
}
diff --git a/parent/pom.xml b/parent/pom.xml
index b0ba6e89f6e..ace6a47d6bc 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -1170,7 +1170,7 @@
<org.lz4.version>1.8.0</org.lz4.version>
<prometheus.client.version>0.6.0</prometheus.client.version>
<protobuf.version>3.21.7</protobuf.version>
- <spifly.version>1.3.5</spifly.version>
+ <spifly.version>1.3.6</spifly.version>
<surefire.version>3.0.0-M9</surefire.version>
<wiremock.version>2.35.0</wiremock.version>
<zero-allocation-hashing.version>0.16</zero-allocation-hashing.version>
diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh
index 8b61dde4e1d..33231890626 100755
--- a/screwdriver/release-container-image.sh
+++ b/screwdriver/release-container-image.sh
@@ -57,7 +57,8 @@ for data in "Dockerfile vespa"; do
--jobs 2 \
--layers=false \
--manifest "vespaengine/$IMAGE_NAME:$VESPA_VERSION" \
- --platform linux/amd64,linux/arm64 | cat
+ --platform linux/amd64,linux/arm64 \
+ --squash | cat
# Test
buildah tag vespaengine/$IMAGE_NAME:$VESPA_VERSION vespaengine/$IMAGE_NAME:latest
diff --git a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt
index 5955ee52756..6e285423931 100644
--- a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt
+++ b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt
@@ -4,5 +4,6 @@ vespa_add_executable(searchcore_attribute_updater_test_app TEST
attribute_updater_test.cpp
DEPENDS
searchcore_pcommon
+ searchlib_test
)
vespa_add_test(NAME searchcore_attribute_updater_test_app COMMAND searchcore_attribute_updater_test_app)
diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp
index 037bf884e6b..892be2c874f 100644
--- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp
+++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp
@@ -6,6 +6,7 @@
#include <vespa/searchlib/attribute/single_raw_attribute.h>
#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h>
+#include <vespa/searchlib/test/attribute_builder.h>
#include <vespa/searchlib/test/weighted_type_test_utils.h>
#include <vespa/searchcommon/attribute/config.h>
#include <vespa/document/base/testdocrepo.h>
@@ -53,6 +54,7 @@ using search::attribute::Config;
using search::attribute::Reference;
using search::attribute::ReferenceAttribute;
using search::attribute::SingleRawAttribute;
+using search::attribute::test::AttributeBuilder;
using search::tensor::ITensorAttribute;
using search::tensor::DenseTensorAttribute;
using search::tensor::SerializedFastValueAttribute;
@@ -124,19 +126,19 @@ struct Fixture {
void applyArrayUpdates(AttributeVector & vec, std::unique_ptr<FieldValue> assign,
std::unique_ptr<FieldValue> first, std::unique_ptr<FieldValue> second) {
- applyValueUpdate(vec, 0, std::make_unique<AssignValueUpdate>(std::move(assign)));
- applyValueUpdate(vec, 1, std::make_unique<AddValueUpdate>(std::move(second)));
- applyValueUpdate(vec, 2, std::make_unique<RemoveValueUpdate>(std::move(first)));
- applyValueUpdate(vec, 3, std::make_unique<ClearValueUpdate>());
+ applyValueUpdate(vec, 1, std::make_unique<AssignValueUpdate>(std::move(assign)));
+ applyValueUpdate(vec, 2, std::make_unique<AddValueUpdate>(std::move(second)));
+ applyValueUpdate(vec, 3, std::make_unique<RemoveValueUpdate>(std::move(first)));
+ applyValueUpdate(vec, 4, std::make_unique<ClearValueUpdate>());
}
void applyWeightedSetUpdates(AttributeVector & vec, std::unique_ptr<FieldValue> assign,
std::unique_ptr<FieldValue> first, std::unique_ptr<FieldValue> copyOfFirst, std::unique_ptr<FieldValue> second) {
- applyValueUpdate(vec, 0, std::make_unique<AssignValueUpdate>(std::move(assign)));
- applyValueUpdate(vec, 1, std::make_unique<AddValueUpdate>(std::move(second), 20));
- applyValueUpdate(vec, 2, std::make_unique<RemoveValueUpdate>(std::move(first)));
- applyValueUpdate(vec, 3, std::make_unique<ClearValueUpdate>());
- applyValueUpdate(vec, 4, std::make_unique<MapValueUpdate>(std::move(copyOfFirst), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)));
+ applyValueUpdate(vec, 1, std::make_unique<AssignValueUpdate>(std::move(assign)));
+ applyValueUpdate(vec, 2, std::make_unique<AddValueUpdate>(std::move(second), 20));
+ applyValueUpdate(vec, 3, std::make_unique<RemoveValueUpdate>(std::move(first)));
+ applyValueUpdate(vec, 4, std::make_unique<ClearValueUpdate>());
+ applyValueUpdate(vec, 5, std::make_unique<MapValueUpdate>(std::move(copyOfFirst), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)));
}
void applyValue(AttributeVector& vec, uint32_t docid, std::unique_ptr<FieldValue> value) {
@@ -144,35 +146,6 @@ struct Fixture {
}
};
-template <typename T, typename VectorType>
-AttributePtr
-create(uint32_t numDocs, T val, int32_t weight,
- const std::string & baseName,
- const Config &info)
-{
- LOG(info, "create attribute vector: %s", baseName.c_str());
- AttributePtr vec = AttributeFactory::createAttribute(baseName, info);
- VectorType * api = static_cast<VectorType *>(vec.get());
- for (uint32_t i = 0; i < numDocs; ++i) {
- if (!api->addDoc(i)) {
- LOG(info, "failed adding doc: %u", i);
- return AttributePtr();
- }
- if (api->hasMultiValue()) {
- if (!api->append(i, val, weight)) {
- LOG(info, "failed append to doc: %u", i);
- }
- } else {
- if (!api->update(i, val)) {
- LOG(info, "failed update doc: %u", i);
- return AttributePtr();
- }
- }
- }
- api->commit();
- return vec;
-}
-
template <typename T>
bool
check(const AttributePtr &vec, uint32_t docId, const std::vector<T> &values)
@@ -223,70 +196,67 @@ TEST_F("require that single attributes are updated", Fixture)
using search::attribute::getUndefined;
CollectionType ct(CollectionType::SINGLE);
{
- BasicType bt(BasicType::INT32);
- AttributePtr vec = create<int32_t, IntegerAttribute>(3, 32, 0, "in1/int", Config(bt, ct));
- f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(64)));
- f.applyValueUpdate(*vec, 1, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10));
- f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>());
- EXPECT_EQUAL(3u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64)}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(42)}));
- EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(getUndefined<int32_t>())}));
+ auto vec = AttributeBuilder("in1/int", Config(BasicType::INT32)).fill({ 32, 32, 32}).get();
+ f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(64)));
+ f.applyValueUpdate(*vec, 2, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10));
+ f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>());
+ EXPECT_EQUAL(4u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64)}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(42)}));
+ EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{WeightedInt(getUndefined<int32_t>())}));
}
{
- BasicType bt(BasicType::FLOAT);
- AttributePtr vec = create<float, FloatingPointAttribute>(3, 55.5f, 0, "in1/float",Config(bt, ct));
- f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<FloatFieldValue>(77.7f)));
- f.applyValueUpdate(*vec, 1, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10));
- f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>());
- EXPECT_EQUAL(3u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f)}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(65.5f)}));
- EXPECT_TRUE(std::isnan(vec->getFloat(2)));
+ auto vec = AttributeBuilder("in1/float", Config(BasicType::FLOAT)).fill({55.5, 55.5, 55.5}).get();
+ f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<FloatFieldValue>(77.7f)));
+ f.applyValueUpdate(*vec, 2, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10));
+ f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>());
+ EXPECT_EQUAL(4u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f)}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(65.5f)}));
+ EXPECT_TRUE(std::isnan(vec->getFloat(3)));
}
{
- BasicType bt(BasicType::STRING);
- AttributePtr vec = create<std::string, StringAttribute>(3, "first", 0, "in1/string",Config(bt, ct));
- f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(StringFieldValue::make("second")));
- f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>());
- EXPECT_EQUAL(3u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second")}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first")}));
- EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("")}));
+ auto vec = AttributeBuilder("in1/string", Config(BasicType::STRING)).fill({"first", "first", "first"}).get();
+ f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(StringFieldValue::make("second")));
+ f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>());
+ EXPECT_EQUAL(4u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second")}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first")}));
+ EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{WeightedString("")}));
}
{
BasicType bt(BasicType::REFERENCE);
Config cfg(bt, ct);
AttributePtr vec = AttributeFactory::createAttribute("in1/ref", cfg);
+ vec->addReservedDoc();
uint32_t startDoc = 0;
uint32_t endDoc = 0;
EXPECT_TRUE(vec->addDocs(startDoc, endDoc, 3));
- EXPECT_EQUAL(0u, startDoc);
- EXPECT_EQUAL(2u, endDoc);
- for (uint32_t docId = 0; docId < 3; ++docId) {
+ EXPECT_EQUAL(1u, startDoc);
+ EXPECT_EQUAL(3u, endDoc);
+ for (uint32_t docId = 1; docId < 4; ++docId) {
asReferenceAttribute(*vec).update(docId, toGid(doc1));
}
vec->commit();
- f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<ReferenceFieldValue>(dynamic_cast<const ReferenceDataType &>(f.docType->getField("ref").getDataType()), DocumentId(doc2))));
- f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>());
- EXPECT_EQUAL(3u, vec->getNumDocs());
- TEST_DO(assertRef(*vec, doc2, 0));
- TEST_DO(assertRef(*vec, doc1, 1));
- TEST_DO(assertNoRef(*vec, 2));
+ f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<ReferenceFieldValue>(dynamic_cast<const ReferenceDataType &>(f.docType->getField("ref").getDataType()), DocumentId(doc2))));
+ f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>());
+ EXPECT_EQUAL(4u, vec->getNumDocs());
+ TEST_DO(assertRef(*vec, doc2, 1));
+ TEST_DO(assertRef(*vec, doc1, 2));
+ TEST_DO(assertNoRef(*vec, 3));
}
{
- BasicType bt(BasicType::RAW);
vespalib::string first_backing("first");
vespalib::ConstArrayRef<char> first(first_backing.data(), first_backing.size());
- AttributePtr vec = create<vespalib::ConstArrayRef<char>, SingleRawAttribute>(4, first, 0, "in1/raw", Config(bt, ct));
- f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<RawFieldValue>("second")));
- f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>());
- f.applyValue(*vec, 3, std::make_unique<RawFieldValue>("third"));
- EXPECT_EQUAL(4u, vec->getNumDocs());
- EXPECT_EQUAL(as_vector("second"), as_vector(vec->get_raw(0)));
- EXPECT_EQUAL(as_vector("first"), as_vector(vec->get_raw(1)));
- EXPECT_EQUAL(as_vector(""), as_vector(vec->get_raw(2)));
- EXPECT_EQUAL(as_vector("third"), as_vector(vec->get_raw(3)));
+ auto vec = AttributeBuilder("in1/raw", Config(BasicType::RAW)).fill({first, first, first, first}).get();
+ f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<RawFieldValue>("second")));
+ f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>());
+ f.applyValue(*vec, 4, std::make_unique<RawFieldValue>("third"));
+ EXPECT_EQUAL(5u, vec->getNumDocs());
+ EXPECT_EQUAL(as_vector("second"), as_vector(vec->get_raw(1)));
+ EXPECT_EQUAL(as_vector("first"), as_vector(vec->get_raw(2)));
+ EXPECT_EQUAL(as_vector(""), as_vector(vec->get_raw(3)));
+ EXPECT_EQUAL(as_vector("third"), as_vector(vec->get_raw(4)));
}
}
@@ -294,52 +264,51 @@ TEST_F("require that array attributes are updated", Fixture)
{
CollectionType ct(CollectionType::ARRAY);
{
- BasicType bt(BasicType::INT32);
- AttributePtr vec = create<int32_t, IntegerAttribute>(5, 32, 1, "in1/aint", Config(bt, ct));
+ using IL = AttributeBuilder::IntList;
+ auto vec = AttributeBuilder("in1/aint", Config(BasicType::INT32, ct)).fill_array({IL{32}, {32}, {32}, {32}, {32}}).get();
auto first = std::make_unique<IntFieldValue>(32);
auto second = std::make_unique<IntFieldValue>(64);
auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("aint").getDataType());
assign->add(*second);
f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second));
- EXPECT_EQUAL(5u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64)}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(32), WeightedInt(64)}));
- EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{}));
+ EXPECT_EQUAL(6u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64)}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(32), WeightedInt(64)}));
EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{}));
- EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{WeightedInt(32)}));
+ EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{}));
+ EXPECT_TRUE(check(vec, 5, std::vector<WeightedInt>{WeightedInt(32)}));
}
{
- BasicType bt(BasicType::FLOAT);
- AttributePtr vec = create<float, FloatingPointAttribute>(5, 55.5f, 1, "in1/afloat", Config(bt, ct));
+ using DL = AttributeBuilder::DoubleList;
+ auto vec = AttributeBuilder("in1/afloat", Config(BasicType::FLOAT, ct)).fill_array({DL{55.5}, {55.5}, {55.5}, {55.5}, {55.5}}).get();
auto first = std::make_unique<FloatFieldValue>(55.5f);
auto second = std::make_unique<FloatFieldValue>(77.7f);
auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("afloat").getDataType());
assign->add(*second);
f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second));
- EXPECT_EQUAL(5u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f)}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(55.5f), WeightedFloat(77.7f)}));
- EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{}));
+ EXPECT_EQUAL(6u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f)}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(55.5f), WeightedFloat(77.7f)}));
EXPECT_TRUE(check(vec, 3, std::vector<WeightedFloat>{}));
- EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{WeightedFloat(55.5f)}));
+ EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{}));
+ EXPECT_TRUE(check(vec, 5, std::vector<WeightedFloat>{WeightedFloat(55.5f)}));
}
{
- BasicType bt(BasicType::STRING);
- AttributePtr vec = create<std::string, StringAttribute>(5, "first", 1, "in1/astring", Config(bt, ct));
+ auto vec = AttributeBuilder("in1/astring", Config(BasicType::STRING, ct)).fill_array({{"first"}, {"first"}, {"first"}, {"first"}, {"first"}}).get();
auto first = StringFieldValue::make("first");
auto second = StringFieldValue::make("second");
auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("astring").getDataType());
assign->add(*second);
f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second));
- EXPECT_EQUAL(5u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second")}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first"), WeightedString("second")}));
- EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{}));
+ EXPECT_EQUAL(6u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second")}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first"), WeightedString("second")}));
EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{}));
- EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{WeightedString("first")}));
+ EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{}));
+ EXPECT_TRUE(check(vec, 5, std::vector<WeightedString>{WeightedString("first")}));
}
}
@@ -347,8 +316,8 @@ TEST_F("require that weighted set attributes are updated", Fixture)
{
CollectionType ct(CollectionType::WSET);
{
- BasicType bt(BasicType::INT32);
- AttributePtr vec = create<int32_t, IntegerAttribute>(5, 32, 100, "in1/wsint", Config(bt, ct));
+ using WIL = AttributeBuilder::WeightedIntList;
+ auto vec = AttributeBuilder("in1/wsint", Config(BasicType::INT32, ct)).fill_wset({WIL{{32, 100}}, {{32, 100}}, {{32, 100}}, {{32, 100}}, {{32, 100}}}).get();
auto first = std::make_unique<IntFieldValue>(32);
auto copyOfFirst = std::make_unique<IntFieldValue>(32);
auto second = std::make_unique<IntFieldValue>(64);
@@ -356,16 +325,16 @@ TEST_F("require that weighted set attributes are updated", Fixture)
assign->add(*second, 20);
f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second));
- EXPECT_EQUAL(5u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64, 20)}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(32, 100), WeightedInt(64, 20)}));
- EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{}));
+ EXPECT_EQUAL(6u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64, 20)}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(32, 100), WeightedInt(64, 20)}));
EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{}));
- EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{WeightedInt(32, 110)}));
+ EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{}));
+ EXPECT_TRUE(check(vec, 5, std::vector<WeightedInt>{WeightedInt(32, 110)}));
}
{
- BasicType bt(BasicType::FLOAT);
- AttributePtr vec = create<float, FloatingPointAttribute>(5, 55.5f, 100, "in1/wsfloat", Config(bt, ct));
+ using WDL = AttributeBuilder::WeightedDoubleList;
+ auto vec = AttributeBuilder("in1/wsfloat", Config(BasicType::FLOAT, ct)).fill_wset({WDL{{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}}).get();
auto first = std::make_unique<FloatFieldValue>(55.5f);
auto copyOfFirst = std::make_unique<FloatFieldValue>(55.5f);
auto second = std::make_unique<FloatFieldValue>(77.7f);
@@ -373,16 +342,15 @@ TEST_F("require that weighted set attributes are updated", Fixture)
assign->add(*second, 20);
f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second));
- EXPECT_EQUAL(5u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f, 20)}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(55.5f, 100), WeightedFloat(77.7f, 20)}));
- EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{}));
+ EXPECT_EQUAL(6u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f, 20)}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(55.5f, 100), WeightedFloat(77.7f, 20)}));
EXPECT_TRUE(check(vec, 3, std::vector<WeightedFloat>{}));
- EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{WeightedFloat(55.5f, 110)}));
+ EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{}));
+ EXPECT_TRUE(check(vec, 5, std::vector<WeightedFloat>{WeightedFloat(55.5f, 110)}));
}
{
- BasicType bt(BasicType::STRING);
- AttributePtr vec = create<std::string, StringAttribute>(5, "first", 100, "in1/wsstring", Config(bt, ct));
+ auto vec = AttributeBuilder("in1/wsstring", Config(BasicType::STRING, ct)).fill_wset({{{"first", 100}}, {{"first", 100}}, {{"first", 100}}, {{"first", 100}}, {{"first", 100}}}).get();
auto first = StringFieldValue::make("first");
auto copyOfFirst = StringFieldValue::make("first");
auto second = StringFieldValue::make("second");
@@ -390,12 +358,12 @@ TEST_F("require that weighted set attributes are updated", Fixture)
assign->add(*second, 20);
f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second));
- EXPECT_EQUAL(5u, vec->getNumDocs());
- EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second", 20)}));
- EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first", 100), WeightedString("second", 20)}));
- EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{}));
+ EXPECT_EQUAL(6u, vec->getNumDocs());
+ EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second", 20)}));
+ EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first", 100), WeightedString("second", 20)}));
EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{}));
- EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{WeightedString("first", 110)}));
+ EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{}));
+ EXPECT_TRUE(check(vec, 5, std::vector<WeightedString>{WeightedString("first", 110)}));
}
}
diff --git a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp
index 020f4ff42c1..70f81d629d5 100644
--- a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp
+++ b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp
@@ -107,7 +107,10 @@ namespace proton {
class MockDocsumFieldWriterFactory : public search::docsummary::IDocsumFieldWriterFactory
{
public:
- std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string&, const vespalib::string&, const vespalib::string&) override {
+ std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string&,
+ const vespalib::string&,
+ const vespalib::string&,
+ std::shared_ptr<search::MatchingElementsFields>) override {
return {};
}
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp
index 7eb43faac18..d2d3ccaad23 100644
--- a/searchlib/src/tests/attribute/attribute_test.cpp
+++ b/searchlib/src/tests/attribute/attribute_test.cpp
@@ -912,8 +912,8 @@ AttributeTest::testSingle()
cfg.setFastSearch(true);
AttributePtr ptr = createAttribute("sv-post-int32", cfg);
ptr->updateStat(true);
- EXPECT_EQ(339020u, ptr->getStatus().getAllocated());
- EXPECT_EQ(101852u, ptr->getStatus().getUsed());
+ EXPECT_EQ(338972u, ptr->getStatus().getAllocated());
+ EXPECT_EQ(101492u, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values);
}
@@ -934,8 +934,8 @@ AttributeTest::testSingle()
cfg.setFastSearch(true);
AttributePtr ptr = createAttribute("sv-post-float", cfg);
ptr->updateStat(true);
- EXPECT_EQ(339020, ptr->getStatus().getAllocated());
- EXPECT_EQ(101852u, ptr->getStatus().getUsed());
+ EXPECT_EQ(338972u, ptr->getStatus().getAllocated());
+ EXPECT_EQ(101492u, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testSingle<FloatingPointAttribute, double, float>(ptr, values);
}
@@ -947,8 +947,8 @@ AttributeTest::testSingle()
{
AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE));
ptr->updateStat(true);
- EXPECT_EQ(117256u + sizeof_large_string_entry, ptr->getStatus().getAllocated());
- EXPECT_EQ(53240u + sizeof_large_string_entry, ptr->getStatus().getUsed());
+ EXPECT_EQ(116528u + sizeof_large_string_entry, ptr->getStatus().getAllocated());
+ EXPECT_EQ(52760u + sizeof_large_string_entry, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testSingle<StringAttribute, string, string>(ptr, values);
}
@@ -957,8 +957,8 @@ AttributeTest::testSingle()
cfg.setFastSearch(true);
AttributePtr ptr = createAttribute("sv-fs-string", cfg);
ptr->updateStat(true);
- EXPECT_EQ(345624u + sizeof_large_string_entry, ptr->getStatus().getAllocated());
- EXPECT_EQ(105176u + sizeof_large_string_entry, ptr->getStatus().getUsed());
+ EXPECT_EQ(344848u + sizeof_large_string_entry, ptr->getStatus().getAllocated());
+ EXPECT_EQ(104408u + sizeof_large_string_entry, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testSingle<StringAttribute, string, string>(ptr, values);
}
@@ -1089,8 +1089,8 @@ AttributeTest::testArray()
{
AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY));
ptr->updateStat(true);
- EXPECT_EQ(512056u, ptr->getStatus().getAllocated());
- EXPECT_EQ(504392u, ptr->getStatus().getUsed());
+ EXPECT_EQ(487480u, ptr->getStatus().getAllocated());
+ EXPECT_EQ(479720u, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
}
@@ -1099,8 +1099,8 @@ AttributeTest::testArray()
cfg.setFastSearch(true);
AttributePtr ptr = createAttribute("flags", cfg);
ptr->updateStat(true);
- EXPECT_EQ(512056u, ptr->getStatus().getAllocated());
- EXPECT_EQ(504392u, ptr->getStatus().getUsed());
+ EXPECT_EQ(487480u, ptr->getStatus().getAllocated());
+ EXPECT_EQ(479720u, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
}
@@ -1109,8 +1109,8 @@ AttributeTest::testArray()
cfg.setFastSearch(true);
AttributePtr ptr = createAttribute("a-fs-int32", cfg);
ptr->updateStat(true);
- EXPECT_EQ(868788u, ptr->getStatus().getAllocated());
- EXPECT_EQ(606264u, ptr->getStatus().getUsed());
+ EXPECT_EQ(844116u, ptr->getStatus().getAllocated());
+ EXPECT_EQ(581232u, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values);
}
@@ -1128,8 +1128,8 @@ AttributeTest::testArray()
cfg.setFastSearch(true);
AttributePtr ptr = createAttribute("a-fs-float", cfg);
ptr->updateStat(true);
- EXPECT_EQ(868788u, ptr->getStatus().getAllocated());
- EXPECT_EQ(606264u, ptr->getStatus().getUsed());
+ EXPECT_EQ(844116u, ptr->getStatus().getAllocated());
+ EXPECT_EQ(581232u, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testArray<FloatingPointAttribute, double>(ptr, values);
}
@@ -1140,8 +1140,8 @@ AttributeTest::testArray()
{
AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY));
ptr->updateStat(true);
- EXPECT_EQ(625088u + sizeof_large_string_entry, ptr->getStatus().getAllocated());
- EXPECT_EQ(557632u + sizeof_large_string_entry, ptr->getStatus().getUsed());
+ EXPECT_EQ(599784u + sizeof_large_string_entry, ptr->getStatus().getAllocated());
+ EXPECT_EQ(532480u + sizeof_large_string_entry, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testArray<StringAttribute, string>(ptr, values);
}
@@ -1150,8 +1150,8 @@ AttributeTest::testArray()
cfg.setFastSearch(true);
AttributePtr ptr = createAttribute("afs-string", cfg);
ptr->updateStat(true);
- EXPECT_EQ(875392u + sizeof_large_string_entry, ptr->getStatus().getAllocated());
- EXPECT_EQ(609588u + sizeof_large_string_entry, ptr->getStatus().getUsed());
+ EXPECT_EQ(849992u + sizeof_large_string_entry, ptr->getStatus().getAllocated());
+ EXPECT_EQ(584148u + sizeof_large_string_entry, ptr->getStatus().getUsed());
addDocs(ptr, numDocs);
testArray<StringAttribute, string>(ptr, values);
}
@@ -2334,6 +2334,10 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib
size_t rounded_size = vespalib::round_up_to_page_size(1);
size_t lid_mapping_size = 1200;
size_t sv_maxlid = 1200;
+ if (rounded_size == 16_Ki) {
+ lid_mapping_size = 4200;
+ sv_maxlid = 1300;
+ }
if (rounded_size == 64_Ki) {
lid_mapping_size = 17000;
sv_maxlid = 1500;
diff --git a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp
index 8f056323733..3f775e99891 100644
--- a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp
+++ b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp
@@ -1,9 +1,25 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include <vespa/vespalib/gtest/gtest.h>
+#include <vespa/eval/eval/fast_value.h>
+#include <vespa/eval/eval/tensor_spec.h>
+#include <vespa/eval/eval/value.h>
+#include <vespa/eval/eval/value_codec.h>
+#include <vespa/searchcommon/attribute/config.h>
#include <vespa/searchlib/attribute/extendableattributes.h>
#include <vespa/searchlib/attribute/single_raw_ext_attribute.h>
+#include <vespa/searchlib/tensor/tensor_ext_attribute.h>
+#include <vespa/searchlib/tensor/vector_bundle.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+using search::attribute::Config;
+using search::attribute::BasicType;
+using search::attribute::CollectionType;
using search::attribute::SingleRawExtAttribute;
+using search::tensor::TensorExtAttribute;
+using vespalib::eval::FastValueBuilderFactory;
+using vespalib::eval::TensorSpec;
+using vespalib::eval::Value;
+using vespalib::eval::ValueType;
namespace search {
@@ -15,8 +31,46 @@ std::vector<char> as_vector(vespalib::ConstArrayRef<char> value) {
return {value.data(), value.data() + value.size()};
}
+std::vector<double> as_vector(vespalib::ConstArrayRef<double> value) {
+ return {value.data(), value.data() + value.size()};
+}
+
+vespalib::string vec_2d_spec("tensor(x[2])");
+vespalib::string vec_mixed_2d_spec("tensor(a{},x[2])");
+
+TensorSpec
+vec_2d(double x0, double x1)
+{
+ return TensorSpec(vec_2d_spec).add({{"x", 0}}, x0).add({{"x", 1}}, x1);
+}
+
+TensorSpec
+vec_mixed_2d(std::vector<std::vector<double>> val)
+{
+ TensorSpec spec(vec_mixed_2d_spec);
+ for (uint32_t a = 0; a < val.size(); ++a) {
+ vespalib::asciistream a_stream;
+ a_stream << a;
+ vespalib::string a_as_string = a_stream.str();
+ for (uint32_t x = 0; x < val[a].size(); ++x) {
+ spec.add({{"a", a_as_string.c_str()},{"x", x}}, val[a][x]);
+ }
+ }
+ return spec;
+}
+
+void add_doc(AttributeVector& attr, uint32_t exp_docid)
+{
+ uint32_t docid(0);
+ EXPECT_EQ(exp_docid, attr.getNumDocs());
+ attr.addDoc(docid);
+ EXPECT_EQ(exp_docid, docid);
+ EXPECT_EQ(exp_docid + 1, attr.getNumDocs());
+}
+
class ExtendAttributeTest : public ::testing::Test
{
+ std::vector<std::unique_ptr<Value>> _tensors;
protected:
ExtendAttributeTest() = default;
~ExtendAttributeTest() override = default;
@@ -27,16 +81,22 @@ protected:
template <typename Attribute>
void testExtendString(Attribute & attr);
void testExtendRaw(AttributeVector& attr);
+ void testExtendTensor(AttributeVector& attr);
+ const Value& create_tensor(const TensorSpec &spec);
};
+const Value&
+ExtendAttributeTest::create_tensor(const TensorSpec &spec)
+{
+ auto value = value_from_spec(spec, FastValueBuilderFactory::get());
+ _tensors.emplace_back(std::move(value));
+ return *_tensors.back();
+}
+
template <typename Attribute>
void ExtendAttributeTest::testExtendInteger(Attribute & attr)
{
- uint32_t docId(0);
- EXPECT_EQ(attr.getNumDocs(), 0u);
- attr.addDoc(docId);
- EXPECT_EQ(docId, 0u);
- EXPECT_EQ(attr.getNumDocs(), 1u);
+ add_doc(attr, 0);
attr.add(1, 10);
EXPECT_EQ(attr.getInt(0), 1);
attr.add(2, 20);
@@ -51,9 +111,7 @@ void ExtendAttributeTest::testExtendInteger(Attribute & attr)
EXPECT_EQ(v[1].getWeight(), 20);
}
}
- attr.addDoc(docId);
- EXPECT_EQ(docId, 1u);
- EXPECT_EQ(attr.getNumDocs(), 2u);
+ add_doc(attr, 1);
attr.add(3, 30);
EXPECT_EQ(attr.getInt(1), 3);
if (attr.hasMultiValue()) {
@@ -69,11 +127,7 @@ void ExtendAttributeTest::testExtendInteger(Attribute & attr)
template <typename Attribute>
void ExtendAttributeTest::testExtendFloat(Attribute & attr)
{
- uint32_t docId(0);
- EXPECT_EQ(attr.getNumDocs(), 0u);
- attr.addDoc(docId);
- EXPECT_EQ(docId, 0u);
- EXPECT_EQ(attr.getNumDocs(), 1u);
+ add_doc(attr, 0);
attr.add(1.7, 10);
EXPECT_EQ(attr.getInt(0), 1);
EXPECT_EQ(attr.getFloat(0), 1.7);
@@ -89,9 +143,7 @@ void ExtendAttributeTest::testExtendFloat(Attribute & attr)
EXPECT_EQ(v[1].getWeight(), 20);
}
}
- attr.addDoc(docId);
- EXPECT_EQ(docId, 1u);
- EXPECT_EQ(attr.getNumDocs(), 2u);
+ add_doc(attr, 1);
attr.add(3.6, 30);
EXPECT_EQ(attr.getFloat(1), 3.6);
if (attr.hasMultiValue()) {
@@ -107,11 +159,7 @@ void ExtendAttributeTest::testExtendFloat(Attribute & attr)
template <typename Attribute>
void ExtendAttributeTest::testExtendString(Attribute & attr)
{
- uint32_t docId(0);
- EXPECT_EQ(attr.getNumDocs(), 0u);
- attr.addDoc(docId);
- EXPECT_EQ(docId, 0u);
- EXPECT_EQ(attr.getNumDocs(), 1u);
+ add_doc(attr, 0);
attr.add("1.7", 10);
auto buf = attr.get_raw(0);
EXPECT_EQ(std::string(buf.data(), buf.size()), "1.7");
@@ -128,9 +176,7 @@ void ExtendAttributeTest::testExtendString(Attribute & attr)
EXPECT_EQ(v[1].getWeight(), 20);
}
}
- attr.addDoc(docId);
- EXPECT_EQ(docId, 1u);
- EXPECT_EQ(attr.getNumDocs(), 2u);
+ add_doc(attr, 1);
attr.add("3.6", 30);
buf = attr.get_raw(1);
EXPECT_EQ(std::string(buf.data(), buf.size()), "3.6");
@@ -150,41 +196,77 @@ void ExtendAttributeTest::testExtendRaw(AttributeVector& attr)
std::vector<char> zeros{10, 0, 0, 11};
auto* ext_attr = attr.getExtendInterface();
EXPECT_NE(nullptr, ext_attr);
- uint32_t docId(0);
- EXPECT_EQ(0u, attr.getNumDocs());
- attr.addDoc(docId);
- EXPECT_EQ(0u, docId);
- EXPECT_EQ(1u, attr.getNumDocs());
+ add_doc(attr, 0);
ext_attr->add(as_vector("1.7"));
auto buf = attr.get_raw(0);
EXPECT_EQ(as_vector("1.7"), as_vector(buf));
ext_attr->add(vespalib::ConstArrayRef<char>(as_vector("2.3")));
buf = attr.get_raw(0);
EXPECT_EQ(as_vector("2.3"), as_vector(buf));
- attr.addDoc(docId);
- EXPECT_EQ(1u, docId);
- EXPECT_EQ(attr.getNumDocs(), 2u);
+ add_doc(attr, 1);
ext_attr->add(as_vector("3.6"));
buf = attr.get_raw(1);
EXPECT_EQ(as_vector("3.6"), as_vector(buf));
buf = attr.get_raw(0);
EXPECT_EQ(as_vector("2.3"), as_vector(buf));
- attr.addDoc(docId);
- EXPECT_EQ(2u, docId);
+ add_doc(attr, 2);
ext_attr->add(zeros);
buf = attr.get_raw(2);
EXPECT_EQ(zeros, as_vector(buf));
- attr.addDoc(docId);
- EXPECT_EQ(3u, docId);
+ add_doc(attr, 3);
buf = attr.get_raw(3);
EXPECT_EQ(empty, as_vector(buf));
- attr.addDoc(docId);
- EXPECT_EQ(4u, docId);
+ add_doc(attr, 4);
ext_attr->add(empty);
buf = attr.get_raw(4);
EXPECT_EQ(empty, as_vector(buf));
}
+void ExtendAttributeTest::testExtendTensor(AttributeVector& attr)
+{
+ std::vector<double> empty_cells{0.0, 0.0};
+ std::vector<double> spec0_dense_cells{1.0, 2.0};
+ std::vector<double> spec0_mixed_cells0{3.0, 4.0};
+ std::vector<double> spec0_mixed_cells1{5.0, 6.0};
+ bool dense = attr.getConfig().tensorType().is_dense();
+ auto* ext_attr = attr.getExtendInterface();
+ EXPECT_NE(nullptr, ext_attr);
+ auto* tensor_attr = attr.asTensorAttribute();
+ EXPECT_NE(nullptr, tensor_attr);
+ add_doc(attr, 0);
+ TensorSpec spec0 = dense ? vec_2d(1.0, 2.0) : vec_mixed_2d({{3.0, 4.0}, {5.0, 6.0}});
+ EXPECT_TRUE(ext_attr->add(create_tensor(spec0)));
+ auto tensor = tensor_attr->getTensor(0);
+ EXPECT_NE(nullptr, tensor.get());
+ EXPECT_EQ(spec0, TensorSpec::from_value(*tensor));
+ EXPECT_EQ(dense, tensor_attr->supports_extract_cells_ref());
+ if (dense) {
+ EXPECT_EQ(spec0_dense_cells, as_vector(tensor_attr->extract_cells_ref(0).typify<double>()));
+ }
+ EXPECT_TRUE(tensor_attr->supports_get_tensor_ref());
+ EXPECT_EQ(spec0, TensorSpec::from_value(tensor_attr->get_tensor_ref(0)));
+ EXPECT_FALSE(tensor_attr->supports_get_serialized_tensor_ref());
+ auto vectors = tensor_attr->get_vectors(0);
+ if (dense) {
+ EXPECT_EQ(1, vectors.subspaces());
+ EXPECT_EQ(spec0_dense_cells, as_vector(vectors.cells(0).typify<double>()));
+ EXPECT_EQ(spec0_dense_cells, as_vector(tensor_attr->get_vector(0, 0).typify<double>()));
+ EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(0, 1).typify<double>()));
+ } else {
+ EXPECT_EQ(2, vectors.subspaces());
+ EXPECT_EQ(spec0_mixed_cells0, as_vector(vectors.cells(0).typify<double>()));
+ EXPECT_EQ(spec0_mixed_cells1, as_vector(vectors.cells(1).typify<double>()));
+ EXPECT_EQ(spec0_mixed_cells0, as_vector(tensor_attr->get_vector(0, 0).typify<double>()));
+ EXPECT_EQ(spec0_mixed_cells1, as_vector(tensor_attr->get_vector(0, 1).typify<double>()));
+ EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(0, 2).typify<double>()));
+ }
+ add_doc(attr, 1);
+ vectors = tensor_attr->get_vectors(1);
+ EXPECT_EQ(0, vectors.subspaces());
+ EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(1, 0).typify<double>()));
+ EXPECT_EQ(nullptr, tensor_attr->getTensor(1).get());
+}
+
TEST_F(ExtendAttributeTest, single_integer_ext_attribute)
{
SingleIntegerExtAttribute siattr("si1");
@@ -255,6 +337,24 @@ TEST_F(ExtendAttributeTest, single_raw_ext_attribute)
testExtendRaw(srattr);
}
+TEST_F(ExtendAttributeTest, tensor_ext_attribute_dense)
+{
+ Config cfg(BasicType::TENSOR, CollectionType::SINGLE);
+ cfg.setTensorType(ValueType::from_spec(vec_2d_spec));
+ TensorExtAttribute tattr("td1", cfg);
+ EXPECT_TRUE(! tattr.hasMultiValue());
+ testExtendTensor(tattr);
+}
+
+TEST_F(ExtendAttributeTest, tensor_ext_attribute_mixed)
+{
+ Config cfg(BasicType::TENSOR, CollectionType::SINGLE);
+ cfg.setTensorType(ValueType::from_spec(vec_mixed_2d_spec));
+ TensorExtAttribute tattr("tm1", cfg);
+ EXPECT_TRUE(! tattr.hasMultiValue());
+ testExtendTensor(tattr);
+}
+
}
GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp
index df2a5ea2ac9..17e852c8b92 100644
--- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp
+++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp
@@ -22,11 +22,11 @@ using vespalib::datastore::CompactionStrategy;
using vespalib::alloc::test::MemoryAllocatorObserver;
using AllocStats = MemoryAllocatorObserver::Stats;
-template <typename EntryT>
+template <typename ElemT>
void
-assertArray(const std::vector<EntryT> &exp, vespalib::ConstArrayRef<EntryT> values)
+assertArray(const std::vector<ElemT> &exp, vespalib::ConstArrayRef<ElemT> values)
{
- EXPECT_EQ(exp, std::vector<EntryT>(values.cbegin(), values.cend()));
+ EXPECT_EQ(exp, std::vector<ElemT>(values.cbegin(), values.cend()));
}
template <class MvMapping>
@@ -69,10 +69,10 @@ public:
constexpr float ALLOC_GROW_FACTOR = 0.2;
-template <typename EntryT>
+template <typename ElemT>
class MappingTestBase : public ::testing::Test {
protected:
- using MvMapping = search::attribute::MultiValueMapping<EntryT>;
+ using MvMapping = search::attribute::MultiValueMapping<ElemT>;
using AttributeType = MyAttribute<MvMapping>;
AllocStats _stats;
std::unique_ptr<MvMapping> _mvMapping;
@@ -82,8 +82,8 @@ protected:
using generation_t = vespalib::GenerationHandler::generation_t;
public:
- using ArrayRef = vespalib::ArrayRef<EntryT>;
- using ConstArrayRef = vespalib::ConstArrayRef<EntryT>;
+ using ArrayRef = vespalib::ArrayRef<ElemT>;
+ using ConstArrayRef = vespalib::ConstArrayRef<ElemT>;
MappingTestBase()
: _stats(),
_mvMapping(),
@@ -99,9 +99,9 @@ public:
_attr = std::make_unique<AttributeType>(*_mvMapping);
_maxSmallArraySize = maxSmallArraySize;
}
- void setup(uint32_t maxSmallArraySize, size_t minArrays, size_t maxArrays, size_t numArraysForNewBuffer, bool enable_free_lists = true) {
+ void setup(uint32_t maxSmallArraySize, size_t min_entries, size_t max_entries, size_t num_entries_for_new_buffer, bool enable_free_lists = true) {
ArrayStoreConfig config(maxSmallArraySize,
- ArrayStoreConfig::AllocSpec(minArrays, maxArrays, numArraysForNewBuffer, ALLOC_GROW_FACTOR));
+ ArrayStoreConfig::AllocSpec(min_entries, max_entries, num_entries_for_new_buffer, ALLOC_GROW_FACTOR));
config.enable_free_lists(enable_free_lists);
_mvMapping = std::make_unique<MvMapping>(config, vespalib::GrowStrategy(), std::make_unique<MemoryAllocatorObserver>(_stats));
_attr = std::make_unique<AttributeType>(*_mvMapping);
@@ -109,12 +109,12 @@ public:
}
~MappingTestBase() { }
- void set(uint32_t docId, const std::vector<EntryT> &values) { _mvMapping->set(docId, values); }
+ void set(uint32_t docId, const std::vector<ElemT> &values) { _mvMapping->set(docId, values); }
ConstArrayRef get(uint32_t docId) { return _mvMapping->get(docId); }
ArrayRef get_writable(uint32_t docId) { return _mvMapping->get_writable(docId); }
- void assertGet(uint32_t docId, const std::vector<EntryT> &exp) {
+ void assertGet(uint32_t docId, const std::vector<ElemT> &exp) {
ConstArrayRef act = get(docId);
- EXPECT_EQ(exp, std::vector<EntryT>(act.cbegin(), act.cend()));
+ EXPECT_EQ(exp, std::vector<ElemT>(act.cbegin(), act.cend()));
}
void assign_generation(generation_t current_gen) { _mvMapping->assign_generation(current_gen); }
void reclaim_memory(generation_t oldest_used_gen) { _mvMapping->reclaim_memory(oldest_used_gen); }
diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
index b57a2f42ea7..69478c09a25 100644
--- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
+++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
@@ -1032,14 +1032,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
auto beforeStats = getFeatureStoreMemStats(_fic);
LOG(info,
- "Before feature compaction: allocElems=%zu, usedElems=%zu"
- ", deadElems=%zu, holdElems=%zu"
+ "Before feature compaction: alloc_entries=%zu, used_entries=%zu"
+ ", dead_entries=%zu, hold_entries=%zu"
", freeBuffers=%u, activeBuffers=%u"
", holdBuffers=%u",
- beforeStats._allocElems,
- beforeStats._usedElems,
- beforeStats._deadElems,
- beforeStats._holdElems,
+ beforeStats._alloc_entries,
+ beforeStats._used_entries,
+ beforeStats._dead_entries,
+ beforeStats._hold_entries,
beforeStats._freeBuffers,
beforeStats._activeBuffers,
beforeStats._holdBuffers);
@@ -1052,14 +1052,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
myCommit(_fic, *_pushThreads);
auto duringStats = getFeatureStoreMemStats(_fic);
LOG(info,
- "During feature compaction: allocElems=%zu, usedElems=%zu"
- ", deadElems=%zu, holdElems=%zu"
+ "During feature compaction: alloc_entries=%zu, used_entries=%zu"
+ ", dead_entries=%zu, hold_entries=%zu"
", freeBuffers=%u, activeBuffers=%u"
", holdBuffers=%u",
- duringStats._allocElems,
- duringStats._usedElems,
- duringStats._deadElems,
- duringStats._holdElems,
+ duringStats._alloc_entries,
+ duringStats._used_entries,
+ duringStats._dead_entries,
+ duringStats._hold_entries,
duringStats._freeBuffers,
duringStats._activeBuffers,
duringStats._holdBuffers);
@@ -1067,14 +1067,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
myCommit(_fic, *_pushThreads);
auto afterStats = getFeatureStoreMemStats(_fic);
LOG(info,
- "After feature compaction: allocElems=%zu, usedElems=%zu"
- ", deadElems=%zu, holdElems=%zu"
+ "After feature compaction: alloc_entries=%zu, used_entries=%zu"
+ ", dead_entries=%zu, hold_entries=%zu"
", freeBuffers=%u, activeBuffers=%u"
", holdBuffers=%u",
- afterStats._allocElems,
- afterStats._usedElems,
- afterStats._deadElems,
- afterStats._holdElems,
+ afterStats._alloc_entries,
+ afterStats._used_entries,
+ afterStats._dead_entries,
+ afterStats._hold_entries,
afterStats._freeBuffers,
afterStats._activeBuffers,
afterStats._holdBuffers);
diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
index 8073fb8d232..e3b9cf9702d 100644
--- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
+++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
@@ -462,8 +462,8 @@ TEST(MemoryIndexTest, require_that_num_docs_and_doc_id_limit_is_returned)
TEST(MemoryIndexTest, require_that_we_understand_the_memory_footprint)
{
- constexpr size_t BASE_ALLOCATED = 361032u;
- constexpr size_t BASE_USED = 151188u;
+ constexpr size_t BASE_ALLOCATED = 360936u;
+ constexpr size_t BASE_USED = 150804u;
{
MySetup setup;
Index index(setup);
diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp
index 4ac4bdc32f0..d30df9dba6e 100644
--- a/searchlib/src/tests/predicate/document_features_store_test.cpp
+++ b/searchlib/src/tests/predicate/document_features_store_test.cpp
@@ -165,17 +165,17 @@ TEST("require that both features and ranges are removed by 'remove'") {
TEST("require that both features and ranges counts towards memory usage") {
DocumentFeaturesStore features_store(10);
- EXPECT_EQUAL(50136u, features_store.getMemoryUsage().usedBytes());
+ EXPECT_EQUAL(50064u, features_store.getMemoryUsage().usedBytes());
PredicateTreeAnnotations annotations;
annotations.features.push_back(PredicateHash::hash64("foo=100-199"));
features_store.insert(annotations, doc_id);
- EXPECT_EQUAL(50144u, features_store.getMemoryUsage().usedBytes());
+ EXPECT_EQUAL(50072u, features_store.getMemoryUsage().usedBytes());
annotations.features.clear();
annotations.range_features.push_back({"foo", 100, 199});
features_store.insert(annotations, doc_id + 1);
- EXPECT_EQUAL(50240u, features_store.getMemoryUsage().usedBytes());
+ EXPECT_EQUAL(50168u, features_store.getMemoryUsage().usedBytes());
}
TEST("require that DocumentFeaturesStore can be serialized") {
@@ -205,17 +205,17 @@ TEST("require that serialization cleans up wordstore") {
PredicateTreeAnnotations annotations;
annotations.range_features.push_back({"foo", 100, 199});
features_store.insert(annotations, doc_id);
- EXPECT_EQUAL(50232u, features_store.getMemoryUsage().usedBytes());
+ EXPECT_EQUAL(50160u, features_store.getMemoryUsage().usedBytes());
annotations.range_features.push_back({"bar", 100, 199});
features_store.insert(annotations, doc_id + 1);
- EXPECT_EQUAL(50620u, features_store.getMemoryUsage().usedBytes());
+ EXPECT_EQUAL(50548u, features_store.getMemoryUsage().usedBytes());
features_store.remove(doc_id + 1);
- EXPECT_EQUAL(50572u, features_store.getMemoryUsage().usedBytes());
+ EXPECT_EQUAL(50500u, features_store.getMemoryUsage().usedBytes());
vespalib::DataBuffer buffer;
features_store.serialize(buffer);
DocumentFeaturesStore features_store2(buffer);
- EXPECT_EQUAL(50232u, features_store2.getMemoryUsage().usedBytes());
+ EXPECT_EQUAL(50160u, features_store2.getMemoryUsage().usedBytes());
}
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h
index 3d14622ca02..e40785911ea 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h
@@ -38,6 +38,8 @@ namespace vespalib::alloc {
class Alloc;
}
+namespace vespalib::eval { struct Value; }
+
namespace search {
template <typename T> class ComponentGuard;
@@ -86,6 +88,7 @@ public:
virtual bool add(double, int32_t = 1) { return false; }
virtual bool add(const char *, int32_t = 1) { return false; }
virtual bool add(vespalib::ConstArrayRef<char>, int32_t = 1) { return false; }
+ virtual bool add(const vespalib::eval::Value&, int32_t = 1) { return false; }
virtual ~IExtendAttribute() = default;
};
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h
index bd624e9f388..4fce64aa762 100644
--- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h
@@ -12,18 +12,18 @@ namespace search::attribute {
/**
* Class for mapping from document id to an array of values.
*/
-template <typename EntryT, typename RefT = vespalib::datastore::EntryRefT<19> >
+template <typename ElemT, typename RefT = vespalib::datastore::EntryRefT<19> >
class MultiValueMapping : public MultiValueMappingBase
{
public:
- using MultiValueType = EntryT;
+ using MultiValueType = ElemT;
using RefType = RefT;
- using ReadView = MultiValueMappingReadView<EntryT, RefT>;
+ using ReadView = MultiValueMappingReadView<ElemT, RefT>;
private:
- using ArrayRef = vespalib::ArrayRef<EntryT>;
- using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>;
+ using ArrayRef = vespalib::ArrayRef<ElemT>;
+ using ArrayStore = vespalib::datastore::ArrayStore<ElemT, RefT>;
using generation_t = vespalib::GenerationHandler::generation_t;
- using ConstArrayRef = vespalib::ConstArrayRef<EntryT>;
+ using ConstArrayRef = vespalib::ConstArrayRef<ElemT>;
ArrayStore _store;
public:
@@ -73,7 +73,7 @@ public:
static vespalib::datastore::ArrayStoreConfig optimizedConfigForHugePage(size_t maxSmallArraySize,
size_t hugePageSize,
size_t smallPageSize,
- size_t minNumArraysForNewBuffer,
+ size_t min_num_entries_for_new_buffer,
float allocGrowFactor,
bool enable_free_lists);
};
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
index b486fa60265..ab68bea58cc 100644
--- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
@@ -7,8 +7,8 @@
namespace search::attribute {
-template <typename EntryT, typename RefT>
-MultiValueMapping<EntryT,RefT>::MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg,
+template <typename ElemT, typename RefT>
+MultiValueMapping<ElemT,RefT>::MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg,
const vespalib::GrowStrategy &gs,
std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator)
: MultiValueMappingBase(gs, ArrayStore::getGenerationHolderLocation(_store), memory_allocator),
@@ -16,12 +16,12 @@ MultiValueMapping<EntryT,RefT>::MultiValueMapping(const vespalib::datastore::Arr
{
}
-template <typename EntryT, typename RefT>
-MultiValueMapping<EntryT,RefT>::~MultiValueMapping() = default;
+template <typename ElemT, typename RefT>
+MultiValueMapping<ElemT,RefT>::~MultiValueMapping() = default;
-template <typename EntryT, typename RefT>
+template <typename ElemT, typename RefT>
void
-MultiValueMapping<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values)
+MultiValueMapping<ElemT,RefT>::set(uint32_t docId, ConstArrayRef values)
{
_indices.ensure_size(docId + 1);
EntryRef oldRef(_indices[docId].load_relaxed());
@@ -31,18 +31,18 @@ MultiValueMapping<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values)
_store.remove(oldRef);
}
-template <typename EntryT, typename RefT>
+template <typename ElemT, typename RefT>
vespalib::MemoryUsage
-MultiValueMapping<EntryT,RefT>::update_stat(const CompactionStrategy& compaction_strategy)
+MultiValueMapping<ElemT,RefT>::update_stat(const CompactionStrategy& compaction_strategy)
{
auto retval = _store.update_stat(compaction_strategy);
retval.merge(_indices.getMemoryUsage());
return retval;
}
-template <typename EntryT, typename RefT>
+template <typename ElemT, typename RefT>
void
-MultiValueMapping<EntryT,RefT>::compact_worst(const CompactionStrategy& compaction_strategy)
+MultiValueMapping<ElemT,RefT>::compact_worst(const CompactionStrategy& compaction_strategy)
{
vespalib::datastore::ICompactionContext::UP compactionContext(_store.compact_worst(compaction_strategy));
if (compactionContext) {
@@ -50,29 +50,29 @@ MultiValueMapping<EntryT,RefT>::compact_worst(const CompactionStrategy& compacti
}
}
-template <typename EntryT, typename RefT>
+template <typename ElemT, typename RefT>
vespalib::MemoryUsage
-MultiValueMapping<EntryT,RefT>::getArrayStoreMemoryUsage() const
+MultiValueMapping<ElemT,RefT>::getArrayStoreMemoryUsage() const
{
return _store.getMemoryUsage();
}
-template <typename EntryT, typename RefT>
+template <typename ElemT, typename RefT>
vespalib::AddressSpace
-MultiValueMapping<EntryT, RefT>::getAddressSpaceUsage() const {
+MultiValueMapping<ElemT, RefT>::getAddressSpaceUsage() const {
return _store.addressSpaceUsage();
}
-template <typename EntryT, typename RefT>
+template <typename ElemT, typename RefT>
vespalib::datastore::ArrayStoreConfig
-MultiValueMapping<EntryT, RefT>::optimizedConfigForHugePage(size_t maxSmallArraySize,
+MultiValueMapping<ElemT, RefT>::optimizedConfigForHugePage(size_t maxSmallArraySize,
size_t hugePageSize,
size_t smallPageSize,
- size_t minNumArraysForNewBuffer,
+ size_t min_num_entries_for_new_buffer,
float allocGrowFactor,
bool enable_free_lists)
{
- auto result = ArrayStore::optimizedConfigForHugePage(maxSmallArraySize, hugePageSize, smallPageSize, minNumArraysForNewBuffer, allocGrowFactor);
+ auto result = ArrayStore::optimizedConfigForHugePage(maxSmallArraySize, hugePageSize, smallPageSize, min_num_entries_for_new_buffer, allocGrowFactor);
result.enable_free_lists(enable_free_lists);
return result;
}
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h
index 116e069e8b4..41138ff0890 100644
--- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h
@@ -11,12 +11,12 @@ namespace search::attribute {
/**
* Class for mapping from document id to an array of values as reader.
*/
-template <typename EntryT, typename RefT = vespalib::datastore::EntryRefT<19> >
+template <typename ElemT, typename RefT = vespalib::datastore::EntryRefT<19> >
class MultiValueMappingReadView
{
using AtomicEntryRef = vespalib::datastore::AtomicEntryRef;
using Indices = vespalib::ConstArrayRef<AtomicEntryRef>;
- using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>;
+ using ArrayStore = vespalib::datastore::ArrayStore<ElemT, RefT>;
Indices _indices;
const ArrayStore* _store;
@@ -31,7 +31,7 @@ public:
_store(store)
{
}
- vespalib::ConstArrayRef<EntryT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); }
+ vespalib::ConstArrayRef<ElemT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); }
bool valid() const noexcept { return _store != nullptr; }
};
diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp
index ea449300aef..1009fa2fb5f 100644
--- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp
@@ -51,7 +51,7 @@ template <typename B, typename M>
MultiValueNumericPostingAttribute<B, M>::~MultiValueNumericPostingAttribute()
{
this->disableFreeLists();
- this->disableElemHoldList();
+ this->disable_entry_hold_list();
clearAllPostings();
}
diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp
index cd46bbb5a8a..19840b5a474 100644
--- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp
@@ -27,7 +27,7 @@ template <typename B, typename T>
MultiValueStringPostingAttributeT<B, T>::~MultiValueStringPostingAttributeT()
{
this->disableFreeLists();
- this->disableElemHoldList();
+ this->disable_entry_hold_list();
clearAllPostings();
}
diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h
index 29440b6ce43..ecf7a46f21e 100644
--- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h
@@ -58,7 +58,7 @@ protected:
void updatePostings(PostingMap &changePost, const vespalib::datastore::EntryComparator &cmp);
void clearAllPostings();
void disableFreeLists() { _postingList.disableFreeLists(); }
- void disableElemHoldList() { _postingList.disableElemHoldList(); }
+ void disable_entry_hold_list() { _postingList.disable_entry_hold_list(); }
void handle_load_posting_lists_and_update_enum_store(enumstore::EnumeratedPostingsLoader& loader);
bool forwardedOnAddDoc(DocId doc, size_t wantSize, size_t wantCapacity);
diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
index 94720212faf..2703201b292 100644
--- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp
@@ -231,7 +231,7 @@ PostingStore<DataT>::dropBitVector(EntryRef &ref)
(void) tree;
(void) docFreq;
_bvs.erase(ref.ref());
- _store.holdElem(iRef, 1);
+ _store.hold_entry(iRef);
_status.decBitVectors();
_bvExtraBytes -= bv->writer().extraByteSize();
ref = ref2;
@@ -267,7 +267,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref)
if (_enableOnlyBitVector) {
BTreeType *tree = getWTreeEntry(iRef);
tree->clear(_allocator);
- _store.holdElem(ref, 1);
+ _store.hold_entry(ref);
} else {
bve->_tree = ref;
}
@@ -590,19 +590,19 @@ PostingStore<DataT>::clear(const EntryRef ref)
assert(isBTree(iRef2));
BTreeType *tree = getWTreeEntry(iRef2);
tree->clear(_allocator);
- _store.holdElem(iRef2, 1);
+ _store.hold_entry(iRef2);
}
_bvs.erase(ref.ref());
_status.decBitVectors();
_bvExtraBytes -= bve->_bv->writer().extraByteSize();
- _store.holdElem(ref, 1);
+ _store.hold_entry(ref);
} else {
BTreeType *tree = getWTreeEntry(iRef);
tree->clear(_allocator);
- _store.holdElem(ref, 1);
+ _store.hold_entry(ref);
}
} else {
- _store.holdElem(ref, clusterSize);
+ _store.hold_entry(ref);
}
}
diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp
index 1775774171d..de4a7157dae 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp
@@ -13,7 +13,7 @@ template <typename B>
SingleValueNumericPostingAttribute<B>::~SingleValueNumericPostingAttribute()
{
this->disableFreeLists();
- this->disableElemHoldList();
+ this->disable_entry_hold_list();
clearAllPostings();
}
diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp
index eef72984e79..1ec9b54a73b 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp
@@ -27,7 +27,7 @@ template <typename B>
SingleValueStringPostingAttributeT<B>::~SingleValueStringPostingAttributeT()
{
this->disableFreeLists();
- this->disableElemHoldList();
+ this->disable_entry_hold_list();
clearAllPostings();
}
diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp
index 72f4a7ae579..4bc7f5b1144 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp
@@ -36,7 +36,7 @@ EntryRef
FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen)
{
uint32_t pad = Aligner::pad(byteLen);
- auto result = _store.rawAllocator<uint8_t>(_typeId).alloc(byteLen + pad, DECODE_SAFETY);
+ auto result = _store.rawAllocator<uint8_t>(_typeId).alloc((byteLen + pad) / buffer_array_size, DECODE_SAFETY_ENTRIES);
uint8_t *dst = result.data;
memcpy(dst, src, byteLen);
dst += byteLen;
@@ -113,7 +113,7 @@ FeatureStore::add_features_guard_bytes()
{
uint32_t len = DECODE_SAFETY;
uint32_t pad = Aligner::pad(len);
- auto result = _store.rawAllocator<uint8_t>(_typeId).alloc(len + pad);
+ auto result = _store.rawAllocator<uint8_t>(_typeId).alloc((len + pad) / buffer_array_size);
memset(result.data, 0, len + pad);
}
diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h
index 53588fa2894..1e48189987e 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h
@@ -29,6 +29,7 @@ private:
using PosOccFieldsParams = bitcompression::PosOccFieldsParams;
static constexpr uint32_t DECODE_SAFETY = 16;
+ static constexpr uint32_t DECODE_SAFETY_ENTRIES = 16 / buffer_array_size;
DataStoreType _store;
@@ -117,7 +118,7 @@ public:
* overrun beyond the compressed data either goes into other features
* already written or into the guard area.
*
- * If buffer type is changed to have a nonzero numArraysForNewBuffer then
+ * If buffer type is changed to have a nonzero num_entries_for_new_buffer then
* extra logic to add guard bytes is needed when switching primary buffer
* to avoid issues if the buffer is resumed as primary buffer later on.
*/
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
index 4be3031303e..8dd76a90b14 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp
@@ -55,9 +55,9 @@ template <bool interleaved_features>
FieldIndex<interleaved_features>::~FieldIndex()
{
_postingListStore.disableFreeLists();
- _postingListStore.disableElemHoldList();
+ _postingListStore.disable_entry_hold_list();
_dict.disableFreeLists();
- _dict.disableElemHoldList();
+ _dict.disable_entry_hold_list();
// XXX: Kludge
for (DictionaryTree::Iterator it = _dict.begin();
it.valid(); ++it) {
diff --git a/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp
index 3e4c38ceb0e..e330dc83055 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp
@@ -27,7 +27,7 @@ WordStore::addWord(const vespalib::stringref word)
{
size_t wordSize = word.size() + 1;
size_t bufferSize = wordSize + Aligner::pad(wordSize);
- auto result = _store.rawAllocator<char>(_typeId).alloc(bufferSize);
+ auto result = _store.rawAllocator<char>(_typeId).alloc(bufferSize / buffer_array_size);
char *be = result.data;
for (size_t i = 0; i < word.size(); ++i) {
*be++ = word[i];
diff --git a/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp b/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp
index 604a467a6e6..a6a82ec09f8 100644
--- a/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp
+++ b/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp
@@ -102,7 +102,7 @@ DocumentFeaturesStore::DocumentFeaturesStore(DataBuffer &buffer)
DocumentFeaturesStore::~DocumentFeaturesStore() {
_word_index.disableFreeLists();
- _word_index.disableElemHoldList();
+ _word_index.disable_entry_hold_list();
_word_index.getAllocator().freeze();
_word_index.clear();
}
diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp
index af809b2fa69..af5aae6e519 100644
--- a/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp
+++ b/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp
@@ -95,7 +95,7 @@ PredicateIntervalStore::remove(EntryRef ref) {
// BufferState &state = _store.getBufferState(buffer_id);
// uint32_t type_id = state.getTypeId();
// uint32_t size = type_id <= MAX_ARRAY_SIZE ? type_id : 1;
- // _store.holdElem(ref, size);
+ // _store.hold_entries(ref, size);
}
}
diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp
index c6f640d72ed..9320488f88e 100644
--- a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp
+++ b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp
@@ -41,7 +41,7 @@ SimpleIndex<Posting, Key, DocId>::insertIntoVectorPosting(vespalib::datastore::E
template <typename Posting, typename Key, typename DocId>
SimpleIndex<Posting, Key, DocId>::~SimpleIndex() {
_btree_posting_lists.disableFreeLists();
- _btree_posting_lists.disableElemHoldList();
+ _btree_posting_lists.disable_entry_hold_list();
for (auto it = _dictionary.begin(); it.valid(); ++it) {
vespalib::datastore::EntryRef ref(it.getData());
@@ -51,13 +51,13 @@ SimpleIndex<Posting, Key, DocId>::~SimpleIndex() {
}
_vector_posting_lists.disableFreeLists();
- _vector_posting_lists.disableElemHoldList();
+ _vector_posting_lists.disable_entry_hold_list();
_vector_posting_lists.clear();
_vector_posting_lists.getAllocator().freeze();
_vector_posting_lists.getAllocator().reclaim_all_memory();
_dictionary.disableFreeLists();
- _dictionary.disableElemHoldList();
+ _dictionary.disable_entry_hold_list();
_dictionary.clear();
_dictionary.getAllocator().freeze();
_dictionary.getAllocator().reclaim_all_memory();
diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
index c8c5d4d4257..313863d8dcb 100644
--- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
@@ -40,6 +40,7 @@ vespa_add_library(searchlib_tensor OBJECT
tensor_buffer_store.cpp
tensor_buffer_type_mapper.cpp
tensor_deserialize.cpp
+ tensor_ext_attribute.cpp
tensor_store.cpp
DEPENDS
)
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp
index c373f6bdcd0..c51d0ec7fd3 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp
@@ -62,10 +62,10 @@ DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, s
DenseTensorStore::BufferType::~BufferType() = default;
void
-DenseTensorStore::BufferType::cleanHold(void *buffer, size_t offset,
- ElemCount numElems, CleanContext)
+DenseTensorStore::BufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext)
{
- memset(static_cast<char *>(buffer) + offset, 0, numElems);
+ auto num_elems = num_entries * getArraySize();
+ memset(static_cast<char *>(buffer) + offset * getArraySize(), 0, num_elems);
}
const vespalib::alloc::MemoryAllocator*
@@ -107,7 +107,7 @@ DenseTensorStore::allocRawBuffer()
{
size_t bufSize = getBufSize();
size_t alignedBufSize = _tensorSizeCalc.alignedSize();
- auto result = _concreteStore.freeListRawAllocator<char>(0u).alloc(alignedBufSize);
+ auto result = _concreteStore.freeListRawAllocator<char>(0u).alloc(1);
clearPadAreaAfterBuffer(result.data, bufSize, alignedBufSize);
return result;
}
@@ -118,7 +118,7 @@ DenseTensorStore::holdTensor(EntryRef ref)
if (!ref.valid()) {
return;
}
- _concreteStore.holdElem(ref, _tensorSizeCalc.alignedSize());
+ _concreteStore.hold_entry(ref);
}
TensorStore::EntryRef
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h
index 9e326e0ab1e..0dd483e7f08 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h
@@ -44,7 +44,7 @@ public:
public:
BufferType(const TensorSizeCalc &tensorSizeCalc, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator);
~BufferType() override;
- void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override;
+ void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override;
const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override;
};
private:
diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp
index fa13ab6303c..8526138fd31 100644
--- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp
@@ -30,11 +30,11 @@ DirectTensorStore::TensorBufferType::TensorBufferType()
}
void
-DirectTensorStore::TensorBufferType::cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx)
+DirectTensorStore::TensorBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext clean_ctx)
{
TensorSP* elem = static_cast<TensorSP*>(buffer) + offset;
const auto& empty = empty_entry();
- for (size_t i = 0; i < num_elems; ++i) {
+ for (size_t i = 0; i < num_entries; ++i) {
clean_ctx.extraBytesCleaned((*elem)->get_memory_usage().allocatedBytes());
*elem = empty;
++elem;
@@ -69,7 +69,7 @@ DirectTensorStore::holdTensor(EntryRef ref)
}
const auto& tensor = _tensor_store.getEntry(ref);
assert(tensor);
- _tensor_store.holdElem(ref, 1, tensor->get_memory_usage().allocatedBytes());
+ _tensor_store.hold_entry(ref, tensor->get_memory_usage().allocatedBytes());
}
EntryRef
diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h
index 01084e89776..1230494fe41 100644
--- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h
+++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h
@@ -20,7 +20,7 @@ namespace search::tensor {
*/
class DirectTensorStore : public TensorStore {
private:
- // Note: Must use SP (instead of UP) because of fallbackCopy() and initializeReservedElements() in BufferType,
+ // Note: Must use SP (instead of UP) because of fallback_copy() and initialize_reserved_entries() in BufferType,
// and implementation of move().
using TensorSP = std::shared_ptr<vespalib::eval::Value>;
using TensorStoreType = vespalib::datastore::DataStore<TensorSP>;
@@ -32,7 +32,7 @@ private:
using CleanContext = typename ParentType::CleanContext;
public:
TensorBufferType();
- void cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) override;
+ void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext clean_ctx) override;
};
TensorStoreType _tensor_store;
diff --git a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp
index cb074348f08..5d3b2206703 100644
--- a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp
@@ -12,7 +12,7 @@ using vespalib::alloc::MemoryAllocator;
namespace search::tensor {
LargeSubspacesBufferType::LargeSubspacesBufferType(const AllocSpec& spec, std::shared_ptr<MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept
- : ParentType(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor),
+ : ParentType(1u, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor),
_memory_allocator(std::move(memory_allocator)),
_ops(type_mapper.get_tensor_buffer_operations())
{
@@ -21,10 +21,10 @@ LargeSubspacesBufferType::LargeSubspacesBufferType(const AllocSpec& spec, std::s
LargeSubspacesBufferType::~LargeSubspacesBufferType() = default;
void
-LargeSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx)
+LargeSubspacesBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx)
{
auto elem = static_cast<ArrayType*>(buffer) + offset;
- for (size_t i = 0; i < numElems; ++i) {
+ for (size_t i = 0; i < num_entries; ++i) {
if (!elem->empty()) {
cleanCtx.extraBytesCleaned(elem->size());
_ops.reclaim_labels({elem->data(), elem->size()});
@@ -35,10 +35,10 @@ LargeSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numEl
}
void
-LargeSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems)
+LargeSubspacesBufferType::destroy_entries(void *buffer, EntryCount num_entries)
{
auto elem = static_cast<ArrayType*>(buffer);
- for (size_t i = 0; i < numElems; ++i) {
+ for (size_t i = 0; i < num_entries; ++i) {
if (!elem->empty()) {
_ops.reclaim_labels({elem->data(), elem->size()});
ArrayType().swap(*elem);
@@ -48,11 +48,11 @@ LargeSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems)
}
void
-LargeSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems)
+LargeSubspacesBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries)
{
auto old_elems = static_cast<const ArrayType*>(oldBuffer);
auto new_elems = static_cast<ArrayType*>(newBuffer);
- for (size_t i = 0; i < numElems; ++i) {
+ for (size_t i = 0; i < num_entries; ++i) {
auto& old_elem = old_elems[i];
new (new_elems + i) ArrayType(old_elem);
if (!old_elem.empty()) {
@@ -62,12 +62,12 @@ LargeSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, E
}
void
-LargeSubspacesBufferType::initializeReservedElements(void *buffer, ElemCount reservedElements)
+LargeSubspacesBufferType::initialize_reserved_entries(void *buffer, EntryCount reserved_entries)
{
- auto new_elems = static_cast<ArrayType*>(buffer);
+ auto new_entries = static_cast<ArrayType*>(buffer);
const auto& empty = empty_entry();
- for (size_t i = 0; i < reservedElements; ++i) {
- new (new_elems + i) ArrayType(empty);
+ for (size_t i = 0; i < reserved_entries; ++i) {
+ new (new_entries + i) ArrayType(empty);
}
}
diff --git a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h
index cfab8ef20af..8cce08e9d81 100644
--- a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h
+++ b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h
@@ -30,10 +30,10 @@ class LargeSubspacesBufferType : public vespalib::datastore::BufferType<vespalib
public:
LargeSubspacesBufferType(const AllocSpec& spec, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept;
~LargeSubspacesBufferType() override;
- void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override;
- void destroyElements(void *buffer, ElemCount numElems) override;
- void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override;
- void initializeReservedElements(void *buffer, ElemCount reservedElements) override;
+ void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override;
+ void destroy_entries(void *buffer, EntryCount num_entries) override;
+ void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override;
+ void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override;
const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override;
};
diff --git a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp
index 6a71388a3b9..7b54182f062 100644
--- a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp
@@ -10,7 +10,7 @@ using vespalib::alloc::MemoryAllocator;
namespace search::tensor {
SmallSubspacesBufferType::SmallSubspacesBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept
- : ParentType(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor),
+ : ParentType(array_size, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor),
_memory_allocator(std::move(memory_allocator)),
_ops(type_mapper.get_tensor_buffer_operations())
{
@@ -19,45 +19,45 @@ SmallSubspacesBufferType::SmallSubspacesBufferType(uint32_t array_size, const Al
SmallSubspacesBufferType::~SmallSubspacesBufferType() = default;
void
-SmallSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext)
+SmallSubspacesBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext)
{
- char* elem = static_cast<char *>(buffer) + offset;
- while (numElems >= getArraySize()) {
+ char* elem = static_cast<char *>(buffer) + offset * getArraySize();
+ while (num_entries >= 1) {
_ops.reclaim_labels(vespalib::ArrayRef<char>(elem, getArraySize()));
elem += getArraySize();
- numElems -= getArraySize();
+ --num_entries;
}
}
void
-SmallSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems)
+SmallSubspacesBufferType::destroy_entries(void *buffer, EntryCount num_entries)
{
char* elem = static_cast<char *>(buffer);
- while (numElems >= getArraySize()) {
+ while (num_entries >= 1) {
_ops.reclaim_labels(vespalib::ArrayRef<char>(elem, getArraySize()));
elem += getArraySize();
- numElems -= getArraySize();
+ --num_entries;
}
}
void
-SmallSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems)
+SmallSubspacesBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries)
{
- if (numElems > 0) {
- memcpy(newBuffer, oldBuffer, numElems);
+ if (num_entries > 0) {
+ memcpy(newBuffer, oldBuffer, num_entries * getArraySize());
}
const char *elem = static_cast<const char *>(oldBuffer);
- while (numElems >= getArraySize()) {
+ while (num_entries >= 1) {
_ops.copied_labels(unconstify(vespalib::ConstArrayRef<char>(elem, getArraySize())));
elem += getArraySize();
- numElems -= getArraySize();
+ --num_entries;
}
}
void
-SmallSubspacesBufferType::initializeReservedElements(void *buffer, ElemCount reservedElements)
+SmallSubspacesBufferType::initialize_reserved_entries(void *buffer, EntryCount reserved_entries)
{
- memset(buffer, 0, reservedElements);
+ memset(buffer, 0, reserved_entries * getArraySize());
}
const vespalib::alloc::MemoryAllocator*
diff --git a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h
index 5622e9970b8..2f287ef1f3d 100644
--- a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h
+++ b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h
@@ -30,10 +30,10 @@ public:
SmallSubspacesBufferType& operator=(SmallSubspacesBufferType&&) noexcept = delete;
SmallSubspacesBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept;
~SmallSubspacesBufferType() override;
- void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override;
- void destroyElements(void *buffer, ElemCount numElems) override;
- void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override;
- void initializeReservedElements(void *buffer, ElemCount reservedElements) override;
+ void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override;
+ void destroy_entries(void *buffer, EntryCount num_entries) override;
+ void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override;
+ void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override;
const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override;
};
diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp
new file mode 100644
index 00000000000..19c8cf6053b
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp
@@ -0,0 +1,181 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "tensor_ext_attribute.h"
+#include "serialized_tensor_ref.h"
+#include "vector_bundle.h"
+#include <vespa/eval/eval/fast_value.h>
+#include <vespa/eval/eval/tensor_spec.h>
+#include <vespa/eval/eval/value.h>
+#include <vespa/eval/eval/value_codec.h>
+#include <vespa/searchcommon/attribute/config.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".searchlib.tensor.tensor_ext_attribute");
+
+using vespalib::eval::FastValueBuilderFactory;
+using vespalib::eval::TensorSpec;
+using vespalib::eval::TypedCells;
+using vespalib::eval::Value;
+using vespalib::eval::ValueType;
+
+namespace search::tensor {
+
+namespace {
+
+std::unique_ptr<Value>
+create_empty_tensor(const ValueType& type)
+{
+ const auto &factory = FastValueBuilderFactory::get();
+ TensorSpec empty_spec(type.to_spec());
+ return vespalib::eval::value_from_spec(empty_spec, factory);
+}
+
+}
+
+TensorExtAttribute::TensorExtAttribute(const vespalib::string& name, const Config& cfg)
+ : NotImplementedAttribute(name, cfg),
+ ITensorAttribute(),
+ IExtendAttribute(),
+ _subspace_type(cfg.tensorType()),
+ _empty(_subspace_type),
+ _empty_tensor(create_empty_tensor(cfg.tensorType()))
+{
+}
+
+TensorExtAttribute::~TensorExtAttribute() = default;
+
+const ITensorAttribute*
+TensorExtAttribute::asTensorAttribute() const
+{
+ return this;
+}
+
+void
+TensorExtAttribute::onCommit()
+{
+ LOG_ABORT("should not be reached");
+}
+
+void
+TensorExtAttribute::onUpdateStat()
+{
+}
+
+bool
+TensorExtAttribute::addDoc(DocId& docId)
+{
+ docId = _data.size();
+ _data.emplace_back(nullptr);
+ incNumDocs();
+ setCommittedDocIdLimit(getNumDocs());
+ return true;
+}
+
+bool
+TensorExtAttribute::add(const vespalib::eval::Value& v, int32_t)
+{
+ _data.back() = &v;
+ return true;
+}
+
+IExtendAttribute*
+TensorExtAttribute::getExtendInterface()
+{
+ return this;
+}
+
+TypedCells
+TensorExtAttribute::get_vector(uint32_t docid, uint32_t subspace) const
+{
+ auto vectors = get_vectors(docid);
+ return (subspace < vectors.subspaces()) ? vectors.cells(subspace) : _empty.cells();
+}
+
+VectorBundle
+TensorExtAttribute::get_vectors(uint32_t docid) const
+{
+ auto tensor = _data[docid];
+ if (tensor == nullptr) {
+ return VectorBundle();
+ }
+ return VectorBundle(tensor->cells().data, tensor->index().size(), _subspace_type);
+}
+
+std::unique_ptr<Value>
+TensorExtAttribute::getTensor(uint32_t docid) const
+{
+ auto tensor = _data[docid];
+ if (tensor == nullptr) {
+ return {};
+ }
+ return FastValueBuilderFactory::get().copy(*tensor);
+}
+
+std::unique_ptr<Value>
+TensorExtAttribute::getEmptyTensor() const
+{
+ return FastValueBuilderFactory::get().copy(*_empty_tensor);
+}
+
+TypedCells
+TensorExtAttribute::extract_cells_ref(uint32_t docid) const
+{
+ return get_vector(docid, 0);
+}
+
+const vespalib::eval::Value&
+TensorExtAttribute::get_tensor_ref(uint32_t docid) const
+{
+ auto tensor = _data[docid];
+ return (tensor == nullptr) ? *_empty_tensor : *tensor;
+}
+
+SerializedTensorRef
+TensorExtAttribute::get_serialized_tensor_ref(uint32_t) const
+{
+ notImplemented();
+}
+
+bool
+TensorExtAttribute::supports_extract_cells_ref() const
+{
+ return getConfig().tensorType().is_dense();
+}
+
+bool
+TensorExtAttribute::supports_get_tensor_ref() const
+{
+ return true;
+}
+
+bool
+TensorExtAttribute::supports_get_serialized_tensor_ref() const
+{
+ return false;
+}
+
+const ValueType&
+TensorExtAttribute::getTensorType() const
+{
+ return getConfig().tensorType();
+}
+
+TensorExtAttribute::DistanceMetric
+TensorExtAttribute::distance_metric() const
+{
+ return getConfig().distance_metric();
+}
+
+uint32_t
+TensorExtAttribute::get_num_docs() const
+{
+ return _data.size();
+}
+
+void
+TensorExtAttribute::get_state(const vespalib::slime::Inserter& inserter) const
+{
+ (void) inserter;
+}
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h
new file mode 100644
index 00000000000..a58426cd146
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h
@@ -0,0 +1,54 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_tensor_attribute.h"
+#include "empty_subspace.h"
+#include "subspace_type.h"
+#include <vespa/searchlib/attribute/not_implemented_attribute.h>
+#include <vespa/vespalib/stllike/allocator.h>
+
+namespace search::tensor {
+
+/**
+ * Attribute vector storing a pointer to single tensor value per
+ * document in streaming search. The tensor is not owned by this
+ * attribute vector.
+ */
+class TensorExtAttribute : public NotImplementedAttribute,
+ public ITensorAttribute,
+ public IExtendAttribute
+{
+ std::vector<const vespalib::eval::Value*> _data;
+ SubspaceType _subspace_type;
+ EmptySubspace _empty;
+ std::unique_ptr<vespalib::eval::Value> _empty_tensor;
+public:
+ TensorExtAttribute(const vespalib::string& name, const Config& cfg);
+ ~TensorExtAttribute() override;
+ const ITensorAttribute* asTensorAttribute() const override;
+ void onCommit() override;
+ void onUpdateStat() override;
+ bool addDoc(DocId& docId) override;
+ bool add(const vespalib::eval::Value& v, int32_t) override;
+ IExtendAttribute* getExtendInterface() override;
+ // DocVectorAccess API
+ vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override;
+ VectorBundle get_vectors(uint32_t docid) const override;
+
+ // ITensorAttribute API
+ std::unique_ptr<vespalib::eval::Value> getTensor(uint32_t docid) const override;
+ std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override;
+ vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override;
+ const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override;
+ SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override;
+ bool supports_extract_cells_ref() const override;
+ bool supports_get_tensor_ref() const override;
+ bool supports_get_serialized_tensor_ref() const override;
+ const vespalib::eval::ValueType & getTensorType() const override;
+ search::attribute::DistanceMetric distance_metric() const override;
+ uint32_t get_num_docs() const override;
+ void get_state(const vespalib::slime::Inserter& inserter) const override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/test/attribute_builder.cpp b/searchlib/src/vespa/searchlib/test/attribute_builder.cpp
index cc84355385d..ce8c86367b1 100644
--- a/searchlib/src/vespa/searchlib/test/attribute_builder.cpp
+++ b/searchlib/src/vespa/searchlib/test/attribute_builder.cpp
@@ -7,6 +7,7 @@
#include <vespa/searchlib/attribute/attributevector.h>
#include <vespa/searchlib/attribute/floatbase.h>
#include <vespa/searchlib/attribute/integerbase.h>
+#include <vespa/searchlib/attribute/single_raw_attribute.h>
#include <vespa/searchlib/attribute/stringbase.h>
#include <vespa/searchlib/tensor/tensor_attribute.h>
#include <cassert>
@@ -158,6 +159,13 @@ AttributeBuilder::fill_wset(std::initializer_list<WeightedStringList> values)
}
AttributeBuilder&
+AttributeBuilder::fill(std::initializer_list<vespalib::ConstArrayRef<char>> values)
+{
+ fill_helper<SingleRawAttribute, vespalib::ConstArrayRef<char>>(_attr, values);
+ return *this;
+}
+
+AttributeBuilder&
AttributeBuilder::fill_tensor(const std::vector<vespalib::string>& values)
{
add_docs(_attr, values.size());
diff --git a/searchlib/src/vespa/searchlib/test/attribute_builder.h b/searchlib/src/vespa/searchlib/test/attribute_builder.h
index 339af4e22f5..e6730b74249 100644
--- a/searchlib/src/vespa/searchlib/test/attribute_builder.h
+++ b/searchlib/src/vespa/searchlib/test/attribute_builder.h
@@ -4,6 +4,7 @@
#include <vespa/searchcommon/attribute/config.h>
#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/arrayref.h>
#include <memory>
#include <utility>
#include <vector>
@@ -52,6 +53,9 @@ public:
AttributeBuilder& fill_array(std::initializer_list<StringList> values);
AttributeBuilder& fill_wset(std::initializer_list<WeightedStringList> values);
+ // Fill function for raw attributes
+ AttributeBuilder& fill(std::initializer_list<vespalib::ConstArrayRef<char>> values);
+
/**
* Fill this tensor attribute with the given tensor values.
*
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp
index d6f06c9161e..6ac4fea2921 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp
@@ -23,8 +23,7 @@ namespace search::docsummary {
DocsumFieldWriterFactory::DocsumFieldWriterFactory(bool use_v8_geo_positions, const IDocsumEnvironment& env, const IQueryTermFilterFactory& query_term_filter_factory)
: _use_v8_geo_positions(use_v8_geo_positions),
_env(env),
- _query_term_filter_factory(query_term_filter_factory),
- _matching_elems_fields(std::make_shared<MatchingElementsFields>())
+ _query_term_filter_factory(query_term_filter_factory)
{
}
@@ -58,7 +57,8 @@ throw_missing_source(const vespalib::string& command)
std::unique_ptr<DocsumFieldWriter>
DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& field_name,
const vespalib::string& command,
- const vespalib::string& source)
+ const vespalib::string& source,
+ std::shared_ptr<MatchingElementsFields> matching_elems_fields)
{
std::unique_ptr<DocsumFieldWriter> fieldWriter;
if (command == command::dynamic_teaser) {
@@ -116,9 +116,9 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie
if (has_attribute_manager()) {
auto attr_ctx = getEnvironment().getAttributeManager()->createContext();
if (attr_ctx->getAttribute(source_field) != nullptr) {
- fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), source_field, true, _matching_elems_fields);
+ fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), source_field, true, matching_elems_fields);
} else {
- fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, _matching_elems_fields);
+ fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, matching_elems_fields);
}
throw_if_nullptr(fieldWriter, command);
}
@@ -126,7 +126,7 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie
const vespalib::string& source_field = source.empty() ? field_name : source;
if (has_attribute_manager()) {
auto attr_ctx = getEnvironment().getAttributeManager()->createContext();
- fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, _matching_elems_fields);
+ fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, matching_elems_fields);
throw_if_nullptr(fieldWriter, command);
}
} else if (command == command::documentid) {
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h
index e50fb85cca6..7175f043701 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h
@@ -20,7 +20,6 @@ class DocsumFieldWriterFactory : public IDocsumFieldWriterFactory
const IDocsumEnvironment& _env;
const IQueryTermFilterFactory& _query_term_filter_factory;
protected:
- std::shared_ptr<MatchingElementsFields> _matching_elems_fields;
const IDocsumEnvironment& getEnvironment() const noexcept { return _env; }
bool has_attribute_manager() const noexcept;
public:
@@ -28,7 +27,8 @@ public:
~DocsumFieldWriterFactory() override;
std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name,
const vespalib::string& command,
- const vespalib::string& source) override;
+ const vespalib::string& source,
+ std::shared_ptr<MatchingElementsFields> matching_elems_fields) override;
};
}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h
index 6a5cd691857..bc2ebe3c40c 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h
@@ -5,6 +5,8 @@
#include <memory>
#include <vespa/vespalib/stllike/string.h>
+namespace search { class MatchingElementsFields; }
+
namespace search::docsummary {
class DocsumFieldWriter;
@@ -21,7 +23,8 @@ public:
*/
virtual std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name,
const vespalib::string& command,
- const vespalib::string& source) = 0;
+ const vespalib::string& source,
+ std::shared_ptr<MatchingElementsFields> matching_elems_fields) = 0;
};
}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
index f620dcb1df5..eddb67f5822 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
@@ -5,6 +5,7 @@
#include "docsum_field_writer_factory.h"
#include "resultclass.h"
#include <vespa/config-summary.h>
+#include <vespa/searchlib/common/matching_elements_fields.h>
#include <vespa/vespalib/stllike/hash_map.hpp>
#include <vespa/vespalib/util/exceptions.h>
#include <atomic>
@@ -124,6 +125,7 @@ ResultConfig::readConfig(const SummaryConfig &cfg, const char *configId, IDocsum
break;
}
resClass->set_omit_summary_features(cfg_class.omitsummaryfeatures);
+ auto matching_elems_fields = std::make_shared<MatchingElementsFields>();
for (const auto & field : cfg_class.fields) {
const char *fieldname = field.name.c_str();
vespalib::string command = field.command;
@@ -134,7 +136,8 @@ ResultConfig::readConfig(const SummaryConfig &cfg, const char *configId, IDocsum
try {
docsum_field_writer = docsum_field_writer_factory.create_docsum_field_writer(fieldname,
command,
- source_name);
+ source_name,
+ matching_elems_fields);
} catch (const vespalib::IllegalArgumentException& ex) {
LOG(error, "Exception during setup of summary result class '%s': field='%s', command='%s', source='%s': %s",
cfg_class.name.c_str(), fieldname, command.c_str(), source_name.c_str(), ex.getMessage().c_str());
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
index 3a9c2b1f1e7..d76b4ecc0b5 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
@@ -1,10 +1,10 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.duper;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.HostInfo;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
import java.util.logging.Level;
import com.yahoo.vespa.service.monitor.DuperModelListener;
@@ -26,8 +26,8 @@ public class DuperModel {
private static Logger logger = Logger.getLogger(DuperModel.class.getName());
private final Map<ApplicationId, ApplicationInfo> applicationsById = new HashMap<>();
- private final Map<HostName, ApplicationId> idsByHostname = new HashMap<>();
- private final Map<ApplicationId, HashSet<HostName>> hostnamesById = new HashMap<>();
+ private final Map<DomainName, ApplicationId> idsByHostname = new HashMap<>();
+ private final Map<ApplicationId, HashSet<DomainName>> hostnamesById = new HashMap<>();
private final List<DuperModelListener> listeners = new ArrayList<>();
private boolean isComplete = false;
@@ -67,7 +67,7 @@ public class DuperModel {
return Optional.ofNullable(applicationsById.get(applicationId));
}
- public Optional<ApplicationInfo> getApplicationInfo(HostName hostName) {
+ public Optional<ApplicationInfo> getApplicationInfo(DomainName hostName) {
return Optional.ofNullable(idsByHostname.get(hostName)).map(applicationsById::get);
}
@@ -76,12 +76,12 @@ public class DuperModel {
}
/** Note: Returns an empty set for unknown application. */
- public Set<HostName> getHostnames(ApplicationId applicationId) {
- HashSet<HostName> set = hostnamesById.get(applicationId);
+ public Set<DomainName> getHostnames(ApplicationId applicationId) {
+ HashSet<DomainName> set = hostnamesById.get(applicationId);
return set == null ? Set.of() : Set.copyOf(set);
}
- public Optional<ApplicationId> getApplicationId(HostName hostname) {
+ public Optional<ApplicationId> getApplicationId(DomainName hostname) {
return Optional.ofNullable(idsByHostname.get(hostname));
}
@@ -103,7 +103,7 @@ public class DuperModel {
}
public void remove(ApplicationId applicationId) {
- Set<HostName> hostnames = hostnamesById.remove(applicationId);
+ Set<DomainName> hostnames = hostnamesById.remove(applicationId);
if (hostnames != null) {
hostnames.forEach(idsByHostname::remove);
}
@@ -117,11 +117,11 @@ public class DuperModel {
/** Update hostnamesById and idsByHostname based on a new applicationInfo. */
private void updateHostnameVsIdMaps(ApplicationInfo applicationInfo, ApplicationId id) {
- Set<HostName> removedHosts = new HashSet<>(hostnamesById.computeIfAbsent(id, k -> new HashSet<>()));
+ Set<DomainName> removedHosts = new HashSet<>(hostnamesById.computeIfAbsent(id, k -> new HashSet<>()));
applicationInfo.getModel().getHosts().stream()
.map(HostInfo::getHostname)
- .map(HostName::of)
+ .map(DomainName::of)
.forEach(hostname -> {
if (!removedHosts.remove(hostname)) {
// hostname has been added
@@ -135,7 +135,7 @@ public class DuperModel {
logger.log(Level.WARNING, hostname + " has been reassigned from " +
previousId.toFullString() + " to " + id.toFullString());
- Set<HostName> previousHostnames = hostnamesById.get(previousId);
+ Set<DomainName> previousHostnames = hostnamesById.get(previousId);
if (previousHostnames != null) {
previousHostnames.remove(hostname);
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
index b8c980a8760..400c4f4b907 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
@@ -1,14 +1,14 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.duper;
-import com.yahoo.component.annotation.Inject;
+import ai.vespa.http.DomainName;
import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.component.annotation.Inject;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.SuperModel;
import com.yahoo.config.model.api.SuperModelListener;
import com.yahoo.config.model.api.SuperModelProvider;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.service.monitor.CriticalRegion;
import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
import com.yahoo.vespa.service.monitor.DuperModelListener;
@@ -133,7 +133,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi
}
@Override
- public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) {
+ public void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames) {
InfraApplication application = supportedInfraApplications.get(applicationId);
if (application == null) {
throw new IllegalArgumentException("There is no infrastructure application with ID '" + applicationId + "'");
@@ -172,7 +172,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi
return lockedSupplier(() -> duperModel.getApplicationInfo(applicationId));
}
- public Optional<ApplicationInfo> getApplicationInfo(HostName hostname) {
+ public Optional<ApplicationInfo> getApplicationInfo(DomainName hostname) {
return lockedSupplier(() -> duperModel.getApplicationInfo(hostname));
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
index e82e2cc0265..07f9d7bddc5 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.duper;
+import ai.vespa.http.DomainName;
import com.yahoo.component.Version;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.HostInfo;
@@ -9,7 +10,6 @@ import com.yahoo.config.model.api.ServiceInfo;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
import com.yahoo.vespa.applicationmodel.ClusterId;
@@ -93,12 +93,12 @@ public abstract class InfraApplication implements InfraApplicationApi {
return new TenantId(application.id().tenant().value());
}
- public ApplicationInfo makeApplicationInfo(List<HostName> hostnames) {
+ public ApplicationInfo makeApplicationInfo(List<DomainName> hostnames) {
List<HostInfo> hostInfos = hostnames.stream().map(this::makeHostInfo).toList();
return new ApplicationInfo(application.id(), 0, new HostsModel(hostInfos));
}
- private HostInfo makeHostInfo(HostName hostname) {
+ private HostInfo makeHostInfo(DomainName hostname) {
PortInfo portInfo = new PortInfo(healthPort, StateV1HealthModel.HTTP_HEALTH_PORT_TAGS);
Map<String, String> properties = new HashMap<>();
@@ -116,7 +116,7 @@ public abstract class InfraApplication implements InfraApplicationApi {
return new HostInfo(hostname.value(), Collections.singletonList(serviceInfo));
}
- public ConfigId configIdFor(HostName hostname) {
+ public ConfigId configIdFor(DomainName hostname) {
// Not necessarily unique, but service monitor doesn't require it to be unique.
return new ConfigId(String.format("%s/%s", clusterSpecId.value(), prefixTo(hostname.value(), '.')));
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
index a654a4557ef..1d832cc0eec 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java
@@ -1,7 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.health;
-import com.yahoo.config.provision.HostName;
+import ai.vespa.http.DomainName;
import com.yahoo.vespa.service.executor.RunletExecutor;
import com.yahoo.vespa.service.monitor.ServiceId;
@@ -22,7 +22,7 @@ class StateV1HealthEndpoint implements HealthEndpoint {
private final RunletExecutor executor;
StateV1HealthEndpoint(ServiceId serviceId,
- HostName hostname,
+ DomainName hostname,
int port,
Duration delay,
Duration requestTimeout,
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
index 3391bbcf82a..3969ef141bd 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java
@@ -1,11 +1,11 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.health;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.HostInfo;
import com.yahoo.config.model.api.PortInfo;
import com.yahoo.config.model.api.ServiceInfo;
-import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.service.executor.RunletExecutor;
import com.yahoo.vespa.service.model.ApplicationInstanceGenerator;
import com.yahoo.vespa.service.monitor.ServiceId;
@@ -45,7 +45,7 @@ public class StateV1HealthModel implements AutoCloseable {
Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>();
for (HostInfo hostInfo : application.getModel().getHosts()) {
- HostName hostname = HostName.of(hostInfo.getHostname());
+ DomainName hostname = DomainName.of(hostInfo.getHostname());
for (ServiceInfo serviceInfo : hostInfo.getServices()) {
ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo);
for (PortInfo portInfo : serviceInfo.getPorts()) {
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
index 815f8de26b4..53db7cc135c 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.model;
+import ai.vespa.http.DomainName;
import com.yahoo.component.annotation.Inject;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.provision.ApplicationId;
@@ -68,7 +69,7 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor {
@Override
public Optional<ApplicationInstanceReference> getApplicationInstanceReference(HostName hostname) {
- return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname))
+ return duperModelManager.getApplicationInfo(DomainName.of(hostname.s()))
.map(ApplicationInfo::getApplicationId)
.map(modelGenerator::toApplicationInstanceReference);
}
@@ -113,11 +114,7 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor {
}
private Optional<ApplicationInfo> getApplicationInfo(HostName hostname) {
- return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname));
+ return duperModelManager.getApplicationInfo(DomainName.of(hostname.s()));
}
- /** The duper model uses HostName from config.provision. */
- private static com.yahoo.config.provision.HostName toConfigProvisionHostName(HostName hostname) {
- return com.yahoo.config.provision.HostName.of(hostname.s());
- }
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
index 91782d5a582..783f0a27e51 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
@@ -1,8 +1,8 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.monitor;
+import ai.vespa.http.DomainName;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
import java.util.List;
import java.util.Optional;
@@ -24,7 +24,7 @@ public interface DuperModelInfraApi {
boolean infraApplicationIsActive(ApplicationId applicationId);
/** Update the DuperModel: A supported infrastructure application has been (re)activated or is active. */
- void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames);
+ void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames);
/** Update the DuperModel: A supported infrastructure application has been removed or is not active. */
void infraApplicationRemoved(ApplicationId applicationId);
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java
index 6cf7d4d9f26..ad7f0f188c0 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java
@@ -1,17 +1,16 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.duper;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.SuperModel;
import com.yahoo.config.model.api.SuperModelListener;
import com.yahoo.config.model.api.SuperModelProvider;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import java.util.List;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.yahoo.vespa.service.duper.DuperModelManager.configServerApplication;
@@ -64,7 +63,7 @@ public class DuperModelManagerTest {
makeManager(false);
ApplicationId id = proxyHostApplication.getApplicationId();
- List<HostName> proxyHostHosts = Stream.of("proxyhost1", "proxyhost2").map(HostName::of).toList();
+ List<DomainName> proxyHostHosts = Stream.of("proxyhost1", "proxyhost2").map(DomainName::of).toList();
verify(duperModel, times(0)).add(any());
manager.infraApplicationActivated(id, proxyHostHosts);
verify(duperModel, times(1)).add(any());
@@ -91,12 +90,12 @@ public class DuperModelManagerTest {
}
private void testEnabledConfigServerLikeInfraApplication(ApplicationId firstId, ApplicationId secondId) {
- List<HostName> hostnames1 = Stream.of("node11", "node12").map(HostName::of).toList();
+ List<DomainName> hostnames1 = Stream.of("node11", "node12").map(DomainName::of).toList();
manager.infraApplicationActivated(firstId, hostnames1);
verify(duperModel, times(1)).add(any());
// Adding the second config server like application will be ignored
- List<HostName> hostnames2 = Stream.of("node21", "node22").map(HostName::of).toList();
+ List<DomainName> hostnames2 = Stream.of("node21", "node22").map(DomainName::of).toList();
assertThrows(IllegalArgumentException.class, () -> manager.infraApplicationActivated(secondId, hostnames2));
verify(duperModel, times(1)).add(any());
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
index 73a49ca8717..568dc3640e2 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
@@ -1,11 +1,11 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.duper;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.HostInfo;
import com.yahoo.config.model.api.Model;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.service.monitor.DuperModelListener;
import org.junit.Before;
import org.junit.Test;
@@ -14,7 +14,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.assertEquals;
@@ -34,12 +33,12 @@ public class DuperModelTest {
private final ApplicationId id1 = ApplicationId.fromSerializedForm("tenant:app1:default");
private final ApplicationInfo application1 = mock(ApplicationInfo.class);
- private final HostName hostname1_1 = HostName.of("hostname1-1");
- private final HostName hostname1_2 = HostName.of("hostname1-2");
+ private final DomainName hostname1_1 = DomainName.of("hostname1-1");
+ private final DomainName hostname1_2 = DomainName.of("hostname1-2");
private final ApplicationId id2 = ApplicationId.fromSerializedForm("tenant:app2:default");
private final ApplicationInfo application2 = mock(ApplicationInfo.class);
- private final HostName hostname2_1 = HostName.of("hostname2-1");
+ private final DomainName hostname2_1 = DomainName.of("hostname2-1");
private final DuperModelListener listener1 = mock(DuperModelListener.class);
@@ -49,7 +48,7 @@ public class DuperModelTest {
setUpApplication(id2, application2, hostname2_1);
}
- private void setUpApplication(ApplicationId id, ApplicationInfo info, HostName... hostnames) {
+ private void setUpApplication(ApplicationId id, ApplicationInfo info, DomainName... hostnames) {
when(info.getApplicationId()).thenReturn(id);
Model model = mock(Model.class);
@@ -129,7 +128,7 @@ public class DuperModelTest {
addAndVerifyApplication1("host1");
addAndVerifyApplication1("host1", "host2");
addAndVerifyApplication1("host2", "host3");
- assertEquals(Optional.empty(), duperModel.getApplicationId(HostName.of("host1")));
+ assertEquals(Optional.empty(), duperModel.getApplicationId(DomainName.of("host1")));
duperModel.remove(id1);
assertEquals(0, duperModel.numberOfApplications());
@@ -138,7 +137,7 @@ public class DuperModelTest {
}
private void addAndVerifyApplication1(String... hostnameStrings) {
- HostName[] hostnameArray = Stream.of(hostnameStrings).map(HostName::of).toArray(HostName[]::new);
+ DomainName[] hostnameArray = Stream.of(hostnameStrings).map(DomainName::of).toArray(DomainName[]::new);
setUpApplication(id1, application1, hostnameArray);
duperModel.add(application1);
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
index d99e882985f..9f46b5dfe8f 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
@@ -1,13 +1,13 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.health;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
-import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.applicationmodel.ServiceStatus;
import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
import com.yahoo.vespa.service.duper.ConfigServerApplication;
-import com.yahoo.vespa.service.monitor.ServiceId;
import com.yahoo.vespa.service.monitor.ConfigserverUtil;
+import com.yahoo.vespa.service.monitor.ServiceId;
import org.junit.Test;
import java.util.HashMap;
@@ -87,7 +87,7 @@ public class ApplicationHealthMonitorTest {
return new ServiceId(configServerApplication.getApplicationId(),
configServerApplication.getClusterId(),
configServerApplication.getServiceType(),
- configServerApplication.configIdFor(HostName.of(hostname)));
+ configServerApplication.configIdFor(DomainName.of(hostname)));
}
private ServiceStatus getStatus(ApplicationHealthMonitor monitor, String hostname) {
@@ -95,7 +95,7 @@ public class ApplicationHealthMonitorTest {
configServerApplication.getApplicationId(),
configServerApplication.getClusterId(),
configServerApplication.getServiceType(),
- configServerApplication.configIdFor(HostName.of(hostname)))
+ configServerApplication.configIdFor(DomainName.of(hostname)))
.serviceStatus();
}
} \ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
index f7ee64b52a6..f9129b5e4f4 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
@@ -1,8 +1,8 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.health;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
-import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.applicationmodel.ServiceStatus;
import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
import com.yahoo.vespa.service.duper.ControllerHostApplication;
@@ -14,7 +14,6 @@ import org.junit.Before;
import org.junit.Test;
import java.util.List;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.assertEquals;
@@ -48,7 +47,7 @@ public class HealthMonitorManagerTest {
public void infrastructureApplication() {
ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true);
- List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::of).toList();
+ List<DomainName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(DomainName::of).toList();
ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
manager.applicationActivated(proxyHostApplicationInfo);
@@ -77,7 +76,7 @@ public class HealthMonitorManagerTest {
infraApplication.getApplicationId(),
infraApplication.getClusterId(),
infraApplication.getServiceType(),
- infraApplication.configIdFor(HostName.of(hostname))).serviceStatus();
+ infraApplication.configIdFor(DomainName.of(hostname))).serviceStatus();
assertEquals(expected, actual);
@@ -85,7 +84,7 @@ public class HealthMonitorManagerTest {
infraApplication.getApplicationId(),
infraApplication.getClusterId(),
infraApplication.getServiceType(),
- infraApplication.configIdFor(HostName.of(hostname)));
+ infraApplication.configIdFor(DomainName.of(hostname)));
}
} \ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
index 18b56f76e88..01b7930a74f 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
@@ -1,10 +1,10 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.health;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.PortInfo;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.applicationmodel.ClusterId;
import com.yahoo.vespa.applicationmodel.ConfigId;
import com.yahoo.vespa.applicationmodel.ServiceStatus;
@@ -18,7 +18,6 @@ import org.junit.Test;
import java.time.Duration;
import java.util.List;
import java.util.Map;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.assertEquals;
@@ -38,7 +37,7 @@ public class StateV1HealthModelTest {
private final Duration requestTimeout = Duration.ofSeconds(2);
private final Duration keepAlive = Duration.ofSeconds(3);
private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
- private final List<HostName> hostnames = Stream.of("host1", "host2").map(HostName::of).toList();
+ private final List<DomainName> hostnames = Stream.of("host1", "host2").map(DomainName::of).toList();
private final ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
private final StateV1HealthModel model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor);
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java
index a2532590f52..e85a00d732c 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java
@@ -1,8 +1,8 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.model;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
-import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.applicationmodel.ApplicationInstance;
import com.yahoo.vespa.applicationmodel.ServiceStatus;
@@ -12,7 +12,6 @@ import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
import org.junit.Test;
import java.util.List;
-import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -34,7 +33,7 @@ public class ApplicationInstanceGeneratorTest {
when(statusProvider.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.NOT_CHECKED));
Zone zone = mock(Zone.class);
ApplicationInfo configServer = configServerApplication.makeApplicationInfo(
- configServerList.stream().map(HostName::of).toList());
+ configServerList.stream().map(DomainName::of).toList());
ApplicationInstance applicationInstance = new ApplicationInstanceGenerator(configServer, zone)
.makeApplicationInstance(statusProvider);
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java
index c93e4dfb7fe..23422640af9 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.model;
+import ai.vespa.http.DomainName;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.HostInfo;
import com.yahoo.config.model.api.Model;
@@ -100,7 +101,7 @@ public class ServiceHostListenerAdapterTest {
}
private Optional<ApplicationInfo> getDuperModelApplicationInfo(String hostname) {
- return duperModel.getApplicationInfo(com.yahoo.config.provision.HostName.of(hostname));
+ return duperModel.getApplicationInfo(DomainName.of(hostname));
}
private void removeAndVerify(ApplicationId id, boolean listenerInvoked) {
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java
index 498a9dfc15e..d2480cd6520 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java
@@ -1,18 +1,18 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.monitor;
+import ai.vespa.http.DomainName;
import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.config.model.api.ApplicationInfo;
-import com.yahoo.config.provision.HostName;
import com.yahoo.vespa.service.duper.ConfigServerApplication;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import java.util.List;
/**
* @author hakon
*/
public class ConfigserverUtil {
+
/** Create a ConfigserverConfig with the given settings. */
public static ConfigserverConfig create(
boolean multitenant,
@@ -31,16 +31,10 @@ public class ConfigserverUtil {
return create(true, "cfg1", "cfg2", "cfg3");
}
- public static ApplicationInfo makeConfigServerApplicationInfo(
- String configServerHostname1,
- String configServerHostname2,
- String configServerHostname3) {
- return new ConfigServerApplication().makeApplicationInfo(
- Stream.of(configServerHostname1, configServerHostname2, configServerHostname3)
- .map(HostName::of).toList());
- }
-
public static ApplicationInfo makeExampleConfigServer() {
- return makeConfigServerApplicationInfo("cfg1", "cfg2", "cfg3");
+ return new ConfigServerApplication().makeApplicationInfo(List.of(DomainName.of("cfg1"),
+ DomainName.of("cfg2"),
+ DomainName.of("cfg3")));
}
+
}
diff --git a/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp b/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp
index 1c0f9ec7a59..e24e97ff2a4 100644
--- a/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp
+++ b/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp
@@ -49,7 +49,7 @@ struct BTreeLockableMap<T>::ValueTraits {
return store.addEntry(value).ref();
}
static void remove_by_wrapped_value(DataStoreType& store, uint64_t value) noexcept {
- store.holdElem(entry_ref_from_value(value), 1);
+ store.hold_entry(entry_ref_from_value(value));
}
static ValueType unwrap_from_key_value(const DataStoreType& store, [[maybe_unused]] uint64_t key, uint64_t value) {
return store.getEntry(entry_ref_from_value(value));
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
index 92b9e832a23..1d29a8795d5 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
@@ -1054,7 +1054,7 @@ FileStorHandlerImpl::Stripe::waitUntilNoLocks() const
{
std::unique_lock guard(*_lock);
while (!_lockedBuckets.empty()) {
- _cond->wait(guard);
+ _cond->wait_for(guard, 100ms);
}
}
@@ -1062,7 +1062,7 @@ void
FileStorHandlerImpl::Stripe::waitInactive(const AbortBucketOperationsCommand& cmd) const {
std::unique_lock guard(*_lock);
while (hasActive(guard, cmd)) {
- _cond->wait(guard);
+ _cond->wait_for(guard, 100ms);
}
}
diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
index d0e3d1f038b..2caf2de1d0b 100644
--- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
@@ -7,6 +7,7 @@
#include <vespa/persistence/spi/docentry.h>
#include <vespa/document/datatype/positiondatatype.h>
#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/datatype/tensor_data_type.h>
#include <vespa/document/datatype/weightedsetdatatype.h>
#include <vespa/document/datatype/mapdatatype.h>
#include <vespa/searchlib/aggregation/modifiers.h>
@@ -14,6 +15,7 @@
#include <vespa/searchlib/common/packets.h>
#include <vespa/searchlib/uca/ucaconverter.h>
#include <vespa/searchlib/features/setup.h>
+#include <vespa/searchlib/tensor/tensor_ext_attribute.h>
#include <vespa/searchcommon/attribute/config.h>
#include <vespa/vespalib/geo/zcurve.h>
#include <vespa/vespalib/objects/nbostream.h>
@@ -99,6 +101,16 @@ createMultiValueAttribute(const vespalib::string & name, const document::FieldVa
return {};
}
+const document::TensorDataType*
+get_tensor_type(const document::FieldValue& fv)
+{
+ auto tfv = dynamic_cast<const document::TensorFieldValue*>(&fv);
+ if (tfv == nullptr) {
+ return nullptr;
+ }
+ return dynamic_cast<const document::TensorDataType*>(tfv->getDataType());
+}
+
AttributeVector::SP
createAttribute(const vespalib::string & name, const document::FieldValue & fv)
{
@@ -111,6 +123,12 @@ createAttribute(const vespalib::string & name, const document::FieldValue & fv)
return std::make_shared<search::SingleStringExtAttribute>(name);
} else if (fv.isA(document::FieldValue::Type::RAW)) {
return std::make_shared<search::attribute::SingleRawExtAttribute>(name);
+ } else if (fv.isA(document::FieldValue::Type::TENSOR) && get_tensor_type(fv) != nullptr) {
+ search::attribute::Config cfg(search::attribute::BasicType::TENSOR, search::attribute::CollectionType::SINGLE);
+ auto tdt = get_tensor_type(fv);
+ assert(tdt != nullptr);
+ cfg.setTensorType(tdt->getTensorType());
+ return std::make_shared<search::tensor::TensorExtAttribute>(name, cfg);
} else {
LOG(debug, "Can not make an attribute out of %s of type '%s'.", name.c_str(), fv.className());
}
@@ -444,6 +462,14 @@ SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const Content & c)
} else if (_attribute.is_raw_type()) {
auto raw_value = value.getAsRaw();
attr.add(vespalib::ConstArrayRef<char>(raw_value.first, raw_value.second), c.getWeight());
+ } else if (_attribute.isTensorType()) {
+ auto tfvalue = dynamic_cast<const document::TensorFieldValue*>(&value);
+ if (tfvalue != nullptr) {
+ auto tensor = tfvalue->getAsTensorPtr();
+ if (tensor != nullptr) {
+ attr.add(*tensor, c.getWeight());
+ }
+ }
} else {
assert(false && "We got an attribute vector that is of an unknown type");
}
diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp
index 36873b713aa..b103d7d85b2 100644
--- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp
+++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp
@@ -48,7 +48,8 @@ DocsumFieldWriterFactory::~DocsumFieldWriterFactory() = default;
std::unique_ptr<DocsumFieldWriter>
DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& field_name,
const vespalib::string& command,
- const vespalib::string& source)
+ const vespalib::string& source,
+ std::shared_ptr<MatchingElementsFields> matching_elems_fields)
{
std::unique_ptr<DocsumFieldWriter> fieldWriter;
using namespace search::docsummary;
@@ -65,10 +66,10 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie
} else if ((command == command::matched_attribute_elements_filter) ||
(command == command::matched_elements_filter)) {
vespalib::string source_field = source.empty() ? field_name : source;
- populate_fields(*_matching_elems_fields, _vsm_fields_config, source_field);
- fieldWriter = MatchedElementsFilterDFW::create(source_field, _matching_elems_fields);
+ populate_fields(*matching_elems_fields, _vsm_fields_config, source_field);
+ fieldWriter = MatchedElementsFilterDFW::create(source_field, matching_elems_fields);
} else {
- return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(field_name, command, source);
+ return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(field_name, command, source, matching_elems_fields);
}
return fieldWriter;
}
diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h
index 078c466d3d2..ac5cae8c49d 100644
--- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h
+++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h
@@ -21,7 +21,8 @@ public:
std::unique_ptr<search::docsummary::DocsumFieldWriter>
create_docsum_field_writer(const vespalib::string& field_name,
const vespalib::string& command,
- const vespalib::string& source) override;
+ const vespalib::string& source,
+ std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) override;
};
}
diff --git a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java
index fda456ace05..14b44ede12a 100644
--- a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java
+++ b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java
@@ -2,6 +2,7 @@
package com.yahoo.vdslib;
public class VisitorStatistics {
+
int bucketsVisited = 0;
long documentsVisited = 0;
long bytesVisited = 0;
@@ -20,8 +21,8 @@ public class VisitorStatistics {
public void setBucketsVisited(int bucketsVisited) { this.bucketsVisited = bucketsVisited; }
/**
- * @return the number of documents matching the document selection in the backend and that
- * has been passed to the client-specified visitor instance (dumpvisitor, searchvisitor etc).
+ * Returns the number of documents matching the document selection in the backend that
+ * has been passed to the client-specified visitor instance (dumpvisitor, searchvisitor etc).
*/
public long getDocumentsVisited() { return documentsVisited; }
public void setDocumentsVisited(long documentsVisited) { this.documentsVisited = documentsVisited; }
@@ -30,9 +31,9 @@ public class VisitorStatistics {
public void setBytesVisited(long bytesVisited) { this.bytesVisited = bytesVisited; }
/**
- * @return Number of documents returned to the visitor client by the backend. This number may
- * be lower than that returned by getDocumentsVisited() since the client-specified visitor
- * instance may further have filtered the set of documents returned by the backend.
+ * Returns the number of documents returned to the visitor client by the backend. This number may
+ * be lower than that returned by getDocumentsVisited() since the client-specified visitor
+ * instance may further have filtered the set of documents returned by the backend.
*/
public long getDocumentsReturned() { return documentsReturned; }
public void setDocumentsReturned(long documentsReturned) { this.documentsReturned = documentsReturned; }
@@ -40,15 +41,14 @@ public class VisitorStatistics {
public long getBytesReturned() { return bytesReturned; }
public void setBytesReturned(long bytesReturned) { this.bytesReturned = bytesReturned; }
+ @Override
public String toString() {
- String out =
+ return
"Buckets visited: " + bucketsVisited + "\n" +
"Documents visited: " + documentsVisited + "\n" +
"Bytes visited: " + bytesVisited + "\n" +
"Documents returned: " + documentsReturned + "\n" +
"Bytes returned: " + bytesReturned + "\n";
-
- return out;
}
}
diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp
index a10aca45b33..b5c756aece9 100644
--- a/vdslib/src/tests/distribution/distributiontest.cpp
+++ b/vdslib/src/tests/distribution/distributiontest.cpp
@@ -17,8 +17,9 @@
#include <vespa/vespalib/util/size_literals.h>
#include <gmock/gmock.h>
#include <chrono>
-#include <thread>
#include <fstream>
+#include <iterator>
+#include <thread>
using namespace ::testing;
@@ -434,7 +435,8 @@ TEST(DistributionTest, test_distribution)
_distribution[i].second = distr.getIdealStorageNodes(
systemState, document::BucketId(26, i));
sort(_distribution[i].second.begin(), _distribution[i].second.end());
- unique(_distribution[i].second.begin(), _distribution[i].second.end());
+ auto unique_nodes = std::distance(_distribution[i].second.begin(), unique(_distribution[i].second.begin(), _distribution[i].second.end()));
+ _distribution[i].second.resize(unique_nodes);
for (unsigned j = 0; j < _distribution[i].second.size(); j++) {
_nodeCount[_distribution[i].second[j]]++;
diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
index 18475a8e4d9..95f1179b1a3 100644
--- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
+++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt
@@ -80,7 +80,7 @@ net.java.dev.jna:jna:5.11.0
net.openhft:zero-allocation-hashing:0.16
org.antlr:antlr-runtime:3.5.3
org.antlr:antlr4-runtime:4.11.1
-org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.5
+org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.6
org.apache.commons:commons-compress:1.22
org.apache.commons:commons-csv:1.8
org.apache.commons:commons-exec:1.3
diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java
index 9195b5ab858..cf9a36f2aa8 100644
--- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java
+++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java
@@ -35,7 +35,7 @@ class ApacheClusterTest {
final WireMockExtension server = new WireMockExtension();
@Test
- void testClient() throws IOException, ExecutionException, InterruptedException, TimeoutException {
+ void testClient() throws Exception {
for (Compression compression : Compression.values()) {
try (ApacheCluster cluster = new ApacheCluster(new FeedClientBuilderImpl(List.of(URI.create("http://localhost:" + server.port())))
.setCompression(compression))) {
@@ -48,25 +48,28 @@ class ApacheClusterTest {
Map.of("name1", () -> "value1",
"name2", () -> "value2"),
"content".getBytes(UTF_8),
- Duration.ofSeconds(20)),
+ Duration.ofSeconds(10)),
vessel);
- HttpResponse response = vessel.get(15, TimeUnit.SECONDS);
- assertEquals("{}", new String(response.body(), UTF_8));
- assertEquals(200, response.code());
- ByteArrayOutputStream buffer = new ByteArrayOutputStream();
- try (OutputStream zip = new GZIPOutputStream(buffer)) { zip.write("content".getBytes(UTF_8)); }
- server.verify(1, anyRequestedFor(anyUrl()));
- RequestPatternBuilder expected = postRequestedFor(urlEqualTo("/path")).withHeader("name1", equalTo("value1"))
- .withHeader("name2", equalTo("value2"))
- .withHeader("Content-Type", equalTo("application/json; charset=UTF-8"))
- .withRequestBody(equalTo("content"));
- expected = switch (compression) {
- case auto, none -> expected.withoutHeader("Content-Encoding");
- case gzip -> expected.withHeader("Content-Encoding", equalTo("gzip"));
+ AutoCloseable verifyResponse = () -> {
+ HttpResponse response = vessel.get(15, TimeUnit.SECONDS);
+ assertEquals("{}", new String(response.body(), UTF_8));
+ assertEquals(200, response.code());
};
- server.verify(1, expected);
- server.resetRequests();
+ AutoCloseable verifyServer = () -> {
+ server.verify(1, anyRequestedFor(anyUrl()));
+ RequestPatternBuilder expected = postRequestedFor(urlEqualTo("/path")).withHeader("name1", equalTo("value1"))
+ .withHeader("name2", equalTo("value2"))
+ .withHeader("Content-Type", equalTo("application/json; charset=UTF-8"))
+ .withRequestBody(equalTo("content"));
+ expected = switch (compression) {
+ case auto, none -> expected.withoutHeader("Content-Encoding");
+ case gzip -> expected.withHeader("Content-Encoding", equalTo("gzip"));
+ };
+ server.verify(1, expected);
+ server.resetRequests();
+ };
+ try (verifyServer; verifyResponse) { }
}
}
}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
index fe1e2b46830..2a688ad078b 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
@@ -1226,7 +1226,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
getProperty(request, TRACELEVEL, integerParser).ifPresent(parameters::setTraceLevel);
- getProperty(request, CONTINUATION).map(ProgressToken::fromSerializedString).ifPresent(parameters::setResumeToken);
+ getProperty(request, CONTINUATION, ProgressToken::fromSerializedString).ifPresent(parameters::setResumeToken);
parameters.setPriority(DocumentProtocol.Priority.NORMAL_4);
getProperty(request, FROM_TIMESTAMP, unsignedLongParser).ifPresent(parameters::setFromTimestamp);
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
index 33e46ebc75f..31f4038c16e 100644
--- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
+++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
@@ -37,15 +37,17 @@ public abstract class Maintainer implements Runnable {
private final AtomicBoolean shutDown = new AtomicBoolean();
private final boolean ignoreCollision;
private final Clock clock;
+ private final Double successFactorBaseline;
public Maintainer(String name, Duration interval, Clock clock, JobControl jobControl,
- JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision) {
+ JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision, Double successFactorBaseline) {
this.name = name;
this.interval = requireInterval(interval);
this.jobControl = Objects.requireNonNull(jobControl);
this.jobMetrics = Objects.requireNonNull(jobMetrics);
this.ignoreCollision = ignoreCollision;
this.clock = clock;
+ this.successFactorBaseline = successFactorBaseline;
var startedAt = clock.instant();
Objects.requireNonNull(clusterHostnames);
Duration initialDelay = staggeredDelay(interval, startedAt, HostName.getLocalhost(), clusterHostnames)
@@ -55,6 +57,10 @@ public abstract class Maintainer implements Runnable {
jobControl.started(name(), this);
}
+ public Maintainer(String name, Duration interval, Clock clock, JobControl jobControl,
+ JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision) {
+ this(name, interval, clock, jobControl, jobMetrics, clusterHostnames, ignoreCollision, 1.0);
+ }
@Override
public void run() {
lockAndMaintain(false);
@@ -91,16 +97,16 @@ public abstract class Maintainer implements Runnable {
/**
* Called once each time this maintenance job should run.
*
- * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success).
+ * @return the degree to which the run was deviated from the successFactorBaseline - a number between -1 (no success), to 0 (complete success).
* Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do
- * nothing, 1.0 should be returned.
+ * nothing, 0.0 should be returned.
*/
protected abstract double maintain();
- /** Convenience methods to convert attempts and failures into a success factor */
- protected final double asSuccessFactor(int attempts, int failures) {
+ /** Convenience methods to convert attempts and failures into a success factor, and return */
+ protected final double asSuccessFactorDeviation(int attempts, int failures) {
double factor = attempts == 0 ? 1.0 : 1 - (double)failures / attempts;
- return new BigDecimal(factor).setScale(2, RoundingMode.HALF_UP).doubleValue();
+ return new BigDecimal(factor - successFactorBaseline).setScale(2, RoundingMode.HALF_UP).doubleValue();
}
/** Returns the interval at which this job is set to run */
@@ -111,14 +117,14 @@ public abstract class Maintainer implements Runnable {
if (!force && !jobControl.isActive(name())) return;
log.log(Level.FINE, () -> "Running " + this.getClass().getSimpleName());
- double successFactor = 0;
+ double successFactorDeviation = 0;
long startTime = clock.millis();
try (var lock = jobControl.lockJob(name())) {
- successFactor = maintain();
+ successFactorDeviation = maintain();
}
catch (UncheckedTimeoutException e) {
if (ignoreCollision)
- successFactor = 1;
+ successFactorDeviation = 0;
else
log.log(Level.WARNING, this + " collided with another run. Will retry in " + interval);
}
@@ -127,7 +133,7 @@ public abstract class Maintainer implements Runnable {
}
finally {
long endTime = clock.millis();
- jobMetrics.completed(name(), successFactor, endTime - startTime);
+ jobMetrics.completed(name(), successFactorDeviation, endTime - startTime);
}
log.log(Level.FINE, () -> "Finished " + this.getClass().getSimpleName());
}
diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java
index f9230ab6df6..6acd0679da2 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java
@@ -166,81 +166,32 @@ public class SlimeUtils {
false);
}
- private static class Equal {
- protected final Inspector rhsInspector;
-
- protected boolean equal = true;
-
- public Equal(Inspector rhsInspector) { this.rhsInspector = rhsInspector; }
-
- public boolean isEqual() { return equal; }
- }
-
- private static class EqualArray extends Equal implements ArrayTraverser {
- public EqualArray(Inspector rhsInspector) { super(rhsInspector); }
-
- @Override
- public void entry(int idx, Inspector inspector) {
- if (equal) {
- equal = inspector.equalTo(rhsInspector.entry(idx));
- }
- }
- }
-
- private static class EqualObject extends Equal implements ObjectTraverser {
- public EqualObject(Inspector rhsInspector) { super(rhsInspector); }
-
- @Override
- public void field(String name, Inspector inspector) {
- if (equal) {
- equal = inspector.equalTo(rhsInspector.field(name));
- }
- }
- }
-
public static boolean equalTo(Inspector a, Inspector b) {
- boolean equal = a.type() == b.type();
-
- if (equal) {
- switch (a.type()) {
- case NIX:
- equal = a.valid() == b.valid();
- break;
- case BOOL:
- equal = a.asBool() == b.asBool();
- break;
- case LONG:
- equal = a.asLong() == b.asLong();
- break;
- case DOUBLE:
- equal = Double.compare(a.asDouble(), b.asDouble()) == 0;
- break;
- case STRING:
- equal = a.asString().equals(b.asString());
- break;
- case DATA:
- equal = Arrays.equals(a.asData(), b.asData());
- break;
- case ARRAY:
- {
- var traverser = new EqualArray(b);
- a.traverse(traverser);
- equal = traverser.isEqual() && (a.entries() == b.entries());
- }
- break;
- case OBJECT:
- {
- var traverser = new EqualObject(b);
- a.traverse(traverser);
- equal = traverser.isEqual() && (a.fields() == b.fields());
+ if (a.type() != b.type()) return false;
+
+ switch (a.type()) {
+ case NIX: return a.valid() == b.valid();
+ case BOOL: return a.asBool() == b.asBool();
+ case LONG: return a.asLong() == b.asLong();
+ case DOUBLE: return Double.compare(a.asDouble(), b.asDouble()) == 0;
+ case STRING: return a.asString().equals(b.asString());
+ case DATA: return Arrays.equals(a.asData(), b.asData());
+ case ARRAY: {
+ if (a.entries() != b.entries()) return false;
+ for (int i = 0; i < a.entries(); i++) {
+ if (!equalTo(a.entry(i), b.entry(i))) return false;
}
- break;
- default:
- assert(false);
- break;
+ return true;
}
+ case OBJECT: {
+ if (a.fields() != b.fields()) return false;
+ boolean[] equal = new boolean[]{ true };
+ a.traverse((String key, Inspector value) -> {
+ if (equal[0] && !equalTo(value, b.field(key))) equal[0] = false;
+ });
+ return equal[0];
+ }
+ default: throw new IllegalStateException("Unexpected type: " + a.type());
}
-
- return equal;
}
}
diff --git a/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java
index c9ff86e7c2e..730c3909b4c 100644
--- a/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java
@@ -2,11 +2,11 @@
package com.yahoo.slime;
import org.junit.Test;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.sameInstance;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertNotSame;
import java.util.List;
import java.util.ArrayList;
@@ -20,17 +20,17 @@ public class ArrayValueTestCase {
@Test
public void testSymbolTableForwarding() {
SymbolTable names = new SymbolTable();
- assertThat(names.symbols(), is(0));
+ assertEquals(0, names.symbols());
new ArrayValue(names).addArray().addObject().setLong("foo", 3);
- assertThat(names.symbols(), is(1));
+ assertEquals(1, names.symbols());
}
@Test
public void testOutOfBoundsAccess() {
var array = makeArray();
array.addBool(true);
- assertThat(array.entry(-1).valid(), is(false));
- assertThat(array.entry(1).valid(), is(false));
+ assertFalse(array.entry(-1).valid());
+ assertFalse(array.entry(1).valid());
}
@Test
@@ -44,8 +44,8 @@ public class ArrayValueTestCase {
var e1 = array.entry(i);
var e2 = array.entry(i);
var e3 = added.get(i);
- assertThat(e1, sameInstance(e2));
- assertThat(e1, sameInstance(e3));
+ assertSame(e2, e1);
+ assertSame(e3, e1);
}
}
@@ -61,12 +61,12 @@ public class ArrayValueTestCase {
var e1 = array.entry(i);
var e2 = array.entry(i);
var e3 = added.get(i);
- assertThat(e1, not(sameInstance(e2)));
- assertThat(e1, not(sameInstance(e3)));
- assertThat(e1.equalTo(e2), is(true));
- assertThat(e1.equalTo(e3), is(true));
- assertThat(e1.type(), is(Type.LONG));
- assertThat(e1.asLong(), is(expect));
+ assertNotSame(e2, e1);
+ assertNotSame(e3, e1);
+ assertTrue(e1.equalTo(e2));
+ assertTrue(e1.equalTo(e3));
+ assertEquals(Type.LONG, e1.type());
+ assertEquals(expect, e1.asLong());
}
}
@@ -82,12 +82,12 @@ public class ArrayValueTestCase {
var e1 = array.entry(i);
var e2 = array.entry(i);
var e3 = added.get(i);
- assertThat(e1, not(sameInstance(e2)));
- assertThat(e1, not(sameInstance(e3)));
- assertThat(e1.equalTo(e2), is(true));
- assertThat(e1.equalTo(e3), is(true));
- assertThat(e1.type(), is(Type.DOUBLE));
- assertThat(e1.asDouble(), is(expect));
+ assertNotSame(e2, e1);
+ assertNotSame(e3, e1);
+ assertTrue(e1.equalTo(e2));
+ assertTrue(e1.equalTo(e3));
+ assertEquals(Type.DOUBLE, e1.type());
+ assertEquals(expect, e1.asDouble(), 0.0);
}
}
@@ -109,20 +109,20 @@ public class ArrayValueTestCase {
case ARRAY: added.add(array.addArray()); break;
case OBJECT: added.add(array.addObject()); break;
}
- assertThat(array.entries(), is(65));
- assertThat(array.entry(64).type(), is(type));
- assertThat(added.get(64), sameInstance(array.entry(64)));
+ assertEquals(65, array.entries());
+ assertEquals(type, array.entry(64).type());
+ assertSame(array.entry(64), added.get(64));
for (int i = 0; i < 64; ++i) {
var e1 = array.entry(i);
var e2 = array.entry(i);
var e3 = added.get(i);
long expect = i;
- assertThat(e1, sameInstance(e2));
- assertThat(e1, not(sameInstance(e3)));
- assertThat(e1.equalTo(e2), is(true));
- assertThat(e1.equalTo(e3), is(true));
- assertThat(e1.type(), is(Type.LONG));
- assertThat(e1.asLong(), is(expect));
+ assertSame(e2, e1);
+ assertNotSame(e3, e1);
+ assertTrue(e1.equalTo(e2));
+ assertTrue(e1.equalTo(e3));
+ assertEquals(Type.LONG, e1.type());
+ assertEquals(expect, e1.asLong());
}
}
}
@@ -146,20 +146,20 @@ public class ArrayValueTestCase {
case ARRAY: added.add(array.addArray()); break;
case OBJECT: added.add(array.addObject()); break;
}
- assertThat(array.entries(), is(65));
- assertThat(array.entry(64).type(), is(type));
- assertThat(added.get(64), sameInstance(array.entry(64)));
+ assertEquals(65, array.entries());
+ assertEquals(type, array.entry(64).type());
+ assertSame(array.entry(64), added.get(64));
for (int i = 0; i < 64; ++i) {
var e1 = array.entry(i);
var e2 = array.entry(i);
var e3 = added.get(i);
double expect = i;
- assertThat(e1, sameInstance(e2));
- assertThat(e1, not(sameInstance(e3)));
- assertThat(e1.equalTo(e2), is(true));
- assertThat(e1.equalTo(e3), is(true));
- assertThat(e1.type(), is(Type.DOUBLE));
- assertThat(e1.asDouble(), is(expect));
+ assertSame(e2, e1);
+ assertNotSame(e3, e1);
+ assertTrue(e1.equalTo(e2));
+ assertTrue(e1.equalTo(e3));
+ assertEquals(Type.DOUBLE, e1.type());
+ assertEquals(expect, e1.asDouble(), 0.0);
}
}
}
@@ -179,9 +179,9 @@ public class ArrayValueTestCase {
case ARRAY: added = array.addArray(); break;
case OBJECT: added = array.addObject(); break;
}
- assertThat(array.entries(), is(1));
- assertThat(array.entry(0).type(), is(type));
- assertThat(added, sameInstance(array.entry(0)));
+ assertEquals(1, array.entries());
+ assertEquals(type, array.entry(0).type());
+ assertSame(array.entry(0), added);
}
}
}
diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java
index db001a9276b..5f96247b77e 100644
--- a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java
@@ -12,9 +12,10 @@ import static com.yahoo.slime.BinaryFormat.decode_zigzag;
import static com.yahoo.slime.BinaryFormat.encode_double;
import static com.yahoo.slime.BinaryFormat.encode_type_and_meta;
import static com.yahoo.slime.BinaryFormat.encode_zigzag;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
public class BinaryFormatTestCase {
@@ -36,50 +37,50 @@ public class BinaryFormatTestCase {
BinaryEncoder bof = new BinaryEncoder(output);
bof.encode_cmpr_int(value);
byte[] actual = output.toArray();
- assertThat(actual, is(expect));
+ assertArrayEquals(expect, actual);
BinaryDecoder bif = new BinaryDecoder();
bif.in = new BufferedInput(expect);
int got = bif.in.read_cmpr_int();
- assertThat(got, is(value));
- assertThat(bif.in.failed(), is(false));
+ assertEquals(value, got);
+ assertFalse(bif.in.failed());
bif = new BinaryDecoder();
bif.in = new BufferedInput(expect);
got = bif.in.skip_cmpr_int();
- assertThat(got, is(expect.length - 1));
- assertThat(bif.in.getPosition(), is(expect.length));
- assertThat(bif.in.failed(), is(false));
+ assertEquals(expect.length - 1, got);
+ assertEquals(expect.length, bif.in.getPosition());
+ assertFalse(bif.in.failed());
- assertThat(BinaryView.peek_cmpr_int_for_testing(expect, 0), is(value));
- assertThat(BinaryView.skip_cmpr_int_for_testing(expect, 0), is(expect.length));
+ assertEquals(value, BinaryView.peek_cmpr_int_for_testing(expect, 0));
+ assertEquals(expect.length, BinaryView.skip_cmpr_int_for_testing(expect, 0));
}
void verify_read_cmpr_int_fails(byte[] data) {
BinaryDecoder bif = new BinaryDecoder();
bif.in = new BufferedInput(data);
int got = bif.in.read_cmpr_int();
- assertThat(got, is(0));
- assertThat(bif.in.failed(), is(true));
+ assertEquals(0, got);
+ assertTrue(bif.in.failed());
bif = new BinaryDecoder();
bif.in = new BufferedInput(data);
got = bif.in.skip_cmpr_int();
- assertThat(got, is(data.length - 1));
- assertThat(bif.in.getPosition(), is(data.length));
- assertThat(bif.in.failed(), is(false));
+ assertEquals(data.length - 1, got);
+ assertEquals(data.length, bif.in.getPosition());
+ assertFalse(bif.in.failed());
- assertThat(BinaryView.skip_cmpr_int_for_testing(data, 0), is(data.length));
+ assertEquals(data.length, BinaryView.skip_cmpr_int_for_testing(data, 0));
}
// was verifyBasic
void verifyEncoding(Slime slime, byte[] expect) {
- assertThat(BinaryFormat.encode(slime), is(expect));
- assertThat(slime.get().equalTo(BinaryView.inspect(expect)), is(true));
+ assertArrayEquals(expect, BinaryFormat.encode(slime));
+ assertTrue(slime.get().equalTo(BinaryView.inspect(expect)));
Compressor compressor = new Compressor(CompressionType.LZ4, 3, 2, 0);
Compressor.Compression result = BinaryFormat.encode_and_compress(slime, compressor);
byte [] decompressed = compressor.decompress(result);
- assertThat(decompressed, is(expect));
+ assertArrayEquals(expect, decompressed);
verifyMultiEncode(expect);
}
@@ -90,65 +91,65 @@ public class BinaryFormatTestCase {
for (int i = 0; i < 5; ++i) {
Slime slime = BinaryFormat.decode(buffers[i]);
buffers[i+1] = BinaryFormat.encode(slime);
- assertThat(buffers[i+1], is(expect));
+ assertArrayEquals(expect, buffers[i+1]);
}
}
@Test
public void testZigZagConversion() {
- assertThat(encode_zigzag(0), is(0L));
- assertThat(decode_zigzag(encode_zigzag(0)), is(0L));
+ assertEquals(0L, encode_zigzag(0));
+ assertEquals(0L, decode_zigzag(encode_zigzag(0)));
- assertThat(encode_zigzag(-1), is(1L));
- assertThat(decode_zigzag(encode_zigzag(-1)), is(-1L));
+ assertEquals(1L, encode_zigzag(-1));
+ assertEquals(-1L, decode_zigzag(encode_zigzag(-1)));
- assertThat(encode_zigzag(1), is(2L));
- assertThat(decode_zigzag(encode_zigzag(1)), is(1L));
+ assertEquals(2L, encode_zigzag(1));
+ assertEquals(1L, decode_zigzag(encode_zigzag(1)));
- assertThat(encode_zigzag(-2), is(3L));
- assertThat(decode_zigzag(encode_zigzag(-2)), is(-2L));
+ assertEquals(3L, encode_zigzag(-2));
+ assertEquals(-2L, decode_zigzag(encode_zigzag(-2)));
- assertThat(encode_zigzag(2), is(4L));
- assertThat(decode_zigzag(encode_zigzag(2)), is(2L));
+ assertEquals(4L, encode_zigzag(2));
+ assertEquals(2L, decode_zigzag(encode_zigzag(2)));
- assertThat(encode_zigzag(-1000), is(1999L));
- assertThat(decode_zigzag(encode_zigzag(-1000)), is(-1000L));
+ assertEquals(1999L, encode_zigzag(-1000));
+ assertEquals(-1000L, decode_zigzag(encode_zigzag(-1000)));
- assertThat(encode_zigzag(1000), is(2000L));
- assertThat(decode_zigzag(encode_zigzag(1000)), is(1000L));
+ assertEquals(2000L, encode_zigzag(1000));
+ assertEquals(1000L, decode_zigzag(encode_zigzag(1000)));
- assertThat(encode_zigzag(-0x8000000000000000L), is(-1L));
- assertThat(decode_zigzag(encode_zigzag(-0x8000000000000000L)), is(-0x8000000000000000L));
+ assertEquals(-1L, encode_zigzag(-0x8000000000000000L));
+ assertEquals(-0x8000000000000000L, decode_zigzag(encode_zigzag(-0x8000000000000000L)));
- assertThat(encode_zigzag(0x7fffffffffffffffL), is(-2L));
- assertThat(decode_zigzag(encode_zigzag(0x7fffffffffffffffL)), is(0x7fffffffffffffffL));
+ assertEquals(-2L, encode_zigzag(0x7fffffffffffffffL));
+ assertEquals(0x7fffffffffffffffL, decode_zigzag(encode_zigzag(0x7fffffffffffffffL)));
}
@Test
public void testDoubleConversion() {
- assertThat(encode_double(0.0), is(0L));
- assertThat(decode_double(encode_double(0.0)), is(0.0));
+ assertEquals(0L, encode_double(0.0));
+ assertEquals(0.0, decode_double(encode_double(0.0)), 0.0);
- assertThat(encode_double(1.0), is(0x3ff0000000000000L));
- assertThat(decode_double(encode_double(1.0)), is(1.0));
+ assertEquals(0x3ff0000000000000L, encode_double(1.0));
+ assertEquals(1.0, decode_double(encode_double(1.0)), 0.0);
- assertThat(encode_double(-1.0), is(0xbff0000000000000L));
- assertThat(decode_double(encode_double(-1.0)), is(-1.0));
+ assertEquals(0xbff0000000000000L, encode_double(-1.0));
+ assertEquals(-1.0, decode_double(encode_double(-1.0)), 0.0);
- assertThat(encode_double(2.0), is(0x4000000000000000L));
- assertThat(decode_double(encode_double(2.0)), is(2.0));
+ assertEquals(0x4000000000000000L, encode_double(2.0));
+ assertEquals(2.0, decode_double(encode_double(2.0)), 0.0);
- assertThat(encode_double(-2.0), is(0xc000000000000000L));
- assertThat(decode_double(encode_double(-2.0)), is(-2.0));
+ assertEquals(0xc000000000000000L, encode_double(-2.0));
+ assertEquals(-2.0, decode_double(encode_double(-2.0)), 0.0);
- assertThat(encode_double(-0.0), is(0x8000000000000000L));
- assertThat(decode_double(encode_double(-0.0)), is(-0.0));
+ assertEquals(0x8000000000000000L, encode_double(-0.0));
+ assertEquals(-0.0, decode_double(encode_double(-0.0)), 0.0);
- assertThat(encode_double(3.5), is(0x400c000000000000L));
- assertThat(decode_double(encode_double(3.5)), is(3.5));
+ assertEquals(0x400c000000000000L, encode_double(3.5));
+ assertEquals(3.5, decode_double(encode_double(3.5)), 0.0);
- assertThat(encode_double(65535.875), is(0x40EFFFFC00000000L));
- assertThat(decode_double(encode_double(65535.875)), is(65535.875));
+ assertEquals(0x40EFFFFC00000000L, encode_double(65535.875));
+ assertEquals(65535.875, decode_double(encode_double(65535.875)), 0.0);
}
@Test
@@ -156,8 +157,8 @@ public class BinaryFormatTestCase {
for (byte type = 0; type < TYPE_LIMIT; ++type) {
for (int meta = 0; meta < META_LIMIT; ++meta) {
byte mangled = encode_type_and_meta(type, meta);
- assertThat(decode_type(mangled).ID, is(type));
- assertThat(decode_meta(mangled), is(meta));
+ assertEquals(type, decode_type(mangled).ID);
+ assertEquals(meta, decode_meta(mangled));
}
}
}
@@ -241,7 +242,7 @@ public class BinaryFormatTestCase {
BinaryEncoder encoder = new BinaryEncoder(actual);
encoder.write_type_and_size(type, size);
}
- assertThat(actual.toArray(), is(expect.toArray()));
+ assertArrayEquals(expect.toArray(), actual.toArray());
byte[] got = expect.toArray();
BinaryDecoder bif = new BinaryDecoder();
@@ -249,12 +250,12 @@ public class BinaryFormatTestCase {
byte b = bif.in.getByte();
Type decodedType = decode_type(b);
int decodedSize = bif.in.read_size(decode_meta(b));
- assertThat(decodedType.ID, is(type));
- assertThat(decodedSize, is(size));
- assertThat(bif.in.getConsumedSize(), is(got.length));
- assertThat(bif.in.failed(), is(false));
+ assertEquals(type, decodedType.ID);
+ assertEquals(size, decodedSize);
+ assertEquals(got.length, bif.in.getConsumedSize());
+ assertFalse(bif.in.failed());
- assertThat(BinaryView.extract_children_for_testing(got, 0), is(size));
+ assertEquals(size, BinaryView.extract_children_for_testing(got, 0));
}
}
@@ -292,21 +293,21 @@ public class BinaryFormatTestCase {
bof.write_type_and_bytes_le(type, bits);
}
byte[] actual = output.toArray();
- assertThat(actual, is(expect));
+ assertArrayEquals(expect, actual);
// test input:
BinaryDecoder bif = new BinaryDecoder();
bif.in = new BufferedInput(expect);
int size = decode_meta(bif.in.getByte());
long decodedBits = (hi != 0) ? bif.read_bytes_be(size) : bif.read_bytes_le(size);
- assertThat(decodedBits, is(bits));
- assertThat(bif.in.getConsumedSize(), is(expect.length));
- assertThat(bif.in.failed(), is(false));
+ assertEquals(bits, decodedBits);
+ assertEquals(expect.length, bif.in.getConsumedSize());
+ assertFalse(bif.in.failed());
if (hi != 0) {
- assertThat(encode_double(BinaryView.extract_double_for_testing(expect, 0)), is(bits));
+ assertEquals(bits, encode_double(BinaryView.extract_double_for_testing(expect, 0)));
} else {
- assertThat(encode_zigzag(BinaryView.extract_long_for_testing(expect, 0)), is(bits));
+ assertEquals(bits, encode_zigzag(BinaryView.extract_long_for_testing(expect, 0)));
}
}
}
@@ -322,7 +323,7 @@ public class BinaryFormatTestCase {
expect.put((byte)0); // nix
byte[] actual = BinaryFormat.encode(slime);
- assertThat(actual, is(expect.toArray()));
+ assertArrayEquals(expect.toArray(), actual);
verifyMultiEncode(expect.toArray());
}
@@ -428,7 +429,7 @@ public class BinaryFormatTestCase {
System.arraycopy(expect, 0, overlappingBuffer, 1, expect.length);
overlappingBuffer[overlappingBuffer.length - 1] = 0;
Slime copy = BinaryFormat.decode(overlappingBuffer, 1, expect.length);
- assertThat(BinaryFormat.encode(slime), is(BinaryFormat.encode(copy)));
+ assertArrayEquals(BinaryFormat.encode(copy), BinaryFormat.encode(slime));
}
@Test
@@ -550,18 +551,17 @@ public class BinaryFormatTestCase {
BinaryDecoder decoder = new BinaryDecoder();
Slime slime = decoder.decode(data);
int consumed = decoder.in.getConsumedSize();
- assertThat(consumed, is(data.length));
+ assertEquals(data.length, consumed);
Cursor c = slime.get();
- assertThat(c.valid(), is(true));
- assertThat(c.type(), is(Type.OBJECT));
- assertThat(c.children(), is(5));
- assertThat(c.field("b").asBool(), is(true));
- assertThat(c.field("c").asLong(), is(5L));
- assertThat(c.field("d").asDouble(), is(3.5));
- assertThat(c.field("e").asString(), is("string"));
+ assertTrue(c.valid());
+ assertEquals(Type.OBJECT, c.type());
+ assertEquals(5, c.children());
+ assertTrue(c.field("b").asBool());
+ assertEquals(5L, c.field("c").asLong());
+ assertEquals(3.5, c.field("d").asDouble(), 0.0);
+ assertEquals("string", c.field("e").asString());
byte[] expd = { 'd', 'a', 't', 'a' };
- assertThat(c.field("f").asData(), is(expd));
- assertThat(c.entry(5).valid(), is(false)); // not ARRAY
+ assertArrayEquals(expd, c.field("f").asData());
+ assertFalse(c.entry(5).valid()); // not ARRAY
}
-
}
diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java
index 568124369d4..920a25b96c9 100644
--- a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java
+++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java
@@ -140,99 +140,43 @@ public class BinaryViewTest {
}
}
- class MyVisitor implements Visitor {
- enum Called { NONE, INVALID, NIX, BOOL, LONG, DOUBLE, UTF8, DATA, ARRAY, OBJECT }
- Called called = Called.NONE;
- boolean boolValue;
- long longValue;
- double doubleValue;
- byte[] bytes;
- Inspector stuff;
- @Override public void visitInvalid() {
- assertEquals(ctx, Called.NONE, called);
- called = Called.INVALID;
- }
- @Override public void visitNix() {
- assertEquals(ctx, Called.NONE, called);
- called = Called.NIX;
- }
- @Override public void visitBool(boolean bit) {
- assertEquals(ctx, Called.NONE, called);
- called = Called.BOOL;
- boolValue = bit;
- }
- @Override public void visitLong(long l) {
- assertEquals(ctx, Called.NONE, called);
- called = Called.LONG;
- longValue = l;
- }
- @Override public void visitDouble(double d) {
- assertEquals(ctx, Called.NONE, called);
- called = Called.DOUBLE;
- doubleValue = d;
- }
- @Override public void visitString(String str) {
- fail(ctx + ", strings are never utf-16 in binary view");
- }
- @Override public void visitString(byte[] utf8) {
- assertEquals(ctx, Called.NONE, called);
- called = Called.UTF8;
- bytes = utf8;
- }
- @Override public void visitData(byte[] data) {
- assertEquals(ctx, Called.NONE, called);
- called = Called.DATA;
- bytes = data;
- }
- @Override public void visitArray(Inspector arr) {
- assertEquals(ctx, Called.NONE, called);
- called = Called.ARRAY;
- stuff = arr;
- }
- @Override public void visitObject(Inspector obj) {
- assertEquals(ctx, Called.NONE, called);
- called = Called.OBJECT;
- stuff = obj;
- }
- };
-
void checkVisitor(Inspector view) {
- var visitor = new MyVisitor();
+ var visitor = new MockVisitor(ctx);
view.accept(visitor);
if (!view.valid()) {
- assertEquals(ctx, MyVisitor.Called.INVALID, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.INVALID, visitor.called);
return;
}
switch (view.type()) {
case NIX:
- assertEquals(ctx, MyVisitor.Called.NIX, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.NIX, visitor.called);
break;
case BOOL:
- assertEquals(ctx, MyVisitor.Called.BOOL, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.BOOL, visitor.called);
assertEquals(ctx, view.asBool(), visitor.boolValue);
break;
case LONG:
- assertEquals(ctx, MyVisitor.Called.LONG, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.LONG, visitor.called);
assertEquals(ctx, view.asLong(), visitor.longValue);
break;
case DOUBLE:
- assertEquals(ctx, MyVisitor.Called.DOUBLE, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.DOUBLE, visitor.called);
assertEquals(ctx, view.asDouble(), visitor.doubleValue, 0.0);
break;
case STRING:
- assertEquals(ctx, MyVisitor.Called.UTF8, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.UTF8, visitor.called);
assertArrayEquals(ctx, view.asUtf8(), visitor.bytes);
break;
case DATA:
- assertEquals(ctx, MyVisitor.Called.DATA, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.DATA, visitor.called);
assertArrayEquals(ctx, view.asData(), visitor.bytes);
break;
case ARRAY:
- assertEquals(ctx, MyVisitor.Called.ARRAY, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.ARRAY, visitor.called);
assertSame(ctx, view, visitor.stuff);
break;
case OBJECT:
- assertEquals(ctx, MyVisitor.Called.OBJECT, visitor.called);
+ assertEquals(ctx, MockVisitor.Called.OBJECT, visitor.called);
assertSame(ctx, view, visitor.stuff);
break;
default:
diff --git a/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java b/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java
new file mode 100644
index 00000000000..a4354adbcd0
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java
@@ -0,0 +1,69 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.slime;
+
+import static org.junit.Assert.assertEquals;
+
+class MockVisitor implements Visitor {
+ enum Called { NONE, INVALID, NIX, BOOL, LONG, DOUBLE, STRING, UTF8, DATA, ARRAY, OBJECT }
+ Called called = Called.NONE;
+ boolean boolValue;
+ long longValue;
+ double doubleValue;
+ String string;
+ byte[] bytes;
+ Inspector stuff;
+ String ctx;
+
+ MockVisitor(String context) {
+ ctx = context;
+ }
+ MockVisitor() { this(""); }
+ @Override public void visitInvalid() {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.INVALID;
+ }
+ @Override public void visitNix() {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.NIX;
+ }
+ @Override public void visitBool(boolean bit) {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.BOOL;
+ boolValue = bit;
+ }
+ @Override public void visitLong(long l) {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.LONG;
+ longValue = l;
+ }
+ @Override public void visitDouble(double d) {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.DOUBLE;
+ doubleValue = d;
+ }
+ @Override public void visitString(String str) {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.STRING;
+ string = str;
+ }
+ @Override public void visitString(byte[] utf8) {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.UTF8;
+ bytes = utf8;
+ }
+ @Override public void visitData(byte[] data) {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.DATA;
+ bytes = data;
+ }
+ @Override public void visitArray(Inspector arr) {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.ARRAY;
+ stuff = arr;
+ }
+ @Override public void visitObject(Inspector obj) {
+ assertEquals(ctx, Called.NONE, called);
+ called = Called.OBJECT;
+ stuff = obj;
+ }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java
index 6ee8fb6c7e2..0a5c69b2d73 100644
--- a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java
@@ -2,10 +2,11 @@
package com.yahoo.slime;
import org.junit.Test;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.sameInstance;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
public class SlimeTestCase {
@@ -21,25 +22,25 @@ public class SlimeTestCase {
public void testTypeIds() {
System.out.println("testing type identifiers...");
- assertThat(Type.NIX.ID, is((byte)0));
- assertThat(Type.BOOL.ID, is((byte)1));
- assertThat(Type.LONG.ID, is((byte)2));
- assertThat(Type.DOUBLE.ID, is((byte)3));
- assertThat(Type.STRING.ID, is((byte)4));
- assertThat(Type.DATA.ID, is((byte)5));
- assertThat(Type.ARRAY.ID, is((byte)6));
- assertThat(Type.OBJECT.ID, is((byte)7));
+ assertEquals((byte)0, Type.NIX.ID);
+ assertEquals((byte)1, Type.BOOL.ID);
+ assertEquals((byte)2, Type.LONG.ID);
+ assertEquals((byte)3, Type.DOUBLE.ID);
+ assertEquals((byte)4, Type.STRING.ID);
+ assertEquals((byte)5, Type.DATA.ID);
+ assertEquals((byte)6, Type.ARRAY.ID);
+ assertEquals((byte)7, Type.OBJECT.ID);
- assertThat(Type.values().length, is(8));
+ assertEquals(8, Type.values().length);
- assertThat(Type.values()[0], sameInstance(Type.NIX));
- assertThat(Type.values()[1], sameInstance(Type.BOOL));
- assertThat(Type.values()[2], sameInstance(Type.LONG));
- assertThat(Type.values()[3], sameInstance(Type.DOUBLE));
- assertThat(Type.values()[4], sameInstance(Type.STRING));
- assertThat(Type.values()[5], sameInstance(Type.DATA));
- assertThat(Type.values()[6], sameInstance(Type.ARRAY));
- assertThat(Type.values()[7], sameInstance(Type.OBJECT));
+ assertSame(Type.NIX, Type.values()[0]);
+ assertSame(Type.BOOL, Type.values()[1]);
+ assertSame(Type.LONG, Type.values()[2]);
+ assertSame(Type.DOUBLE, Type.values()[3]);
+ assertSame(Type.STRING, Type.values()[4]);
+ assertSame(Type.DATA, Type.values()[5]);
+ assertSame(Type.ARRAY, Type.values()[6]);
+ assertSame(Type.OBJECT, Type.values()[7]);
}
@Test
@@ -50,41 +51,41 @@ public class SlimeTestCase {
for (int i = 0; i < 2; i++) {
if (i == 0) {
cur = slime.get();
- assertThat(cur.valid(), is(true));
+ assertTrue(cur.valid());
} else {
cur = NixValue.invalid();
- assertThat(cur.valid(), is(false));
+ assertFalse(cur.valid());
}
- assertThat(cur.type(), is(Type.NIX));
- assertThat(cur.children(), is(0));
- assertThat(cur.asBool(), is(false));
- assertThat(cur.asLong(), is((long)0));
- assertThat(cur.asDouble(), is(0.0));
- assertThat(cur.asString(), is(""));
- assertThat(cur.asData(), is(new byte[0]));
- assertThat(cur.entry(0).valid(), is(false));
- assertThat(cur.field(0).valid(), is(false));
- assertThat(cur.field("foo").valid(), is(false));
+ assertEquals(Type.NIX, cur.type());
+ assertEquals(0, cur.children());
+ assertFalse(cur.asBool());
+ assertEquals(0L, cur.asLong());
+ assertEquals(0.0, cur.asDouble(), 0.0);
+ assertEquals("", cur.asString());
+ assertArrayEquals(new byte[0], cur.asData());
+ assertFalse(cur.entry(0).valid());
+ assertFalse(cur.field(0).valid());
+ assertFalse(cur.field("foo").valid());
}
Inspector insp;
for (int i = 0; i < 2; i++) {
if (i == 0) {
insp = slime.get();
- assertThat(insp.valid(), is(true));
+ assertTrue(insp.valid());
} else {
insp = NixValue.invalid();
- assertThat(insp.valid(), is(false));
+ assertFalse(insp.valid());
}
- assertThat(insp.type(), is(Type.NIX));
- assertThat(insp.children(), is(0));
- assertThat(insp.asBool(), is(false));
- assertThat(insp.asLong(), is((long)0));
- assertThat(insp.asDouble(), is(0.0));
- assertThat(insp.asString(), is(""));
- assertThat(insp.asData(), is(new byte[0]));
- assertThat(insp.entry(0).valid(), is(false));
- assertThat(insp.field(0).valid(), is(false));
- assertThat(insp.field("foo").valid(), is(false));
+ assertEquals(Type.NIX, insp.type());
+ assertEquals(0, insp.children());
+ assertFalse(insp.asBool());
+ assertEquals(0L, insp.asLong());
+ assertEquals(0.0, insp.asDouble(), 0.0);
+ assertEquals("", insp.asString());
+ assertArrayEquals(new byte[0], insp.asData());
+ assertFalse(insp.entry(0).valid());
+ assertFalse(insp.field(0).valid());
+ assertFalse(insp.field("foo").valid());
}
}
@@ -96,63 +97,63 @@ public class SlimeTestCase {
System.out.println("testing boolean value");
slime.setBool(true);
Inspector insp = slime.get();
- assertThat(insp.valid(), is(true));
- assertThat(insp.type(), sameInstance(Type.BOOL));
- assertThat(insp.asBool(), is(true));
+ assertTrue(insp.valid());
+ assertSame(Type.BOOL, insp.type());
+ assertTrue(insp.asBool());
Cursor cur = slime.get();
- assertThat(cur.valid(), is(true));
- assertThat(cur.type(), sameInstance(Type.BOOL));
- assertThat(cur.asBool(), is(true));
+ assertTrue(cur.valid());
+ assertSame(Type.BOOL, cur.type());
+ assertTrue(cur.asBool());
System.out.println("testing long value");
slime.setLong(42);
cur = slime.get();
insp = slime.get();
- assertThat(cur.valid(), is(true));
- assertThat(insp.valid(), is(true));
- assertThat(cur.type(), sameInstance(Type.LONG));
- assertThat(insp.type(), sameInstance(Type.LONG));
- assertThat(cur.asLong(), is((long)42));
- assertThat(insp.asLong(), is((long)42));
+ assertTrue(cur.valid());
+ assertTrue(insp.valid());
+ assertSame(Type.LONG, cur.type());
+ assertSame(Type.LONG, insp.type());
+ assertEquals(42L, cur.asLong());
+ assertEquals(42L, insp.asLong());
System.out.println("testing double value");
slime.setDouble(4.2);
cur = slime.get();
insp = slime.get();
- assertThat(cur.valid(), is(true));
- assertThat(insp.valid(), is(true));
- assertThat(cur.type(), sameInstance(Type.DOUBLE));
- assertThat(insp.type(), sameInstance(Type.DOUBLE));
- assertThat(cur.asDouble(), is(4.2));
- assertThat(insp.asDouble(), is(4.2));
+ assertTrue(cur.valid());
+ assertTrue(insp.valid());
+ assertSame(Type.DOUBLE, cur.type());
+ assertSame(Type.DOUBLE, insp.type());
+ assertEquals(4.2, cur.asDouble(), 0.0);
+ assertEquals(4.2, insp.asDouble(), 0.0);
System.out.println("testing string value");
slime.setString("fortytwo");
cur = slime.get();
insp = slime.get();
- assertThat(cur.valid(), is(true));
- assertThat(insp.valid(), is(true));
- assertThat(cur.type(), sameInstance(Type.STRING));
- assertThat(insp.type(), sameInstance(Type.STRING));
- assertThat(cur.asString(), is("fortytwo"));
- assertThat(insp.asString(), is("fortytwo"));
+ assertTrue(cur.valid());
+ assertTrue(insp.valid());
+ assertSame(Type.STRING, cur.type());
+ assertSame(Type.STRING, insp.type());
+ assertEquals("fortytwo", cur.asString());
+ assertEquals("fortytwo", insp.asString());
System.out.println("testing data value");
byte[] data = { (byte)4, (byte)2 };
slime.setData(data);
cur = slime.get();
insp = slime.get();
- assertThat(cur.valid(), is(true));
- assertThat(insp.valid(), is(true));
- assertThat(cur.type(), sameInstance(Type.DATA));
- assertThat(insp.type(), sameInstance(Type.DATA));
- assertThat(cur.asData(), is(data));
- assertThat(insp.asData(), is(data));
+ assertTrue(cur.valid());
+ assertTrue(insp.valid());
+ assertSame(Type.DATA, cur.type());
+ assertSame(Type.DATA, insp.type());
+ assertArrayEquals(data, cur.asData());
+ assertArrayEquals(data, insp.asData());
data[0] = 10;
data[1] = 20;
byte[] data2 = { 10, 20 };
- assertThat(cur.asData(), is(data2));
- assertThat(insp.asData(), is(data2));
+ assertArrayEquals(data2, cur.asData());
+ assertArrayEquals(data2, insp.asData());
}
@Test
@@ -160,13 +161,13 @@ public class SlimeTestCase {
System.out.println("testing array values...");
Slime slime = new Slime();
Cursor c = slime.setArray();
- assertThat(c.valid(), is(true));
- assertThat(c.type(), is(Type.ARRAY));
- assertThat(c.children(), is(0));
+ assertTrue(c.valid());
+ assertEquals(Type.ARRAY, c.type());
+ assertEquals(0, c.children());
Inspector i = slime.get();
- assertThat(i.valid(), is(true));
- assertThat(i.type(), is(Type.ARRAY));
- assertThat(i.children(), is(0));
+ assertTrue(i.valid());
+ assertEquals(Type.ARRAY, i.type());
+ assertEquals(0, i.children());
c.addNix();
c.addBool(true);
c.addLong(5);
@@ -174,23 +175,23 @@ public class SlimeTestCase {
c.addString("string");
byte[] data = { (byte)'d', (byte)'a', (byte)'t', (byte)'a' };
c.addData(data);
- assertThat(c.children(), is(6));
- assertThat(c.entry(0).valid(), is(true));
- assertThat(c.entry(1).asBool(), is(true));
- assertThat(c.entry(2).asLong(), is((long)5));
- assertThat(c.entry(3).asDouble(), is(3.5));
- assertThat(c.entry(4).asString(), is("string"));
- assertThat(c.entry(5).asData(), is(data));
- assertThat(c.field(5).valid(), is(false)); // not OBJECT
+ assertEquals(6, c.children());
+ assertTrue(c.entry(0).valid());
+ assertTrue(c.entry(1).asBool());
+ assertEquals(5L, c.entry(2).asLong());
+ assertEquals(3.5, c.entry(3).asDouble(), 0.0);
+ assertEquals("string", c.entry(4).asString());
+ assertArrayEquals(data, c.entry(5).asData());
+ assertFalse(c.field(5).valid()); // not OBJECT
- assertThat(i.children(), is(6));
- assertThat(i.entry(0).valid(), is(true));
- assertThat(i.entry(1).asBool(), is(true));
- assertThat(i.entry(2).asLong(), is((long)5));
- assertThat(i.entry(3).asDouble(), is(3.5));
- assertThat(i.entry(4).asString(), is("string"));
- assertThat(i.entry(5).asData(), is(data));
- assertThat(i.field(5).valid(), is(false)); // not OBJECT
+ assertEquals(6, i.children());
+ assertTrue(i.entry(0).valid());
+ assertTrue(i.entry(1).asBool());
+ assertEquals(5L, i.entry(2).asLong());
+ assertEquals(3.5, i.entry(3).asDouble(), 0.0);
+ assertEquals("string", i.entry(4).asString());
+ assertArrayEquals(data, i.entry(5).asData());
+ assertFalse(i.field(5).valid()); // not OBJECT
}
@Test
@@ -199,13 +200,13 @@ public class SlimeTestCase {
Slime slime = new Slime();
Cursor c = slime.setObject();
- assertThat(c.valid(), is(true));
- assertThat(c.type(), is(Type.OBJECT));
- assertThat(c.children(), is(0));
+ assertTrue(c.valid());
+ assertEquals(Type.OBJECT, c.type());
+ assertEquals(0, c.children());
Inspector i = slime.get();
- assertThat(i.valid(), is(true));
- assertThat(i.type(), is(Type.OBJECT));
- assertThat(i.children(), is(0));
+ assertTrue(i.valid());
+ assertEquals(Type.OBJECT, i.type());
+ assertEquals(0, i.children());
c.setNix("a");
c.setBool("b", true);
@@ -215,23 +216,23 @@ public class SlimeTestCase {
byte[] data = { (byte)'d', (byte)'a', (byte)'t', (byte)'a' };
c.setData("f", data);
- assertThat(c.children(), is(6));
- assertThat(c.field("a").valid(), is(true));
- assertThat(c.field("b").asBool(), is(true));
- assertThat(c.field("c").asLong(), is((long)5));
- assertThat(c.field("d").asDouble(), is(3.5));
- assertThat(c.field("e").asString(), is("string"));
- assertThat(c.field("f").asData(), is(data));
- assertThat(c.entry(4).valid(), is(false)); // not ARRAY
+ assertEquals(6, c.children());
+ assertTrue(c.field("a").valid());
+ assertTrue(c.field("b").asBool());
+ assertEquals(5L, c.field("c").asLong());
+ assertEquals(3.5, c.field("d").asDouble(), 0.0);
+ assertEquals("string", c.field("e").asString());
+ assertArrayEquals(data, c.field("f").asData());
+ assertFalse(c.entry(4).valid()); // not ARRAY
- assertThat(i.children(), is(6));
- assertThat(i.field("a").valid(), is(true));
- assertThat(i.field("b").asBool(), is(true));
- assertThat(i.field("c").asLong(), is((long)5));
- assertThat(i.field("d").asDouble(), is(3.5));
- assertThat(i.field("e").asString(), is("string"));
- assertThat(i.field("f").asData(), is(data));
- assertThat(i.entry(4).valid(), is(false)); // not ARRAY
+ assertEquals(6, i.children());
+ assertTrue(i.field("a").valid());
+ assertTrue(i.field("b").asBool());
+ assertEquals(5L, i.field("c").asLong());
+ assertEquals(3.5, i.field("d").asDouble(), 0.0);
+ assertEquals("string", i.field("e").asString());
+ assertArrayEquals(data, i.field("f").asData());
+ assertFalse(i.entry(4).valid()); // not ARRAY
}
@Test
@@ -240,12 +241,12 @@ public class SlimeTestCase {
{
Slime slime = new Slime();
Cursor c = slime.setArray();
- assertThat(c.addLong(5).asLong(), is((long)5));
+ assertEquals(5L, c.addLong(5).asLong());
}
{
Slime slime = new Slime();
Cursor c = slime.setObject();
- assertThat(c.setLong("a", 5).asLong(), is((long)5));
+ assertEquals(5L, c.setLong("a", 5).asLong());
}
}
@@ -256,10 +257,10 @@ public class SlimeTestCase {
Slime slime = new Slime();
Cursor c = slime.setLong(10);
Inspector i1 = c;
- assertThat(i1.asLong(), is((long)10));
+ assertEquals(10L, i1.asLong());
Inspector i2 = slime.get();
- assertThat(i2.asLong(), is((long)10));
+ assertEquals(10L, i2.asLong());
}
@Test
@@ -275,14 +276,14 @@ public class SlimeTestCase {
c3.setLong("answer", 42);
}
Inspector i = slime.get();
- assertThat(i.field("bar").asLong(), is((long)10));
- assertThat(i.field("foo").entry(0).asLong(), is((long)20));
- assertThat(i.field("foo").entry(1).field("answer").asLong(), is((long)42));
+ assertEquals(10L, i.field("bar").asLong());
+ assertEquals(20L, i.field("foo").entry(0).asLong());
+ assertEquals(42L, i.field("foo").entry(1).field("answer").asLong());
Cursor c = slime.get();
- assertThat(c.field("bar").asLong(), is((long)10));
- assertThat(c.field("foo").entry(0).asLong(), is((long)20));
- assertThat(c.field("foo").entry(1).field("answer").asLong(), is((long)42));
+ assertEquals(10L, c.field("bar").asLong());
+ assertEquals(20L, c.field("foo").entry(0).asLong());
+ assertEquals(42L, c.field("foo").entry(1).field("answer").asLong());
}
@Test
@@ -293,19 +294,19 @@ public class SlimeTestCase {
Cursor c = slime.setObject();
for (int i = 0; i < n; i++) {
String str = ("" + i + "_str_" + i);
- assertThat(slime.lookup(str), is(SymbolTable.INVALID));
- assertThat(c.field(str).type(), sameInstance(Type.NIX));
+ assertEquals(SymbolTable.INVALID, slime.lookup(str));
+ assertSame(Type.NIX, c.field(str).type());
switch (i % 2) {
- case 0: assertThat((int)c.setLong(str, i).asLong(), is(i)); break;
- case 1: assertThat(slime.insert(str), is(i)); break;
+ case 0: assertEquals(i, (int)c.setLong(str, i).asLong()); break;
+ case 1: assertEquals(i, slime.insert(str)); break;
}
}
for (int i = 0; i < n; i++) {
String str = ("" + i + "_str_" + i);
- assertThat(slime.lookup(str), is(i));
+ assertEquals(i, slime.lookup(str));
switch (i % 2) {
- case 0: assertThat((int)c.field(str).asLong(), is(i)); break;
- case 1: assertThat((int)c.field(str).asLong(), is(0)); break;
+ case 0: assertEquals(i, (int)c.field(str).asLong()); break;
+ case 1: assertEquals(0, (int)c.field(str).asLong()); break;
}
}
}
@@ -317,12 +318,12 @@ public class SlimeTestCase {
Slime slime = new Slime();
Cursor c = slime.setArray();
for (int i = 0; i < n; i++) {
- assertThat((int)c.addLong(i).asLong(), is(i));
+ assertEquals(i, (int)c.addLong(i).asLong());
}
for (int i = 0; i < n; i++) {
- assertThat((int)c.entry(i).asLong(), is(i));
+ assertEquals(i, (int)c.entry(i).asLong());
}
- assertThat((int)c.entry(n).asLong(), is(0));
+ assertEquals(0, (int)c.entry(n).asLong());
}
@Test
@@ -332,8 +333,8 @@ public class SlimeTestCase {
c1.addLong(20);
Cursor c2 = c1.addObject();
c2.setLong("answer", 42);
- assertThat(slime.get().toString(), is("[20,{\"answer\":42}]"));
+ assertEquals("[20,{\"answer\":42}]", slime.get().toString());
c1.addString("\u2008");
- assertThat(slime.get().toString(), is("[20,{\"answer\":42},\"\u2008\"]"));
+ assertEquals("[20,{\"answer\":42},\"\u2008\"]", slime.get().toString());
}
}
diff --git a/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java
index 916cddac56b..cc4bf5d16e2 100644
--- a/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java
@@ -2,100 +2,97 @@
package com.yahoo.slime;
import org.junit.Test;
-import org.mockito.Mockito;
-
-import static org.hamcrest.CoreMatchers.sameInstance;
-import static org.mockito.hamcrest.MockitoHamcrest.argThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertSame;
public class VisitorTestCase {
@Test
public void testVisitInvalid() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().get().field("invalid");
inspector.accept(visitor);
- Mockito.verify(visitor).visitInvalid();
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.INVALID, visitor.called);
}
@Test
public void testVisitNix() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().get();
inspector.accept(visitor);
- Mockito.verify(visitor).visitNix();
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.NIX, visitor.called);
}
@Test
public void testVisitBool() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().setBool(true);
inspector.accept(visitor);
- Mockito.verify(visitor).visitBool(true);
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.BOOL, visitor.called);
+ assertEquals(true, visitor.boolValue);
}
@Test
public void testVisitLong() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().setLong(123);
inspector.accept(visitor);
- Mockito.verify(visitor).visitLong(123);
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.LONG, visitor.called);
+ assertEquals(123, visitor.longValue);
}
@Test
public void testVisitDouble() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().setDouble(123.0);
inspector.accept(visitor);
- Mockito.verify(visitor).visitDouble(123.0);
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.DOUBLE, visitor.called);
+ assertEquals(123.0, visitor.doubleValue, 0.0);
}
@Test
public void testVisitStringUtf16() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().setString("abc");
inspector.accept(visitor);
- Mockito.verify(visitor).visitString("abc");
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.STRING, visitor.called);
+ assertEquals("abc", visitor.string);
}
@Test
public void testVisitStringUtf8() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().setString(new byte[] {65,66,67});
inspector.accept(visitor);
- Mockito.verify(visitor).visitString(new byte[] {65,66,67});
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.UTF8, visitor.called);
+ assertArrayEquals(new byte[] {65,66,67}, visitor.bytes);
}
@Test
public void testVisitData() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().setData(new byte[] {1,2,3});
inspector.accept(visitor);
- Mockito.verify(visitor).visitData(new byte[] {1,2,3});
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.DATA, visitor.called);
+ assertArrayEquals(new byte[] {1,2,3}, visitor.bytes);
}
@Test
public void testVisitArray() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().setArray();
inspector.accept(visitor);
- Mockito.verify(visitor).visitArray(argThat(sameInstance(inspector)));
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.ARRAY, visitor.called);
+ assertSame(inspector, visitor.stuff);
}
@Test
public void testVisitObject() {
- Visitor visitor = Mockito.mock(Visitor.class);
+ var visitor = new MockVisitor();
Inspector inspector = new Slime().setObject();
inspector.accept(visitor);
- Mockito.verify(visitor).visitObject(argThat(sameInstance(inspector)));
- Mockito.verifyNoMoreInteractions(visitor);
+ assertEquals(MockVisitor.Called.OBJECT, visitor.called);
+ assertSame(inspector, visitor.stuff);
}
}
diff --git a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp
index 44decb9bf91..31113f2b4f2 100644
--- a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp
+++ b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp
@@ -53,7 +53,7 @@ public:
~RealIntStore();
EntryRef add(uint32_t value) { return _store.addEntry(value); }
AtomicEntryRef add_relaxed(uint32_t value) { return AtomicEntryRef(add(value)); }
- void hold(const AtomicEntryRef& ref) { _store.holdElem(ref.load_relaxed(), 1); }
+ void hold(const AtomicEntryRef& ref) { _store.hold_entry(ref.load_relaxed()); }
EntryRef move(EntryRef ref);
void assign_generation(generation_t current_gen) { _store.assign_generation(current_gen); }
void reclaim_memory(generation_t gen) { _store.reclaim_memory(gen); }
diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp
index b8da9ea6042..afad3523fa3 100644
--- a/vespalib/src/tests/btree/btree_test.cpp
+++ b/vespalib/src/tests/btree/btree_test.cpp
@@ -1065,7 +1065,7 @@ adjustAllocatedBytes(size_t nodeCount, size_t nodeSize)
TEST_F(BTreeTest, require_that_memory_usage_is_calculated)
{
constexpr size_t BASE_ALLOCATED = 28744u;
- constexpr size_t BASE_USED = 24984;
+ constexpr size_t BASE_USED = 24936;
typedef BTreeNodeAllocator<int32_t, int8_t,
btree::NoAggregated,
MyTraits::INTERNAL_SLOTS, MyTraits::LEAF_SLOTS> NodeAllocator;
diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp
index ccc3ab88c31..1df03f6eb0a 100644
--- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp
+++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp
@@ -29,16 +29,16 @@ constexpr float ALLOC_GROW_FACTOR = 0.2;
}
-template <typename TestT, typename EntryT, typename RefT = EntryRefT<19> >
+template <typename TestT, typename ElemT, typename RefT = EntryRefT<19> >
struct ArrayStoreTest : public TestT
{
using EntryRefType = RefT;
- using ArrayStoreType = ArrayStore<EntryT, RefT>;
+ using ArrayStoreType = ArrayStore<ElemT, RefT>;
using LargeArray = typename ArrayStoreType::LargeArray;
using ConstArrayRef = typename ArrayStoreType::ConstArrayRef;
- using EntryVector = std::vector<EntryT>;
- using value_type = EntryT;
- using ReferenceStore = vespalib::hash_map<EntryRef, EntryVector>;
+ using ElemVector = std::vector<ElemT>;
+ using value_type = ElemT;
+ using ReferenceStore = vespalib::hash_map<EntryRef, ElemVector>;
AllocStats stats;
ArrayStoreType store;
@@ -61,11 +61,11 @@ struct ArrayStoreTest : public TestT
add_using_allocate(false)
{}
~ArrayStoreTest() override;
- void assertAdd(const EntryVector &input) {
+ void assertAdd(const ElemVector &input) {
EntryRef ref = add(input);
assertGet(ref, input);
}
- EntryRef add(const EntryVector &input) {
+ EntryRef add(const ElemVector &input) {
EntryRef result;
if (add_using_allocate) {
result = store.allocate(input.size());
@@ -82,16 +82,16 @@ struct ArrayStoreTest : public TestT
refStore.insert(std::make_pair(result, input));
return result;
}
- void assertGet(EntryRef ref, const EntryVector &exp) const {
+ void assertGet(EntryRef ref, const ElemVector &exp) const {
ConstArrayRef act = store.get(ref);
- EXPECT_EQ(exp, EntryVector(act.begin(), act.end()));
+ EXPECT_EQ(exp, ElemVector(act.begin(), act.end()));
}
void remove(EntryRef ref) {
ASSERT_EQ(1u, refStore.count(ref));
store.remove(ref);
refStore.erase(ref);
}
- void remove(const EntryVector &input) {
+ void remove(const ElemVector &input) {
remove(getEntryRef(input));
}
uint32_t getBufferId(EntryRef ref) const {
@@ -99,14 +99,14 @@ struct ArrayStoreTest : public TestT
}
void assertBufferState(EntryRef ref, const MemStats& expStats) {
EXPECT_EQ(expStats._used, store.bufferState(ref).size());
- EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_elems());
- EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_elems());
+ EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_entries());
+ EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_entries());
}
void assert_buffer_stats(EntryRef ref, const TestBufferStats& exp_stats) {
const auto& state = store.bufferState(ref);
EXPECT_EQ(exp_stats._used, state.size());
- EXPECT_EQ(exp_stats._hold, state.stats().hold_elems());
- EXPECT_EQ(exp_stats._dead, state.stats().dead_elems());
+ EXPECT_EQ(exp_stats._hold, state.stats().hold_entries());
+ EXPECT_EQ(exp_stats._dead, state.stats().dead_entries());
EXPECT_EQ(exp_stats._extra_used, state.stats().extra_used_bytes());
EXPECT_EQ(exp_stats._extra_hold, state.stats().extra_hold_bytes());
}
@@ -121,7 +121,7 @@ struct ArrayStoreTest : public TestT
assertGet(elem.first, elem.second);
}
}
- void assert_ref_reused(const EntryVector& first, const EntryVector& second, bool should_reuse) {
+ void assert_ref_reused(const ElemVector& first, const ElemVector& second, bool should_reuse) {
EntryRef ref1 = add(first);
remove(ref1);
reclaim_memory();
@@ -129,7 +129,7 @@ struct ArrayStoreTest : public TestT
EXPECT_EQ(should_reuse, (ref2 == ref1));
assertGet(ref2, second);
}
- EntryRef getEntryRef(const EntryVector &input) {
+ EntryRef getEntryRef(const ElemVector &input) {
for (auto itr = refStore.begin(); itr != refStore.end(); ++itr) {
if (itr->second == input) {
return itr->first;
@@ -160,12 +160,12 @@ struct ArrayStoreTest : public TestT
}
refStore = compactedRefStore;
}
- size_t entrySize() const { return sizeof(EntryT); }
+ size_t elem_size() const { return sizeof(ElemT); }
size_t largeArraySize() const { return sizeof(LargeArray); }
};
-template <typename TestT, typename EntryT, typename RefT>
-ArrayStoreTest<TestT, EntryT, RefT>::~ArrayStoreTest() = default;
+template <typename TestT, typename ElemT, typename RefT>
+ArrayStoreTest<TestT, ElemT, RefT>::~ArrayStoreTest() = default;
struct TestParam {
bool add_using_allocate;
@@ -214,8 +214,8 @@ TEST_P(NumberStoreTest, control_static_sizes) {
EXPECT_EQ(240u + sizeof_deque, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType));
EXPECT_EQ(104u, sizeof(NumberStoreTest::ArrayStoreType::SmallBufferType));
MemoryUsage usage = store.getMemoryUsage();
- EXPECT_EQ(202120u, usage.allocatedBytes());
- EXPECT_EQ(197752u, usage.usedBytes());
+ EXPECT_EQ(202116u, usage.allocatedBytes());
+ EXPECT_EQ(197656u, usage.usedBytes());
}
TEST_P(NumberStoreTest, add_and_get_small_arrays_of_trivial_type)
@@ -246,15 +246,15 @@ TEST_F(StringStoreTest, add_and_get_large_arrays_of_non_trivial_type)
assertAdd({"ddd", "eee", "ffff", "gggg", "hhhh"});
}
-TEST_P(NumberStoreTest, elements_are_put_on_hold_when_a_small_array_is_removed)
+TEST_P(NumberStoreTest, entries_are_put_on_hold_when_a_small_array_is_removed)
{
EntryRef ref = add({1,2,3});
- assertBufferState(ref, MemStats().used(3).hold(0));
+ assertBufferState(ref, MemStats().used(1).hold(0));
store.remove(ref);
- assertBufferState(ref, MemStats().used(3).hold(3));
+ assertBufferState(ref, MemStats().used(1).hold(1));
}
-TEST_P(NumberStoreTest, elements_are_put_on_hold_when_a_large_array_is_removed)
+TEST_P(NumberStoreTest, entries_are_put_on_hold_when_a_large_array_is_removed)
{
EntryRef ref = add({1,2,3,4});
// Note: The first buffer has the first element reserved -> we expect 2 elements used here.
@@ -319,7 +319,7 @@ test_compaction(NumberStoreBasicTest &f)
f.remove(f.add({5,5}));
f.reclaim_memory();
f.assertBufferState(size1Ref, MemStats().used(1).dead(0));
- f.assertBufferState(size2Ref, MemStats().used(4).dead(2));
+ f.assertBufferState(size2Ref, MemStats().used(2).dead(1));
f.assertBufferState(size3Ref, MemStats().used(2).dead(1)); // Note: First element is reserved
uint32_t size1BufferId = f.getBufferId(size1Ref);
uint32_t size2BufferId = f.getBufferId(size2Ref);
@@ -363,8 +363,8 @@ void testCompaction(NumberStoreTest &f, bool compactMemory, bool compactAddressS
f.remove(f.add({7}));
f.reclaim_memory();
f.assertBufferState(size1Ref, MemStats().used(3).dead(2));
- f.assertBufferState(size2Ref, MemStats().used(2).dead(0));
- f.assertBufferState(size3Ref, MemStats().used(6).dead(3));
+ f.assertBufferState(size2Ref, MemStats().used(1).dead(0));
+ f.assertBufferState(size3Ref, MemStats().used(2).dead(1));
uint32_t size1BufferId = f.getBufferId(size1Ref);
uint32_t size2BufferId = f.getBufferId(size2Ref);
uint32_t size3BufferId = f.getBufferId(size3Ref);
@@ -434,22 +434,22 @@ TEST_P(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_small_a
{
MemStats exp(store.getMemoryUsage());
add({1,2,3});
- assertMemoryUsage(exp.used(entrySize() * 3));
+ assertMemoryUsage(exp.used(elem_size() * 3));
remove({1,2,3});
- assertMemoryUsage(exp.hold(entrySize() * 3));
+ assertMemoryUsage(exp.hold(elem_size() * 3));
reclaim_memory();
- assertMemoryUsage(exp.holdToDead(entrySize() * 3));
+ assertMemoryUsage(exp.holdToDead(elem_size() * 3));
}
TEST_P(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_large_arrays)
{
MemStats exp(store.getMemoryUsage());
add({1,2,3,4});
- assertMemoryUsage(exp.used(largeArraySize() + entrySize() * 4));
+ assertMemoryUsage(exp.used(largeArraySize() + elem_size() * 4));
remove({1,2,3,4});
- assertMemoryUsage(exp.hold(largeArraySize() + entrySize() * 4));
+ assertMemoryUsage(exp.hold(largeArraySize() + elem_size() * 4));
reclaim_memory();
- assertMemoryUsage(exp.decUsed(entrySize() * 4).decHold(largeArraySize() + entrySize() * 4).
+ assertMemoryUsage(exp.decUsed(elem_size() * 4).decHold(largeArraySize() + elem_size() * 4).
dead(largeArraySize()));
}
diff --git a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp
index fcf1acd5cd3..171e7216638 100644
--- a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp
+++ b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp
@@ -22,32 +22,32 @@ struct Fixture
Fixture(uint32_t maxSmallArrayTypeId,
size_t hugePageSize,
size_t smallPageSize,
- size_t minNumArraysForNewBuffer)
+ size_t min_num_entries_for_new_buffer)
: cfg(ArrayStoreConfig::optimizeForHugePage(maxSmallArrayTypeId,
[](size_t type_id) noexcept { return type_id; },
hugePageSize, smallPageSize,
sizeof(int), EntryRefType::offsetSize(),
- minNumArraysForNewBuffer,
+ min_num_entries_for_new_buffer,
ALLOC_GROW_FACTOR)) { }
- void assertSpec(uint32_t type_id, uint32_t numArraysForNewBuffer) {
+ void assertSpec(uint32_t type_id, uint32_t num_entries_for_new_buffer) {
assertSpec(type_id, AllocSpec(0, EntryRefType::offsetSize(),
- numArraysForNewBuffer, ALLOC_GROW_FACTOR));
+ num_entries_for_new_buffer, ALLOC_GROW_FACTOR));
}
void assertSpec(uint32_t type_id, const AllocSpec &expSpec) {
const auto& actSpec = cfg.spec_for_type_id(type_id);
- EXPECT_EQUAL(expSpec.minArraysInBuffer, actSpec.minArraysInBuffer);
- EXPECT_EQUAL(expSpec.maxArraysInBuffer, actSpec.maxArraysInBuffer);
- EXPECT_EQUAL(expSpec.numArraysForNewBuffer, actSpec.numArraysForNewBuffer);
+ EXPECT_EQUAL(expSpec.min_entries_in_buffer, actSpec.min_entries_in_buffer);
+ EXPECT_EQUAL(expSpec.max_entries_in_buffer, actSpec.max_entries_in_buffer);
+ EXPECT_EQUAL(expSpec.num_entries_for_new_buffer, actSpec.num_entries_for_new_buffer);
EXPECT_EQUAL(expSpec.allocGrowFactor, actSpec.allocGrowFactor);
}
};
AllocSpec
-makeSpec(size_t minArraysInBuffer,
- size_t maxArraysInBuffer,
- size_t numArraysForNewBuffer)
+makeSpec(size_t min_entries_in_buffer,
+ size_t max_entries_in_buffer,
+ size_t num_entries_for_new_buffer)
{
- return AllocSpec(minArraysInBuffer, maxArraysInBuffer, numArraysForNewBuffer, ALLOC_GROW_FACTOR);
+ return AllocSpec(min_entries_in_buffer, max_entries_in_buffer, num_entries_for_new_buffer, ALLOC_GROW_FACTOR);
}
constexpr size_t KB = 1024;
diff --git a/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp b/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp
index 09b2590a5f3..fec8d5949f8 100644
--- a/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp
+++ b/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp
@@ -9,10 +9,10 @@ using namespace vespalib::datastore;
TEST(BufferStatsTest, buffer_stats_to_memory_stats)
{
InternalBufferStats buf;
- buf.set_alloc_elems(17);
+ buf.set_alloc_entries(17);
buf.pushed_back(7);
- buf.set_dead_elems(5);
- buf.set_hold_elems(3);
+ buf.set_dead_entries(5);
+ buf.set_hold_entries(3);
buf.inc_extra_used_bytes(13);
buf.inc_extra_hold_bytes(11);
@@ -20,10 +20,10 @@ TEST(BufferStatsTest, buffer_stats_to_memory_stats)
constexpr size_t es = 8;
buf.add_to_mem_stats(es, mem);
- EXPECT_EQ(17, mem._allocElems);
- EXPECT_EQ(7, mem._usedElems);
- EXPECT_EQ(5, mem._deadElems);
- EXPECT_EQ(3, mem._holdElems);
+ EXPECT_EQ(17, mem._alloc_entries);
+ EXPECT_EQ(7, mem._used_entries);
+ EXPECT_EQ(5, mem._dead_entries);
+ EXPECT_EQ(3, mem._hold_entries);
EXPECT_EQ(17 * es + 13, mem._allocBytes);
EXPECT_EQ(7 * es + 13, mem._usedBytes);
EXPECT_EQ(5 * es, mem._deadBytes);
diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp
index de7d899e68a..9f7535a3676 100644
--- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp
+++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp
@@ -7,40 +7,40 @@ using namespace vespalib::datastore;
using IntBufferType = BufferType<int>;
constexpr uint32_t ARRAYS_SIZE(4);
-constexpr uint32_t MAX_ARRAYS(128);
-constexpr uint32_t NUM_ARRAYS_FOR_NEW_BUFFER(0);
+constexpr uint32_t MAX_ENTRIES(128);
+constexpr uint32_t NUM_ENTRIES_FOR_NEW_BUFFER(0);
struct Setup {
- uint32_t _minArrays;
- std::atomic<ElemCount> _usedElems;
- ElemCount _neededElems;
- std::atomic<ElemCount> _deadElems;
+ uint32_t _min_entries;
+ std::atomic<EntryCount> _used_entries;
+ EntryCount _needed_entries;
+ std::atomic<EntryCount> _dead_entries;
uint32_t _bufferId;
float _allocGrowFactor;
bool _resizing;
Setup()
- : _minArrays(0),
- _usedElems(0),
- _neededElems(0),
- _deadElems(0),
+ : _min_entries(0),
+ _used_entries(0),
+ _needed_entries(0),
+ _dead_entries(0),
_bufferId(1),
_allocGrowFactor(0.5),
_resizing(false)
{}
Setup(const Setup& rhs) noexcept;
- Setup &minArrays(uint32_t value) { _minArrays = value; return *this; }
- Setup &used(size_t value) { _usedElems = value; return *this; }
- Setup &needed(size_t value) { _neededElems = value; return *this; }
- Setup &dead(size_t value) { _deadElems = value; return *this; }
+ Setup &min_entries(uint32_t value) { _min_entries = value; return *this; }
+ Setup &used(size_t value) { _used_entries = value; return *this; }
+ Setup &needed(size_t value) { _needed_entries = value; return *this; }
+ Setup &dead(size_t value) { _dead_entries = value; return *this; }
Setup &bufferId(uint32_t value) { _bufferId = value; return *this; }
Setup &resizing(bool value) { _resizing = value; return *this; }
};
Setup::Setup(const Setup& rhs) noexcept
- : _minArrays(rhs._minArrays),
- _usedElems(rhs._usedElems.load(std::memory_order_relaxed)),
- _neededElems(rhs._neededElems),
- _deadElems(rhs._deadElems.load(std::memory_order_relaxed)),
+ : _min_entries(rhs._min_entries),
+ _used_entries(rhs._used_entries.load(std::memory_order_relaxed)),
+ _needed_entries(rhs._needed_entries),
+ _dead_entries(rhs._dead_entries.load(std::memory_order_relaxed)),
_bufferId(rhs._bufferId),
_allocGrowFactor(rhs._allocGrowFactor),
_resizing(rhs._resizing)
@@ -53,7 +53,7 @@ struct Fixture {
int buffer[ARRAYS_SIZE];
Fixture(const Setup &setup_)
: setups(),
- bufferType(ARRAYS_SIZE, setup_._minArrays, MAX_ARRAYS, NUM_ARRAYS_FOR_NEW_BUFFER, setup_._allocGrowFactor),
+ bufferType(ARRAYS_SIZE, setup_._min_entries, MAX_ENTRIES, NUM_ENTRIES_FOR_NEW_BUFFER, setup_._allocGrowFactor),
buffer()
{
setups.reserve(4);
@@ -61,121 +61,121 @@ struct Fixture {
}
~Fixture() {
for (auto& setup : setups) {
- bufferType.onHold(setup._bufferId, &setup._usedElems, &setup._deadElems);
- bufferType.onFree(setup._usedElems);
+ bufferType.on_hold(setup._bufferId, &setup._used_entries, &setup._dead_entries);
+ bufferType.on_free(setup._used_entries);
}
}
Setup& curr_setup() {
return setups.back();
}
void add_setup(const Setup& setup_in) {
- // The buffer type stores pointers to ElemCount (from Setup) and we must ensure these do not move in memory.
+ // The buffer type stores pointers to EntryCount (from Setup) and we must ensure these do not move in memory.
assert(setups.size() < setups.capacity());
setups.push_back(setup_in);
}
void onActive() {
- bufferType.onActive(curr_setup()._bufferId, &curr_setup()._usedElems, &curr_setup()._deadElems, &buffer[0]);
+ bufferType.on_active(curr_setup()._bufferId, &curr_setup()._used_entries, &curr_setup()._dead_entries, &buffer[0]);
}
- size_t arraysToAlloc() {
- return bufferType.calcArraysToAlloc(curr_setup()._bufferId, curr_setup()._neededElems, curr_setup()._resizing);
+ size_t entries_to_alloc() {
+ return bufferType.calc_entries_to_alloc(curr_setup()._bufferId, curr_setup()._needed_entries, curr_setup()._resizing);
}
- void assertArraysToAlloc(size_t exp) {
+ void assert_entries_to_alloc(size_t exp) {
onActive();
- EXPECT_EQUAL(exp, arraysToAlloc());
+ EXPECT_EQUAL(exp, entries_to_alloc());
}
};
void
-assertArraysToAlloc(size_t exp, const Setup &setup)
+assert_entries_to_alloc(size_t exp, const Setup &setup)
{
Fixture f(setup);
- f.assertArraysToAlloc(exp);
+ f.assert_entries_to_alloc(exp);
}
-TEST("require that complete arrays are allocated")
+TEST("require that entries are allocated")
{
- TEST_DO(assertArraysToAlloc(1, Setup().needed(1)));
- TEST_DO(assertArraysToAlloc(1, Setup().needed(2)));
- TEST_DO(assertArraysToAlloc(1, Setup().needed(3)));
- TEST_DO(assertArraysToAlloc(1, Setup().needed(4)));
- TEST_DO(assertArraysToAlloc(2, Setup().needed(5)));
+ TEST_DO(assert_entries_to_alloc(1, Setup().needed(1)));
+ TEST_DO(assert_entries_to_alloc(2, Setup().needed(2)));
+ TEST_DO(assert_entries_to_alloc(3, Setup().needed(3)));
+ TEST_DO(assert_entries_to_alloc(4, Setup().needed(4)));
+ TEST_DO(assert_entries_to_alloc(5, Setup().needed(5)));
}
-TEST("require that reserved elements are taken into account when not resizing")
+TEST("require that reserved entries are taken into account when not resizing")
{
- TEST_DO(assertArraysToAlloc(2, Setup().needed(1).bufferId(0)));
- TEST_DO(assertArraysToAlloc(2, Setup().needed(4).bufferId(0)));
- TEST_DO(assertArraysToAlloc(3, Setup().needed(5).bufferId(0)));
+ TEST_DO(assert_entries_to_alloc(2, Setup().needed(1).bufferId(0)));
+ TEST_DO(assert_entries_to_alloc(5, Setup().needed(4).bufferId(0)));
+ TEST_DO(assert_entries_to_alloc(6, Setup().needed(5).bufferId(0)));
}
-TEST("require that arrays to alloc is based on currently used elements (no resizing)")
+TEST("require that entries to alloc is based on currently used entries (no resizing)")
{
- TEST_DO(assertArraysToAlloc(2, Setup().used(4 * 4).needed(4)));
- TEST_DO(assertArraysToAlloc(4, Setup().used(8 * 4).needed(4)));
+ TEST_DO(assert_entries_to_alloc(2, Setup().used(4).needed(1)));
+ TEST_DO(assert_entries_to_alloc(4, Setup().used(8).needed(1)));
}
-TEST("require that arrays to alloc is based on currently used elements (with resizing)")
+TEST("require that entries to alloc is based on currently used entries (with resizing)")
{
- TEST_DO(assertArraysToAlloc(4 + 2, Setup().used(4 * 4).needed(4).resizing(true)));
- TEST_DO(assertArraysToAlloc(8 + 4, Setup().used(8 * 4).needed(4).resizing(true)));
- TEST_DO(assertArraysToAlloc(4 + 3, Setup().used(4 * 4).needed(3 * 4).resizing(true)));
+ TEST_DO(assert_entries_to_alloc(4 + 2, Setup().used(4).needed(1).resizing(true)));
+ TEST_DO(assert_entries_to_alloc(8 + 4, Setup().used(8).needed(1).resizing(true)));
+ TEST_DO(assert_entries_to_alloc(4 + 3, Setup().used(4).needed(3).resizing(true)));
}
-TEST("require that arrays to alloc always contain elements needed")
+TEST("require that entries to alloc always contain entries needed")
{
- TEST_DO(assertArraysToAlloc(2, Setup().used(4 * 4).needed(2 * 4)));
- TEST_DO(assertArraysToAlloc(3, Setup().used(4 * 4).needed(3 * 4)));
- TEST_DO(assertArraysToAlloc(4, Setup().used(4 * 4).needed(4 * 4)));
+ TEST_DO(assert_entries_to_alloc(2, Setup().used(4).needed(2)));
+ TEST_DO(assert_entries_to_alloc(3, Setup().used(4).needed(3)));
+ TEST_DO(assert_entries_to_alloc(4, Setup().used(4).needed(4)));
}
-TEST("require that arrays to alloc is capped to max arrays")
+TEST("require that entries to alloc is capped to max entries")
{
- TEST_DO(assertArraysToAlloc(127, Setup().used(254 * 4).needed(4)));
- TEST_DO(assertArraysToAlloc(128, Setup().used(256 * 4).needed(4)));
- TEST_DO(assertArraysToAlloc(128, Setup().used(258 * 4).needed(8)));
+ TEST_DO(assert_entries_to_alloc(127, Setup().used(254).needed(1)));
+ TEST_DO(assert_entries_to_alloc(128, Setup().used(256).needed(1)));
+ TEST_DO(assert_entries_to_alloc(128, Setup().used(258).needed(2)));
}
TEST("require that arrays to alloc is capped to min arrays")
{
- TEST_DO(assertArraysToAlloc(16, Setup().used(30 * 4).needed(4).minArrays(16)));
- TEST_DO(assertArraysToAlloc(16, Setup().used(32 * 4).needed(4).minArrays(16)));
- TEST_DO(assertArraysToAlloc(17, Setup().used(34 * 4).needed(4).minArrays(16)));
+ TEST_DO(assert_entries_to_alloc(16, Setup().used(30).needed(1).min_entries(16)));
+ TEST_DO(assert_entries_to_alloc(16, Setup().used(32).needed(1).min_entries(16)));
+ TEST_DO(assert_entries_to_alloc(17, Setup().used(34).needed(1).min_entries(16)));
}
-TEST("arrays to alloc considers used elements across all active buffers of same type (no resizing)")
+TEST("entries to alloc considers used entries across all active buffers of same type (no resizing)")
{
- Fixture f(Setup().used(6 * 4));
- f.assertArraysToAlloc(6 * 0.5);
- f.add_setup(Setup().used(8 * 4).bufferId(2));
- f.assertArraysToAlloc((6 + 8) * 0.5);
- f.add_setup(Setup().used(10 * 4).bufferId(3));
- f.assertArraysToAlloc((6 + 8 + 10) * 0.5);
+ Fixture f(Setup().used(6));
+ f.assert_entries_to_alloc(6 * 0.5);
+ f.add_setup(Setup().used(8).bufferId(2));
+ f.assert_entries_to_alloc((6 + 8) * 0.5);
+ f.add_setup(Setup().used(10).bufferId(3));
+ f.assert_entries_to_alloc((6 + 8 + 10) * 0.5);
}
-TEST("arrays to alloc considers used elements across all active buffers of same type when resizing")
+TEST("entries to alloc considers used entries across all active buffers of same type when resizing")
{
- Fixture f(Setup().used(6 * 4));
- f.assertArraysToAlloc(6 * 0.5);
- f.add_setup(Setup().used(8 * 4).resizing(true).bufferId(2));
- f.assertArraysToAlloc(8 + (6 + 8) * 0.5);
+ Fixture f(Setup().used(6));
+ f.assert_entries_to_alloc(6 * 0.5);
+ f.add_setup(Setup().used(8).resizing(true).bufferId(2));
+ f.assert_entries_to_alloc(8 + (6 + 8) * 0.5);
}
-TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers of same type (no resizing)")
+TEST("entries to alloc considers (and subtracts) dead entries across all active buffers of same type (no resizing)")
{
- Fixture f(Setup().used(6 * 4).dead(2 * 4));
- f.assertArraysToAlloc((6 - 2) * 0.5);
- f.add_setup(Setup().used(12 * 4).dead(4 * 4).bufferId(2));
- f.assertArraysToAlloc((6 - 2 + 12 - 4) * 0.5);
- f.add_setup(Setup().used(20 * 4).dead(6 * 4).bufferId(3));
- f.assertArraysToAlloc((6 - 2 + 12 - 4 + 20 - 6) * 0.5);
+ Fixture f(Setup().used(6).dead(2));
+ f.assert_entries_to_alloc((6 - 2) * 0.5);
+ f.add_setup(Setup().used(12).dead(4).bufferId(2));
+ f.assert_entries_to_alloc((6 - 2 + 12 - 4) * 0.5);
+ f.add_setup(Setup().used(20).dead(6).bufferId(3));
+ f.assert_entries_to_alloc((6 - 2 + 12 - 4 + 20 - 6) * 0.5);
}
TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers of same type when resizing")
{
- Fixture f(Setup().used(6 * 4).dead(2 * 4));
- f.assertArraysToAlloc((6 - 2) * 0.5);
- f.add_setup(Setup().used(12 * 4).dead(4 * 4).resizing(true).bufferId(2));
- f.assertArraysToAlloc(12 + (6 - 2 + 12 - 4) * 0.5);
+ Fixture f(Setup().used(6).dead(2));
+ f.assert_entries_to_alloc((6 - 2) * 0.5);
+ f.add_setup(Setup().used(12).dead(4).resizing(true).bufferId(2));
+ f.assert_entries_to_alloc(12 + (6 - 2 + 12 - 4) * 0.5);
}
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp
index df347267c7e..9e27ed37dd3 100644
--- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp
+++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp
@@ -26,8 +26,8 @@ public:
void holdBuffer(uint32_t bufferId) {
ParentType::holdBuffer(bufferId);
}
- void holdElem(EntryRef ref, uint64_t len) {
- ParentType::holdElem(ref, len);
+ void hold_entry(EntryRef ref) {
+ ParentType::hold_entry(ref);
}
void assign_generation(generation_t current_gen) {
ParentType::assign_generation(current_gen);
@@ -35,8 +35,8 @@ public:
void reclaim_entry_refs(generation_t oldest_used_gen) override {
ParentType::reclaim_entry_refs(oldest_used_gen);
}
- void ensureBufferCapacity(size_t sizeNeeded) {
- ParentType::ensureBufferCapacity(0, sizeNeeded);
+ void ensure_buffer_capacity(size_t entries_needed) {
+ ParentType::ensure_buffer_capacity(0, entries_needed);
}
void enableFreeLists() {
ParentType::enableFreeLists();
@@ -66,10 +66,10 @@ class GrowStore
BufferType<DataType> _type;
uint32_t _typeId;
public:
- GrowStore(size_t arraySize, size_t minArrays, size_t maxArrays, size_t numArraysForNewBuffer)
+ GrowStore(size_t arraySize, size_t min_entries, size_t max_entries, size_t num_entries_for_new_buffer)
: _store(),
- _firstType(1, 1, maxArrays, 0, ALLOC_GROW_FACTOR),
- _type(arraySize, minArrays, maxArrays, numArraysForNewBuffer, ALLOC_GROW_FACTOR),
+ _firstType(1, 1, max_entries, 0, ALLOC_GROW_FACTOR),
+ _type(arraySize, min_entries, max_entries, num_entries_for_new_buffer, ALLOC_GROW_FACTOR),
_typeId(0)
{
(void) _store.addType(&_firstType);
@@ -87,7 +87,7 @@ public:
while (sizes.size() < bufs) {
RefType iRef = (_type.getArraySize() == 1) ?
(_store.template allocator<DataType>(_typeId).alloc().ref) :
- (_store.template allocator<DataType>(_typeId).allocArray(_type.getArraySize()).ref);
+ (_store.template allocator<DataType>(_typeId).allocArray().ref);
int bufferId = iRef.bufferId();
if (bufferId != prevBufferId) {
if (prevBufferId >= 0) {
@@ -126,7 +126,7 @@ public:
while (buffers.size() < bufs) {
RefType iRef = (_type.getArraySize() == 1) ?
(_store.template allocator<DataType>(_typeId).alloc().ref) :
- (_store.template allocator<DataType>(_typeId).allocArray(_type.getArraySize()).ref);
+ (_store.template allocator<DataType>(_typeId).allocArray().ref);
int buffer_id = iRef.bufferId();
if (buffers.empty() || buffers.back() != buffer_id) {
buffers.push_back(buffer_id);
@@ -143,10 +143,10 @@ void
assertMemStats(const MemoryStats &exp,
const MemoryStats &act)
{
- EXPECT_EQ(exp._allocElems, act._allocElems);
- EXPECT_EQ(exp._usedElems, act._usedElems);
- EXPECT_EQ(exp._deadElems, act._deadElems);
- EXPECT_EQ(exp._holdElems, act._holdElems);
+ EXPECT_EQ(exp._alloc_entries, act._alloc_entries);
+ EXPECT_EQ(exp._used_entries, act._used_entries);
+ EXPECT_EQ(exp._dead_entries, act._dead_entries);
+ EXPECT_EQ(exp._hold_entries, act._hold_entries);
EXPECT_EQ(exp._freeBuffers, act._freeBuffers);
EXPECT_EQ(exp._activeBuffers, act._activeBuffers);
EXPECT_EQ(exp._holdBuffers, act._holdBuffers);
@@ -304,13 +304,13 @@ TEST(DataStoreTest, require_that_we_can_hold_and_trim_elements)
{
MyStore s;
MyRef r1 = s.addEntry(1);
- s.holdElem(r1, 1);
+ s.hold_entry(r1);
s.assign_generation(10);
MyRef r2 = s.addEntry(2);
- s.holdElem(r2, 1);
+ s.hold_entry(r2);
s.assign_generation(20);
MyRef r3 = s.addEntry(3);
- s.holdElem(r3, 1);
+ s.hold_entry(r3);
s.assign_generation(30);
EXPECT_EQ(1, s.getEntry(r1));
EXPECT_EQ(2, s.getEntry(r2));
@@ -358,11 +358,11 @@ TEST(DataStoreTest, require_that_we_can_use_free_lists)
MyStore s;
s.enableFreeLists();
auto r1 = s.addEntry(1);
- s.holdElem(r1, 1);
+ s.hold_entry(r1);
s.assign_generation(10);
auto r2 = s.addEntry(2);
expect_successive_refs(r1, r2);
- s.holdElem(r2, 1);
+ s.hold_entry(r2);
s.assign_generation(20);
s.reclaim_entry_refs(11);
auto r3 = s.addEntry(3); // reuse r1
@@ -389,21 +389,21 @@ TEST(DataStoreTest, require_that_we_can_use_free_lists_with_raw_allocator)
s.enableFreeLists();
auto allocator = s.freeListRawAllocator<int>(grow_store.typeId());
- auto h1 = allocator.alloc(3);
- auto h2 = allocator.alloc(3);
+ auto h1 = allocator.alloc(1);
+ auto h2 = allocator.alloc(1);
expect_successive_handles(h1, h2);
- s.holdElem(h1.ref, 3);
- s.holdElem(h2.ref, 3);
+ s.hold_entry(h1.ref);
+ s.hold_entry(h2.ref);
s.assign_generation(10);
s.reclaim_entry_refs(11);
- auto h3 = allocator.alloc(3); // reuse h2.ref from free list
+ auto h3 = allocator.alloc(1); // reuse h2.ref from free list
EXPECT_EQ(h2, h3);
- auto h4 = allocator.alloc(3); // reuse h1.ref from free list
+ auto h4 = allocator.alloc(1); // reuse h1.ref from free list
EXPECT_EQ(h1, h4);
- auto h5 = allocator.alloc(3);
+ auto h5 = allocator.alloc(1);
expect_successive_handles(h2, h5);
expect_successive_handles(h3, h5);
}
@@ -412,10 +412,10 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated)
{
MyStore s;
MemoryStats m;
- m._allocElems = MyRef::offsetSize();
- m._usedElems = 1; // ref = 0 is reserved
- m._deadElems = 1; // ref = 0 is reserved
- m._holdElems = 0;
+ m._alloc_entries = MyRef::offsetSize();
+ m._used_entries = 1; // ref = 0 is reserved
+ m._dead_entries = 1; // ref = 0 is reserved
+ m._hold_entries = 0;
m._activeBuffers = 1;
m._freeBuffers = MyRef::numBuffers() - 1;
m._holdBuffers = 0;
@@ -423,7 +423,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated)
// add entry
MyRef r = s.addEntry(10);
- m._usedElems++;
+ m._used_entries++;
assertMemStats(m, s.getMemStats());
// hold buffer
@@ -431,9 +431,9 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated)
s.addEntry(30);
s.holdBuffer(r.bufferId());
s.assign_generation(100);
- m._usedElems += 2;
- m._holdElems = m._usedElems;
- m._deadElems = 0;
+ m._used_entries += 2;
+ m._hold_entries = m._used_entries;
+ m._dead_entries = 0;
m._activeBuffers--;
m._holdBuffers++;
assertMemStats(m, s.getMemStats());
@@ -441,17 +441,17 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated)
// new active buffer
s.switch_primary_buffer();
s.addEntry(40);
- m._allocElems += MyRef::offsetSize();
- m._usedElems++;
+ m._alloc_entries += MyRef::offsetSize();
+ m._used_entries++;
m._activeBuffers++;
m._freeBuffers--;
// trim hold buffer
s.reclaim_memory(101);
- m._allocElems -= MyRef::offsetSize();
- m._usedElems = 1;
- m._deadElems = 0;
- m._holdElems = 0;
+ m._alloc_entries -= MyRef::offsetSize();
+ m._used_entries = 1;
+ m._dead_entries = 0;
+ m._hold_entries = 0;
m._freeBuffers = MyRef::numBuffers() - 1;
m._holdBuffers = 0;
assertMemStats(m, s.getMemStats());
@@ -466,7 +466,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated)
{ // increase extra hold bytes
auto prev_stats = s.getMemStats();
- s.get_active_buffer_state().hold_elems(0, 30);
+ s.get_active_buffer_state().hold_entries(0, 30);
auto curr_stats = s.getMemStats();
EXPECT_EQ(prev_stats._holdBytes + 30, curr_stats._holdBytes);
}
@@ -475,7 +475,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated)
TEST(DataStoreTest, require_that_memory_usage_is_calculated)
{
constexpr size_t BASE_ALLOCATED = 4228;
- constexpr size_t BASE_USED = 308;
+ constexpr size_t BASE_USED = 284;
MyStore s;
MyRef r = s.addEntry(10);
s.addEntry(20);
@@ -494,7 +494,7 @@ TEST(DataStoreTest, require_that_memory_usage_is_calculated)
TEST(DataStoreTest, require_that_we_can_disable_elemement_hold_list)
{
constexpr size_t BASE_ALLOCATED = 4228;
- constexpr size_t BASE_USED = 308;
+ constexpr size_t BASE_USED = 284;
MyStore s;
MyRef r1 = s.addEntry(10);
MyRef r2 = s.addEntry(20);
@@ -505,14 +505,14 @@ TEST(DataStoreTest, require_that_we_can_disable_elemement_hold_list)
EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes());
EXPECT_EQ(1 * sizeof(int), m.deadBytes());
EXPECT_EQ(0 * sizeof(int), m.allocatedBytesOnHold());
- s.holdElem(r1, 1);
+ s.hold_entry(r1);
m = s.getMemoryUsage();
EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes());
EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes());
EXPECT_EQ(1 * sizeof(int), m.deadBytes());
EXPECT_EQ(1 * sizeof(int), m.allocatedBytesOnHold());
- s.disableElemHoldList();
- s.holdElem(r2, 1);
+ s.disable_entry_hold_list();
+ s.hold_entry(r2);
m = s.getMemoryUsage();
EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes());
EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes());
@@ -529,11 +529,11 @@ namespace {
void assertGrowStats(GrowthStats expSizes,
GrowthStats expFirstBufSizes,
size_t expInitMemUsage,
- size_t minArrays, size_t numArraysForNewBuffer, size_t maxArrays = 128)
+ size_t min_entries, size_t num_entries_for_new_buffer, size_t max_entries = 128)
{
- EXPECT_EQ(expSizes, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getGrowthStats(expSizes.size()));
- EXPECT_EQ(expFirstBufSizes, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getFirstBufGrowStats());
- EXPECT_EQ(expInitMemUsage, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getMemoryUsage().allocatedBytes());
+ EXPECT_EQ(expSizes, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getGrowthStats(expSizes.size()));
+ EXPECT_EQ(expFirstBufSizes, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getFirstBufGrowStats());
+ EXPECT_EQ(expInitMemUsage, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getMemoryUsage().allocatedBytes());
}
}
@@ -574,10 +574,10 @@ namespace {
template <typename DataType>
void assertGrowStats(GrowthStats expSizes, uint32_t arraySize)
{
- uint32_t minArrays = 2048;
- uint32_t maxArrays = RefType15::offsetSize();
- uint32_t numArraysForNewBuffer = 2048;
- GrowStore<DataType, RefType15> store(arraySize, minArrays, maxArrays, numArraysForNewBuffer);
+ uint32_t min_entries = 2048;
+ uint32_t max_entries = RefType15::offsetSize();
+ uint32_t num_entries_for_new_buffer = 2048;
+ GrowStore<DataType, RefType15> store(arraySize, min_entries, max_entries, num_entries_for_new_buffer);
EXPECT_EQ(expSizes, store.getGrowthStats(expSizes.size()));
}
@@ -594,14 +594,14 @@ TEST(DataStoreTest, require_that_offset_in_EntryRefT_is_within_bounds_when_alloc
* 3) Round up bytes to alloc to match the underlying allocator (power of 2 if less than huge page size):
* After this we might end up with more bytes than the offset in EntryRef can handle. In this case this is 32768.
* 4) Cap bytes to alloc to the max offset EntryRef can handle.
- * The max bytes to alloc is: maxArrays * arraySize * elementSize.
+ * The max bytes to alloc is: max_entries * arraySize * elementSize.
*/
- assertGrowStats<uint8_t>({8192,16384,16384,65536,65536,98304,98304,98304,98304,98304,98304,98304}, 3);
- assertGrowStats<uint8_t>({16384,16384,65536,65536,131072,131072,163840,163840,163840,163840,163840,163840}, 5);
- assertGrowStats<uint8_t>({16384,32768,32768,131072,131072,229376,229376,229376,229376,229376,229376,229376}, 7);
- assertGrowStats<uint32_t>({8192,16384,16384,65536,65536,98304,98304,98304,98304,98304,98304,98304}, 3);
- assertGrowStats<uint32_t>({16384,16384,65536,65536,131072,131072,163840,163840,163840,163840,163840,163840}, 5);
- assertGrowStats<uint32_t>({16384,32768,32768,131072,131072,229376,229376,229376,229376,229376,229376,229376}, 7);
+ assertGrowStats<uint8_t>({2730,5461,5461,21845,21845,32768,32768,32768,32768,32768,32768,32768}, 3);
+ assertGrowStats<uint8_t>({3276,3276,13107,13107,26214,26214,32768,32768,32768,32768,32768,32768}, 5);
+ assertGrowStats<uint8_t>({2340,4681,4681,18724,18724,32768,32768,32768,32768,32768,32768,32768}, 7);
+ assertGrowStats<uint32_t>({2730,5461,5461,21845,21845,32768,32768,32768,32768,32768,32768,32768}, 3);
+ assertGrowStats<uint32_t>({3276,3276,13107,13107,26214,26214,32768,32768,32768,32768,32768,32768}, 5);
+ assertGrowStats<uint32_t>({2340,4681,4681,18724,18724,32768,32768,32768,32768,32768,32768,32768}, 7);
}
namespace {
@@ -667,9 +667,9 @@ TEST(DataStoreTest, can_reuse_active_buffer_as_primary_buffer)
TEST(DataStoreTest, control_static_sizes) {
EXPECT_EQ(88, sizeof(BufferTypeBase));
EXPECT_EQ(24, sizeof(FreeList));
- EXPECT_EQ(56, sizeof(BufferFreeList));
+ EXPECT_EQ(48, sizeof(BufferFreeList));
EXPECT_EQ(1, sizeof(BufferState::State));
- EXPECT_EQ(144, sizeof(BufferState));
+ EXPECT_EQ(120, sizeof(BufferState));
BufferState bs;
EXPECT_EQ(0, bs.size());
}
@@ -685,11 +685,11 @@ void test_free_element_to_held_buffer(bool before_hold_buffer)
EXPECT_EQ(1u, s.primary_buffer_id());
if (before_hold_buffer) {
- s.holdElem(ref, 1);
+ s.hold_entry(ref);
}
s.holdBuffer(0); // hold last buffer
if (!before_hold_buffer) {
- ASSERT_DEATH({ s.holdElem(ref, 1); }, "isActive\\(\\)");
+ ASSERT_DEATH({ s.hold_entry(ref); }, "isActive\\(\\)");
}
s.assign_generation(100);
s.reclaim_memory(101);
diff --git a/vespalib/src/tests/datastore/free_list/free_list_test.cpp b/vespalib/src/tests/datastore/free_list/free_list_test.cpp
index 44e11b2316b..ec14d0dd28c 100644
--- a/vespalib/src/tests/datastore/free_list/free_list_test.cpp
+++ b/vespalib/src/tests/datastore/free_list/free_list_test.cpp
@@ -8,20 +8,18 @@
using namespace vespalib::datastore;
using MyEntryRef = EntryRefT<8, 4>;
-constexpr uint32_t array_size = 6;
struct FreeListTest : public testing::Test
{
FreeList list;
- std::atomic<ElemCount> dead_elems;
+ std::atomic<EntryCount> dead_entries;
std::vector<BufferFreeList> bufs;
FreeListTest()
: list(),
bufs()
{
for (size_t i = 0; i < 3; ++i) {
- bufs.emplace_back(dead_elems);
- bufs.back().set_array_size(array_size);
+ bufs.emplace_back(dead_entries);
}
}
void TearDown() override {
@@ -126,13 +124,13 @@ TEST_F(FreeListTest, buffer_free_list_can_be_disabled_and_detached_when_not_curr
EXPECT_TRUE(list.empty());
}
-TEST_F(FreeListTest, dead_elems_count_is_updated_when_popping_an_entry)
+TEST_F(FreeListTest, dead_entries_count_is_updated_when_popping_an_entry)
{
enable(0);
push_entry({10, 0});
- dead_elems.store(18, std::memory_order_relaxed);
+ dead_entries.store(18, std::memory_order_relaxed);
pop_entry();
- EXPECT_EQ(18 - array_size, dead_elems.load(std::memory_order_relaxed));
+ EXPECT_EQ(17, dead_entries.load(std::memory_order_relaxed));
}
GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
index 5ccf9a8908c..a09a7213bf5 100644
--- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
+++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
@@ -96,8 +96,8 @@ struct TestBase : public ::testing::Test {
}
void assertBufferState(EntryRef ref, const TestBufferStats expStats) const {
EXPECT_EQ(expStats._used, store.bufferState(ref).size());
- EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_elems());
- EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_elems());
+ EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_entries());
+ EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_entries());
}
void assertStoreContent() const {
for (const auto &elem : refStore) {
@@ -147,10 +147,7 @@ struct TestBase : public ::testing::Test {
auto getBuilder(uint32_t uniqueValuesHint) { return store.getBuilder(uniqueValuesHint); }
auto getEnumerator(bool sort_unique_values) { return store.getEnumerator(sort_unique_values); }
size_t get_reserved(EntryRef ref) {
- return store.bufferState(ref).getTypeHandler()->getReservedElements(getBufferId(ref));
- }
- size_t get_array_size(EntryRef ref) {
- return store.bufferState(ref).getArraySize();
+ return store.bufferState(ref).getTypeHandler()->get_reserved_entries(getBufferId(ref));
}
};
@@ -309,29 +306,27 @@ TYPED_TEST(TestBase, can_add_and_get_values)
}
}
-TYPED_TEST(TestBase, elements_are_put_on_hold_when_value_is_removed)
+TYPED_TEST(TestBase, entries_are_put_on_hold_when_value_is_removed)
{
EntryRef ref = this->add(this->values()[0]);
size_t reserved = this->get_reserved(ref);
- size_t array_size = this->get_array_size(ref);
- this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved));
+ this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved));
this->store.remove(ref);
- this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(array_size).dead(reserved));
+ this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(1).dead(reserved));
}
-TYPED_TEST(TestBase, elements_are_reference_counted)
+TYPED_TEST(TestBase, entries_are_reference_counted)
{
EntryRef ref = this->add(this->values()[0]);
EntryRef ref2 = this->add(this->values()[0]);
EXPECT_EQ(ref.ref(), ref2.ref());
- // Note: The first buffer have the first element reserved -> we expect 2 elements used here.
+ // Note: The first buffer have the first entry reserved -> we expect 2 entries used here.
size_t reserved = this->get_reserved(ref);
- size_t array_size = this->get_array_size(ref);
- this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved));
+ this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved));
this->store.remove(ref);
- this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved));
+ this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved));
this->store.remove(ref);
- this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(array_size).dead(reserved));
+ this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(1).dead(reserved));
}
TEST_F(SmallOffsetNumberTest, new_underlying_buffer_is_allocated_when_current_is_full)
@@ -360,8 +355,7 @@ TYPED_TEST(TestBase, store_can_be_compacted)
this->remove(this->add(this->values()[2]));
this->reclaim_memory();
size_t reserved = this->get_reserved(val0Ref);
- size_t array_size = this->get_array_size(val0Ref);
- this->assertBufferState(val0Ref, TestBufferStats().used(reserved + 3 * array_size).dead(reserved + array_size));
+ this->assertBufferState(val0Ref, TestBufferStats().used(reserved + 3).dead(reserved + 1));
uint32_t val1BufferId = this->getBufferId(val0Ref);
EXPECT_EQ(2u, this->refStore.size());
@@ -389,8 +383,7 @@ TYPED_TEST(TestBase, store_can_be_instantiated_with_builder)
EntryRef val0Ref = builder.mapEnumValueToEntryRef(1);
EntryRef val1Ref = builder.mapEnumValueToEntryRef(2);
size_t reserved = this->get_reserved(val0Ref);
- size_t array_size = this->get_array_size(val0Ref);
- this->assertBufferState(val0Ref, TestBufferStats().used(2 * array_size + reserved).dead(reserved)); // Note: First element is reserved
+ this->assertBufferState(val0Ref, TestBufferStats().used(2 + reserved).dead(reserved)); // Note: First entry is reserved
EXPECT_TRUE(val0Ref.valid());
EXPECT_TRUE(val1Ref.valid());
EXPECT_NE(val0Ref.ref(), val1Ref.ref());
@@ -472,13 +465,13 @@ TEST_F(DoubleTest, nan_is_handled)
TEST_F(DoubleTest, control_memory_usage) {
static constexpr size_t sizeof_deque = vespalib::datastore::DataStoreBase::sizeof_entry_ref_hold_list_deque;
EXPECT_EQ(368u + sizeof_deque, sizeof(store));
- EXPECT_EQ(144u, sizeof(BufferState));
+ EXPECT_EQ(120u, sizeof(BufferState));
EXPECT_EQ(28740u, store.get_values_memory_usage().allocatedBytes());
- EXPECT_EQ(24804u, store.get_values_memory_usage().usedBytes());
+ EXPECT_EQ(24780u, store.get_values_memory_usage().usedBytes());
EXPECT_EQ(126952u, store.get_dictionary_memory_usage().allocatedBytes());
- EXPECT_EQ(25248u, store.get_dictionary_memory_usage().usedBytes());
+ EXPECT_EQ(25200u, store.get_dictionary_memory_usage().usedBytes());
EXPECT_EQ(155692u, store.getMemoryUsage().allocatedBytes());
- EXPECT_EQ(50052, store.getMemoryUsage().usedBytes());
+ EXPECT_EQ(49980u, store.getMemoryUsage().usedBytes());
}
GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp
index 7d4451556c8..8ea7f807f56 100644
--- a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp
+++ b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp
@@ -62,8 +62,8 @@ struct TestBase : public ::testing::Test {
void assert_buffer_state(EntryRef ref, const TestBufferStats expStats) {
auto & stats = buffer_state(ref).stats();
EXPECT_EQ(expStats._used, buffer_state(ref).size());
- EXPECT_EQ(expStats._hold, stats.hold_elems());
- EXPECT_EQ(expStats._dead, stats.dead_elems());
+ EXPECT_EQ(expStats._hold, stats.hold_entries());
+ EXPECT_EQ(expStats._dead, stats.dead_entries());
EXPECT_EQ(expStats._extra_used, stats.extra_used_bytes());
EXPECT_EQ(expStats._extra_hold, stats.extra_hold_bytes());
}
@@ -83,14 +83,14 @@ TEST_F(StringTest, can_add_and_get_values)
assert_add(spaces1000.c_str());
}
-TEST_F(StringTest, elements_are_put_on_hold_when_value_is_removed)
+TEST_F(StringTest, entries_are_put_on_hold_when_value_is_removed)
{
EntryRef ref = add(small.c_str());
- assert_buffer_state(ref, TestBufferStats().used(16).hold(0).dead(0));
+ assert_buffer_state(ref, TestBufferStats().used(1).hold(0).dead(0));
remove(ref);
- assert_buffer_state(ref, TestBufferStats().used(16).hold(16).dead(0));
+ assert_buffer_state(ref, TestBufferStats().used(1).hold(1).dead(0));
reclaim_memory();
- assert_buffer_state(ref, TestBufferStats().used(16).hold(0).dead(16));
+ assert_buffer_state(ref, TestBufferStats().used(1).hold(0).dead(1));
}
TEST_F(StringTest, extra_bytes_used_is_tracked)
@@ -139,7 +139,7 @@ TEST_F(StringTest, free_list_is_used_when_enabled)
EntryRef ref4 = add(spaces1000.c_str());
EXPECT_EQ(ref1, ref3);
EXPECT_EQ(ref2, ref4);
- assert_buffer_state(ref1, TestBufferStats().used(16).hold(0).dead(0));
+ assert_buffer_state(ref1, TestBufferStats().used(1).hold(0).dead(0));
assert_buffer_state(ref2, TestBufferStats().used(2).hold(0).dead(1).extra_used(1001));
}
@@ -155,7 +155,7 @@ TEST_F(StringTest, free_list_is_not_used_when_disabled)
EntryRef ref4 = add(spaces1000.c_str());
EXPECT_NE(ref1, ref3);
EXPECT_NE(ref2, ref4);
- assert_buffer_state(ref1, TestBufferStats().used(32).hold(0).dead(16));
+ assert_buffer_state(ref1, TestBufferStats().used(2).hold(0).dead(1));
assert_buffer_state(ref2, TestBufferStats().used(3).hold(0).dead(2).extra_used(1001));
}
@@ -173,7 +173,7 @@ TEST_F(StringTest, free_list_is_never_used_for_move_on_compact)
EntryRef ref6 = move_on_compact(ref2);
EXPECT_NE(ref5, ref3);
EXPECT_NE(ref6, ref4);
- assert_buffer_state(ref1, TestBufferStats().used(48).hold(0).dead(16));
+ assert_buffer_state(ref1, TestBufferStats().used(3).hold(0).dead(1));
assert_buffer_state(ref2, TestBufferStats().used(4).hold(0).dead(2).extra_used(2002));
}
diff --git a/vespalib/src/tests/signalhandler/CMakeLists.txt b/vespalib/src/tests/signalhandler/CMakeLists.txt
index 4f78eb2e82d..88be14f994f 100644
--- a/vespalib/src/tests/signalhandler/CMakeLists.txt
+++ b/vespalib/src/tests/signalhandler/CMakeLists.txt
@@ -5,6 +5,11 @@ vespa_add_library(vespalib_signalhandler_test_my_shared_library TEST
DEPENDS
vespalib
)
+
+# Don't convert call to jump when returning a value from a function with
+# a compatible stack.
+set_source_files_properties(my_shared_library.cpp PROPERTIES COMPILE_OPTIONS "-fno-optimize-sibling-calls")
+
vespa_add_executable(vespalib_signalhandler_test_app TEST
SOURCES
signalhandler_test.cpp
diff --git a/vespalib/src/vespa/vespalib/btree/btree.h b/vespalib/src/vespa/vespalib/btree/btree.h
index 32b538b65ec..c2f5aac01b7 100644
--- a/vespalib/src/vespa/vespalib/btree/btree.h
+++ b/vespalib/src/vespa/vespalib/btree/btree.h
@@ -61,9 +61,9 @@ public:
}
void
- disableElemHoldList()
+ disable_entry_hold_list()
{
- _alloc.disableElemHoldList();
+ _alloc.disable_entry_hold_list();
}
// Inherit doc from BTreeRoot
diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h
index 784e95e3817..b537602c703 100644
--- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h
+++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h
@@ -60,8 +60,8 @@ public:
_nodeStore.disableFreeLists();
}
- void disableElemHoldList() {
- _nodeStore.disableElemHoldList();
+ void disable_entry_hold_list() {
+ _nodeStore.disable_entry_hold_list();
}
/**
diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp
index a38b68afe73..d23c8fc2054 100644
--- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp
+++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp
@@ -162,7 +162,7 @@ holdNode(BTreeNode::Ref nodeRef,
InternalNodeType *node)
{
if (node->getFrozen()) {
- _nodeStore.holdElem(nodeRef);
+ _nodeStore.hold_entry(nodeRef);
} else {
node->clean();
_internalHoldUntilFreeze.push_back(nodeRef);
@@ -178,7 +178,7 @@ holdNode(BTreeNode::Ref nodeRef,
LeafNodeType *node)
{
if (node->getFrozen()) {
- _nodeStore.holdElem(nodeRef);
+ _nodeStore.hold_entry(nodeRef);
} else {
node->clean();
_leafHoldUntilFreeze.push_back(nodeRef);
@@ -235,7 +235,7 @@ freeze()
InternalNodeType *inode = mapInternalRef(i);
(void) inode;
assert(inode->getFrozen());
- _nodeStore.holdElem(i);
+ _nodeStore.hold_entry(i);
}
_internalHoldUntilFreeze.clear();
}
@@ -245,7 +245,7 @@ freeze()
LeafNodeType *lnode = mapLeafRef(i);
(void) lnode;
assert(lnode->getFrozen());
- _nodeStore.holdElem(i);
+ _nodeStore.hold_entry(i);
}
_leafHoldUntilFreeze.clear();
}
diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h
index c59092bf75c..38bf4e5ed4e 100644
--- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h
+++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h
@@ -29,16 +29,16 @@ class BTreeNodeBufferType : public datastore::BufferType<EntryType, FrozenBtreeN
using ParentType = datastore::BufferType<EntryType, FrozenBtreeNode<EntryType>>;
using ParentType::empty_entry;
using ParentType::_arraySize;
- using ElemCount = typename ParentType::ElemCount;
+ using EntryCount = typename ParentType::EntryCount;
using CleanContext = typename ParentType::CleanContext;
public:
- BTreeNodeBufferType(uint32_t minArrays, uint32_t maxArrays)
- : ParentType(1, minArrays, maxArrays)
+ BTreeNodeBufferType(uint32_t min_entries, uint32_t max_entries)
+ : ParentType(1, min_entries, max_entries)
{ }
- void initializeReservedElements(void *buffer, ElemCount reservedElements) override;
+ void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override;
- void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override;
+ void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override;
};
@@ -79,7 +79,7 @@ public:
~BTreeNodeStore();
void disableFreeLists() { _store.disableFreeLists(); }
- void disableElemHoldList() { _store.disableElemHoldList(); }
+ void disable_entry_hold_list() { _store.disable_entry_hold_list(); }
static bool isValidRef(EntryRef ref) { return ref.valid(); }
@@ -152,8 +152,8 @@ public:
return _store.freeListAllocator<InternalNodeType, BTreeNodeReclaimer>(NODETYPE_INTERNAL).alloc(rhs);
}
- void holdElem(EntryRef ref) {
- _store.holdElem(ref, 1);
+ void hold_entry(EntryRef ref) {
+ _store.hold_entry(ref);
}
std::unique_ptr<vespalib::datastore::CompactingBuffers> start_compact_worst(const CompactionStrategy& compaction_strategy);
diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp
index a1ffb4d445d..99054f35d61 100644
--- a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp
+++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp
@@ -11,11 +11,11 @@ namespace vespalib::btree {
template <typename EntryType>
void
-BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, ElemCount reservedElements)
+BTreeNodeBufferType<EntryType>::initialize_reserved_entries(void *buffer, EntryCount reserved_entries)
{
- ParentType::initializeReservedElements(buffer, reservedElements);
+ ParentType::initialize_reserved_entries(buffer, reserved_entries);
EntryType *e = static_cast<EntryType *>(buffer);
- for (size_t j = reservedElements; j != 0; --j) {
+ for (size_t j = reserved_entries; j != 0; --j) {
e->freeze();
++e;
}
@@ -24,10 +24,10 @@ BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, ElemCou
template <typename EntryType>
void
-BTreeNodeBufferType<EntryType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext)
+BTreeNodeBufferType<EntryType>::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext)
{
EntryType *e = static_cast<EntryType *>(buffer) + offset;
- for (size_t j = numElems; j != 0; --j) {
+ for (size_t j = num_entries; j != 0; --j) {
e->cleanFrozen();
++e;
}
diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.h b/vespalib/src/vespa/vespalib/btree/btreestore.h
index 9fdade850f1..7dd839f1529 100644
--- a/vespalib/src/vespa/vespalib/btree/btreestore.h
+++ b/vespalib/src/vespa/vespalib/btree/btreestore.h
@@ -50,8 +50,8 @@ public:
using CompactionSpec = datastore::CompactionSpec;
using CompactionStrategy = datastore::CompactionStrategy;
using EntryRef = datastore::EntryRef;
- template <typename EntryType>
- using BufferType = datastore::BufferType<EntryType>;
+ template <typename ElemT>
+ using BufferType = datastore::BufferType<ElemT>;
using BufferState = datastore::BufferState;
static constexpr uint32_t clusterLimit = 8;
@@ -105,9 +105,9 @@ public:
_allocator.disableFreeLists();
}
- void disableElemHoldList() {
- _store.disableElemHoldList();
- _allocator.disableElemHoldList();
+ void disable_entry_hold_list() {
+ _store.disable_entry_hold_list();
+ _allocator.disable_entry_hold_list();
}
BTreeTypeRefPair allocNewBTree() {
diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.hpp b/vespalib/src/vespa/vespalib/btree/btreestore.hpp
index a19d0b34aa6..90c302af5e4 100644
--- a/vespalib/src/vespa/vespalib/btree/btreestore.hpp
+++ b/vespalib/src/vespa/vespalib/btree/btreestore.hpp
@@ -74,7 +74,7 @@ allocNewKeyData(uint32_t clusterSize)
{
assert(clusterSize >= 1 && clusterSize <= clusterLimit);
uint32_t typeId = clusterSize - 1;
- return _store.allocator<KeyDataType>(typeId).allocArray(clusterSize);
+ return _store.allocator<KeyDataType>(typeId).allocArray();
}
@@ -87,7 +87,7 @@ allocKeyData(uint32_t clusterSize)
{
assert(clusterSize >= 1 && clusterSize <= clusterLimit);
uint32_t typeId = clusterSize - 1;
- return _store.freeListAllocator<KeyDataType, datastore::DefaultReclaimer<KeyDataType>>(typeId).allocArray(clusterSize);
+ return _store.freeListAllocator<KeyDataType, datastore::DefaultReclaimer<KeyDataType>>(typeId).allocArray();
}
@@ -156,7 +156,7 @@ makeTree(EntryRef &ref,
lNode->freeze();
BTreeTypeRefPair tPair(allocBTree());
tPair.data->setRoots(lPair.ref);
- _store.holdElem(ref, clusterSize);
+ _store.hold_entry(ref);
ref = tPair.ref;
}
@@ -176,7 +176,7 @@ makeArray(EntryRef &ref, EntryRef root, LeafNodeType *leafNode)
kd->setData(leafNode->getData(idx));
}
assert(kd == kPair.data + clusterSize);
- _store.holdElem(ref, 1);
+ _store.hold_entry(ref);
if (!leafNode->getFrozen()) {
leafNode->freeze();
}
@@ -255,7 +255,7 @@ insert(EntryRef &ref,
kd->setData(i->getData());
}
assert(kd == kPair.data + clusterSize + 1);
- _store.holdElem(ref, clusterSize);
+ _store.hold_entry(ref);
ref = kPair.ref;
return true;
}
@@ -284,7 +284,7 @@ insert(EntryRef &ref,
lNode->freeze();
BTreeTypeRefPair tPair(allocBTree());
tPair.data->setRoots(lPair.ref); // allow immediate access to readers
- _store.holdElem(ref, clusterSize);
+ _store.hold_entry(ref);
ref = tPair.ref;
return true;
#endif
@@ -339,7 +339,7 @@ remove(EntryRef &ref,
if (oldi == olde || comp(key, oldi->_key))
return false; // not found
if (clusterSize == 1) {
- _store.holdElem(ref, 1);
+ _store.hold_entry(ref);
ref = EntryRef();
return true;
}
@@ -357,7 +357,7 @@ remove(EntryRef &ref,
kd->setData(i->getData());
}
assert(kd == kPair.data + clusterSize - 1);
- _store.holdElem(ref, clusterSize);
+ _store.hold_entry(ref);
ref = kPair.ref;
return true;
}
@@ -670,7 +670,7 @@ applyCluster(EntryRef &ref,
if (newSizeMin <= clusterLimit) {
uint32_t newSize = getNewClusterSize(ob, oe, a, ae, r, re, comp);
if (newSize == 0) {
- _store.holdElem(ref, clusterSize);
+ _store.hold_entry(ref);
ref = EntryRef();
return true;
}
@@ -678,7 +678,7 @@ applyCluster(EntryRef &ref,
KeyDataTypeRefPair kPair(allocKeyData(newSize));
applyCluster(ob, oe, kPair.data, kPair.data + newSize,
a, ae, r, re, comp);
- _store.holdElem(ref, clusterSize);
+ _store.hold_entry(ref);
ref = kPair.ref;
return true;
}
@@ -735,7 +735,7 @@ normalizeTree(EntryRef &ref,
{
EntryRef root = tree->getRoot();
if (!NodeAllocatorType::isValidRef(root)) {
- _store.holdElem(ref, 1);
+ _store.hold_entry(ref);
ref = EntryRef();
return;
}
@@ -798,10 +798,8 @@ clear(const EntryRef ref)
if (clusterSize == 0) {
BTreeType *tree = getWTreeEntry(iRef);
tree->clear(_allocator);
- _store.holdElem(ref, 1);
- } else {
- _store.holdElem(ref, clusterSize);
}
+ _store.hold_entry(ref);
}
diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.h b/vespalib/src/vespa/vespalib/datastore/allocator.h
index 297270af0f5..30938bdc1c1 100644
--- a/vespalib/src/vespa/vespalib/datastore/allocator.h
+++ b/vespalib/src/vespa/vespalib/datastore/allocator.h
@@ -30,7 +30,7 @@ public:
HandleType alloc(Args && ... args);
HandleType allocArray(ConstArrayRef array);
- HandleType allocArray(size_t size);
+ HandleType allocArray();
};
}
diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.hpp b/vespalib/src/vespa/vespalib/datastore/allocator.hpp
index 85f0e842519..fa97ef9a5f5 100644
--- a/vespalib/src/vespa/vespalib/datastore/allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/allocator.hpp
@@ -20,7 +20,7 @@ template <typename ... Args>
typename Allocator<EntryT, RefT>::HandleType
Allocator<EntryT, RefT>::alloc(Args && ... args)
{
- _store.ensureBufferCapacity(_typeId, 1);
+ _store.ensure_buffer_capacity(_typeId, 1);
uint32_t buffer_id = _store.primary_buffer_id(_typeId);
BufferState &state = _store.getBufferState(buffer_id);
assert(state.isActive());
@@ -36,39 +36,35 @@ template <typename EntryT, typename RefT>
typename Allocator<EntryT, RefT>::HandleType
Allocator<EntryT, RefT>::allocArray(ConstArrayRef array)
{
- _store.ensureBufferCapacity(_typeId, array.size());
+ _store.ensure_buffer_capacity(_typeId, 1);
uint32_t buffer_id = _store.primary_buffer_id(_typeId);
BufferState &state = _store.getBufferState(buffer_id);
assert(state.isActive());
assert(state.getArraySize() == array.size());
- size_t oldBufferSize = state.size();
- assert((oldBufferSize % array.size()) == 0);
- RefT ref((oldBufferSize / array.size()), buffer_id);
+ RefT ref(state.size(), buffer_id);
EntryT *buf = _store.template getEntryArray<EntryT>(ref, array.size());
for (size_t i = 0; i < array.size(); ++i) {
new (static_cast<void *>(buf + i)) EntryT(array[i]);
}
- state.stats().pushed_back(array.size());
+ state.stats().pushed_back(1);
return HandleType(ref, buf);
}
template <typename EntryT, typename RefT>
typename Allocator<EntryT, RefT>::HandleType
-Allocator<EntryT, RefT>::allocArray(size_t size)
+Allocator<EntryT, RefT>::allocArray()
{
- _store.ensureBufferCapacity(_typeId, size);
+ _store.ensure_buffer_capacity(_typeId, 1);
uint32_t buffer_id = _store.primary_buffer_id(_typeId);
BufferState &state = _store.getBufferState(buffer_id);
assert(state.isActive());
- assert(state.getArraySize() == size);
- size_t oldBufferSize = state.size();
- assert((oldBufferSize % size) == 0);
- RefT ref((oldBufferSize / size), buffer_id);
- EntryT *buf = _store.template getEntryArray<EntryT>(ref, size);
- for (size_t i = 0; i < size; ++i) {
+ RefT ref(state.size(), buffer_id);
+ auto array_size = state.getArraySize();
+ EntryT *buf = _store.template getEntryArray<EntryT>(ref, array_size);
+ for (size_t i = 0; i < array_size; ++i) {
new (static_cast<void *>(buf + i)) EntryT();
}
- state.stats().pushed_back(size);
+ state.stats().pushed_back(1);
return HandleType(ref, buf);
}
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h
index c2b65d72f03..809ac10f6e3 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store.h
+++ b/vespalib/src/vespa/vespalib/datastore/array_store.h
@@ -19,7 +19,7 @@
namespace vespalib::datastore {
/**
- * Datastore for storing arrays of type EntryT that is accessed via a 32-bit EntryRef.
+ * Datastore for storing arrays of type ElemT that is accessed via a 32-bit EntryRef.
*
* The default EntryRef type uses 19 bits for offset (524288 values) and 13 bits for buffer id (8192 buffers).
*
@@ -29,16 +29,18 @@ namespace vespalib::datastore {
*
* The max value of maxSmallArrayTypeId is (2^bufferBits - 1).
*/
-template <typename EntryT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreSimpleTypeMapper<EntryT> >
+template <typename ElemT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreSimpleTypeMapper<ElemT> >
class ArrayStore : public ICompactable
{
public:
using AllocSpec = ArrayStoreConfig::AllocSpec;
- using ArrayRef = vespalib::ArrayRef<EntryT>;
- using ConstArrayRef = vespalib::ConstArrayRef<EntryT>;
+ using ArrayRef = vespalib::ArrayRef<ElemT>;
+ using ConstArrayRef = vespalib::ConstArrayRef<ElemT>;
using DataStoreType = DataStoreT<RefT>;
- using LargeArray = vespalib::Array<EntryT>;
+ using ElemType = ElemT;
+ using LargeArray = vespalib::Array<ElemT>;
using LargeBufferType = typename TypeMapperT::LargeBufferType;
+ using RefType = RefT;
using SmallBufferType = typename TypeMapperT::SmallBufferType;
using TypeMapper = TypeMapperT;
private:
@@ -58,7 +60,7 @@ private:
EntryRef addLargeArray(const ConstArrayRef &array);
EntryRef allocate_large_array(size_t array_size);
ConstArrayRef getSmallArray(RefT ref, size_t arraySize) const {
- const EntryT *buf = _store.template getEntryArray<EntryT>(ref, arraySize);
+ const ElemT *buf = _store.template getEntryArray<ElemT>(ref, arraySize);
return ConstArrayRef(buf, arraySize);
}
ConstArrayRef getLargeArray(RefT ref) const {
@@ -85,7 +87,7 @@ public:
}
/**
- * Allocate an array of the given size without any instantiation of EntryT elements.
+ * Allocate an array of the given size without any instantiation of ElemT elements.
*
* Use get_writable() to get a reference to the array for writing.
*
@@ -148,14 +150,14 @@ public:
static ArrayStoreConfig optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId,
size_t hugePageSize,
size_t smallPageSize,
- size_t minNumArraysForNewBuffer,
+ size_t min_num_entries_for_new_buffer,
float allocGrowFactor);
static ArrayStoreConfig optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId,
const TypeMapper& mapper,
size_t hugePageSize,
size_t smallPageSize,
- size_t minNumArraysForNewBuffer,
+ size_t min_num_entries_for_new_buffer,
float allocGrowFactor);
};
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp
index 301cff1e414..8e9fe779ba9 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp
@@ -15,9 +15,9 @@
namespace vespalib::datastore {
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
void
-ArrayStore<EntryT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator)
+ArrayStore<ElemT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator)
{
_largeArrayTypeId = _store.addType(&_largeArrayType);
assert(_largeArrayTypeId == 0);
@@ -31,14 +31,14 @@ ArrayStore<EntryT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cf
}
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
-ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator)
+template <typename ElemT, typename RefT, typename TypeMapperT>
+ArrayStore<ElemT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator)
: ArrayStore(cfg, memory_allocator, TypeMapper())
{
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
-ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator,
+template <typename ElemT, typename RefT, typename TypeMapperT>
+ArrayStore<ElemT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator,
TypeMapper&& mapper)
: _largeArrayTypeId(0),
_maxSmallArrayTypeId(cfg.maxSmallArrayTypeId()),
@@ -56,25 +56,25 @@ ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, s
}
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
vespalib::MemoryUsage
-ArrayStore<EntryT, RefT, TypeMapperT>::getMemoryUsage() const {
+ArrayStore<ElemT, RefT, TypeMapperT>::getMemoryUsage() const {
vespalib::MemoryUsage usage = _store.getMemoryUsage();
usage.incAllocatedBytes(_smallArrayTypes.capacity() * sizeof(SmallBufferType));
usage.incUsedBytes(_smallArrayTypes.size() * sizeof(SmallBufferType));
return usage;
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
-ArrayStore<EntryT, RefT, TypeMapperT>::~ArrayStore()
+template <typename ElemT, typename RefT, typename TypeMapperT>
+ArrayStore<ElemT, RefT, TypeMapperT>::~ArrayStore()
{
_store.reclaim_all_memory();
_store.dropBuffers();
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
EntryRef
-ArrayStore<EntryT, RefT, TypeMapperT>::add(const ConstArrayRef &array)
+ArrayStore<ElemT, RefT, TypeMapperT>::add(const ConstArrayRef &array)
{
if (array.size() == 0) {
return EntryRef();
@@ -86,9 +86,9 @@ ArrayStore<EntryT, RefT, TypeMapperT>::add(const ConstArrayRef &array)
}
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
EntryRef
-ArrayStore<EntryT, RefT, TypeMapperT>::allocate(size_t array_size)
+ArrayStore<ElemT, RefT, TypeMapperT>::allocate(size_t array_size)
{
if (array_size == 0) {
return EntryRef();
@@ -100,94 +100,93 @@ ArrayStore<EntryT, RefT, TypeMapperT>::allocate(size_t array_size)
}
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
EntryRef
-ArrayStore<EntryT, RefT, TypeMapperT>::addSmallArray(const ConstArrayRef &array)
+ArrayStore<ElemT, RefT, TypeMapperT>::addSmallArray(const ConstArrayRef &array)
{
uint32_t typeId = _mapper.get_type_id(array.size());
- using NoOpReclaimer = DefaultReclaimer<EntryT>;
- return _store.template freeListAllocator<EntryT, NoOpReclaimer>(typeId).allocArray(array).ref;
+ using NoOpReclaimer = DefaultReclaimer<ElemT>;
+ return _store.template freeListAllocator<ElemT, NoOpReclaimer>(typeId).allocArray(array).ref;
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
EntryRef
-ArrayStore<EntryT, RefT, TypeMapperT>::allocate_small_array(size_t array_size)
+ArrayStore<ElemT, RefT, TypeMapperT>::allocate_small_array(size_t array_size)
{
uint32_t type_id = _mapper.get_type_id(array_size);
- return _store.template freeListRawAllocator<EntryT>(type_id).alloc(array_size).ref;
+ return _store.template freeListRawAllocator<ElemT>(type_id).alloc(1).ref;
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
EntryRef
-ArrayStore<EntryT, RefT, TypeMapperT>::addLargeArray(const ConstArrayRef &array)
+ArrayStore<ElemT, RefT, TypeMapperT>::addLargeArray(const ConstArrayRef &array)
{
using NoOpReclaimer = DefaultReclaimer<LargeArray>;
auto handle = _store.template freeListAllocator<LargeArray, NoOpReclaimer>(_largeArrayTypeId)
.alloc(array.cbegin(), array.cend());
auto& state = _store.getBufferState(RefT(handle.ref).bufferId());
- state.stats().inc_extra_used_bytes(sizeof(EntryT) * array.size());
+ state.stats().inc_extra_used_bytes(sizeof(ElemT) * array.size());
return handle.ref;
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
EntryRef
-ArrayStore<EntryT, RefT, TypeMapperT>::allocate_large_array(size_t array_size)
+ArrayStore<ElemT, RefT, TypeMapperT>::allocate_large_array(size_t array_size)
{
using NoOpReclaimer = DefaultReclaimer<LargeArray>;
auto handle = _store.template freeListAllocator<LargeArray, NoOpReclaimer>(_largeArrayTypeId).alloc(array_size);
auto& state = _store.getBufferState(RefT(handle.ref).bufferId());
- state.stats().inc_extra_used_bytes(sizeof(EntryT) * array_size);
+ state.stats().inc_extra_used_bytes(sizeof(ElemT) * array_size);
return handle.ref;
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
void
-ArrayStore<EntryT, RefT, TypeMapperT>::remove(EntryRef ref)
+ArrayStore<ElemT, RefT, TypeMapperT>::remove(EntryRef ref)
{
if (ref.valid()) {
RefT internalRef(ref);
uint32_t typeId = _store.getTypeId(internalRef.bufferId());
if (typeId != _largeArrayTypeId) {
- size_t arraySize = _mapper.get_array_size(typeId);
- _store.holdElem(ref, arraySize);
+ _store.hold_entry(ref);
} else {
- _store.holdElem(ref, 1, sizeof(EntryT) * get(ref).size());
+ _store.hold_entry(ref, sizeof(ElemT) * get(ref).size());
}
}
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
EntryRef
-ArrayStore<EntryT, RefT, TypeMapperT>::move_on_compact(EntryRef ref)
+ArrayStore<ElemT, RefT, TypeMapperT>::move_on_compact(EntryRef ref)
{
return add(get(ref));
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
ICompactionContext::UP
-ArrayStore<EntryT, RefT, TypeMapperT>::compact_worst(const CompactionStrategy &compaction_strategy)
+ArrayStore<ElemT, RefT, TypeMapperT>::compact_worst(const CompactionStrategy &compaction_strategy)
{
auto compacting_buffers = _store.start_compact_worst_buffers(_compaction_spec, compaction_strategy);
return std::make_unique<CompactionContext>(*this, std::move(compacting_buffers));
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
std::unique_ptr<CompactingBuffers>
-ArrayStore<EntryT, RefT, TypeMapperT>::start_compact_worst_buffers(const CompactionStrategy &compaction_strategy)
+ArrayStore<ElemT, RefT, TypeMapperT>::start_compact_worst_buffers(const CompactionStrategy &compaction_strategy)
{
return _store.start_compact_worst_buffers(_compaction_spec, compaction_strategy);
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
vespalib::AddressSpace
-ArrayStore<EntryT, RefT, TypeMapperT>::addressSpaceUsage() const
+ArrayStore<ElemT, RefT, TypeMapperT>::addressSpaceUsage() const
{
return _store.getAddressSpaceUsage();
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
vespalib::MemoryUsage
-ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& compaction_strategy)
+ArrayStore<ElemT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& compaction_strategy)
{
auto address_space_usage = _store.getAddressSpaceUsage();
auto memory_usage = getMemoryUsage();
@@ -195,20 +194,20 @@ ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& com
return memory_usage;
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
const BufferState &
-ArrayStore<EntryT, RefT, TypeMapperT>::bufferState(EntryRef ref)
+ArrayStore<ElemT, RefT, TypeMapperT>::bufferState(EntryRef ref)
{
RefT internalRef(ref);
return _store.getBufferState(internalRef.bufferId());
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
ArrayStoreConfig
-ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId,
+ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId,
size_t hugePageSize,
size_t smallPageSize,
- size_t minNumArraysForNewBuffer,
+ size_t min_num_entries_for_new_buffer,
float allocGrowFactor)
{
TypeMapper mapper;
@@ -216,26 +215,26 @@ ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSm
mapper,
hugePageSize,
smallPageSize,
- minNumArraysForNewBuffer,
+ min_num_entries_for_new_buffer,
allocGrowFactor);
}
-template <typename EntryT, typename RefT, typename TypeMapperT>
+template <typename ElemT, typename RefT, typename TypeMapperT>
ArrayStoreConfig
-ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId,
+ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId,
const TypeMapper& mapper,
size_t hugePageSize,
size_t smallPageSize,
- size_t minNumArraysForNewBuffer,
+ size_t min_num_entries_for_new_buffer,
float allocGrowFactor)
{
return ArrayStoreConfig::optimizeForHugePage(mapper.get_max_small_array_type_id(maxSmallArrayTypeId),
[&](uint32_t type_id) noexcept { return mapper.get_array_size(type_id); },
hugePageSize,
smallPageSize,
- sizeof(EntryT),
+ sizeof(ElemT),
RefT::offsetSize(),
- minNumArraysForNewBuffer,
+ min_num_entries_for_new_buffer,
allocGrowFactor);
}
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp b/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp
index d5587841745..1df9354cd6c 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp
@@ -49,19 +49,19 @@ ArrayStoreConfig::optimizeForHugePage(uint32_t maxSmallArrayTypeId,
std::function<size_t(uint32_t)> type_id_to_array_size,
size_t hugePageSize,
size_t smallPageSize,
- size_t entrySize,
+ size_t elem_size,
size_t maxEntryRefOffset,
- size_t minNumArraysForNewBuffer,
+ size_t min_num_entries_for_new_buffer,
float allocGrowFactor)
{
AllocSpecVector allocSpecs;
- allocSpecs.emplace_back(0, maxEntryRefOffset, minNumArraysForNewBuffer, allocGrowFactor); // large array spec;
+ allocSpecs.emplace_back(0, maxEntryRefOffset, min_num_entries_for_new_buffer, allocGrowFactor); // large array spec;
for (uint32_t type_id = 1; type_id <= maxSmallArrayTypeId; ++type_id) {
size_t arraySize = type_id_to_array_size(type_id);
- size_t numArraysForNewBuffer = hugePageSize / (entrySize * arraySize);
- numArraysForNewBuffer = capToLimits(numArraysForNewBuffer, minNumArraysForNewBuffer, maxEntryRefOffset);
- numArraysForNewBuffer = alignToSmallPageSize(numArraysForNewBuffer, minNumArraysForNewBuffer, smallPageSize);
- allocSpecs.emplace_back(0, maxEntryRefOffset, numArraysForNewBuffer, allocGrowFactor);
+ size_t num_entries_for_new_buffer = hugePageSize / (elem_size * arraySize);
+ num_entries_for_new_buffer = capToLimits(num_entries_for_new_buffer, min_num_entries_for_new_buffer, maxEntryRefOffset);
+ num_entries_for_new_buffer = alignToSmallPageSize(num_entries_for_new_buffer, min_num_entries_for_new_buffer, smallPageSize);
+ allocSpecs.emplace_back(0, maxEntryRefOffset, num_entries_for_new_buffer, allocGrowFactor);
}
return ArrayStoreConfig(allocSpecs);
}
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_config.h b/vespalib/src/vespa/vespalib/datastore/array_store_config.h
index a326c00d042..cae241dba10 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store_config.h
+++ b/vespalib/src/vespa/vespalib/datastore/array_store_config.h
@@ -19,21 +19,21 @@ public:
* Specification of buffer allocation strategy for arrays of a given size.
*/
struct AllocSpec {
- // Minimum number of arrays to allocate in a buffer.
- size_t minArraysInBuffer;
- // Maximum number of arrays to allocate in a buffer.
- size_t maxArraysInBuffer;
- // Number of arrays needed before allocating a new buffer instead of just resizing the first one.
- size_t numArraysForNewBuffer;
+ // Minimum number of entries to allocate in a buffer.
+ size_t min_entries_in_buffer;
+ // Maximum number of entries to allocate in a buffer.
+ size_t max_entries_in_buffer;
+ // Number of entries needed before allocating a new buffer instead of just resizing the first one.
+ size_t num_entries_for_new_buffer;
// Grow factor used when allocating a new buffer.
float allocGrowFactor;
- AllocSpec(size_t minArraysInBuffer_,
- size_t maxArraysInBuffer_,
- size_t numArraysForNewBuffer_,
+ AllocSpec(size_t min_entries_in_buffer_,
+ size_t max_entries_in_buffer_,
+ size_t num_entries_for_new_buffer_,
float allocGrowFactor_) noexcept
- : minArraysInBuffer(minArraysInBuffer_),
- maxArraysInBuffer(maxArraysInBuffer_),
- numArraysForNewBuffer(numArraysForNewBuffer_),
+ : min_entries_in_buffer(min_entries_in_buffer_),
+ max_entries_in_buffer(max_entries_in_buffer_),
+ num_entries_for_new_buffer(num_entries_for_new_buffer_),
allocGrowFactor(allocGrowFactor_) {}
};
@@ -76,9 +76,9 @@ public:
std::function<size_t(uint32_t)> type_id_to_array_size,
size_t hugePageSize,
size_t smallPageSize,
- size_t entrySize,
+ size_t elem_size,
size_t maxEntryRefOffset,
- size_t minNumArraysForNewBuffer,
+ size_t min_num_entries_for_new_buffer,
float allocGrowFactor);
};
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h
index a0cd7827b2d..5f9728a43cc 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h
+++ b/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h
@@ -15,11 +15,11 @@ namespace vespalib::datastore {
*
* A more complex mapping can be used by creating a custom mapper and BufferType implementations.
*/
-template <typename EntryT>
+template <typename ElemT>
class ArrayStoreSimpleTypeMapper {
public:
- using SmallBufferType = SmallArrayBufferType<EntryT>;
- using LargeBufferType = LargeArrayBufferType<EntryT>;
+ using SmallBufferType = SmallArrayBufferType<ElemT>;
+ using LargeBufferType = LargeArrayBufferType<ElemT>;
uint32_t get_type_id(size_t array_size) const { return array_size; }
size_t get_array_size(uint32_t type_id) const { return type_id; }
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h
index e707627de19..2a406a39bf9 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h
+++ b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h
@@ -12,7 +12,7 @@ namespace vespalib::datastore {
* This class provides mapping between type ids and array sizes needed for
* storing a value with size smaller than or equal to the array size.
*
- * The array sizes vector is a monotic increasing sequence that might end
+ * The array sizes vector is a monotonic strictly increasing sequence that might end
* with exponential growth.
*/
class ArrayStoreTypeMapper
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp
index 224ed4b0c8f..851db0222c2 100644
--- a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp
@@ -20,9 +20,8 @@ BufferFreeList::detach()
_free_list->detach(*this);
}
-BufferFreeList::BufferFreeList(std::atomic<ElemCount>& dead_elems)
- : _dead_elems(dead_elems),
- _array_size(0),
+BufferFreeList::BufferFreeList(std::atomic<EntryCount>& dead_entries)
+ : _dead_entries(dead_entries),
_free_list(),
_free_refs()
{
@@ -66,7 +65,7 @@ BufferFreeList::pop_entry() {
if (empty()) {
detach();
}
- _dead_elems.store(_dead_elems.load(std::memory_order_relaxed) - _array_size, std::memory_order_relaxed);
+ _dead_entries.store(_dead_entries.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed);
return ret;
}
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h
index 148ddd8db88..4348a41af04 100644
--- a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h
@@ -19,8 +19,7 @@ class BufferFreeList {
private:
using EntryRefArray = vespalib::Array<EntryRef>;
- std::atomic<ElemCount>& _dead_elems;
- uint32_t _array_size;
+ std::atomic<EntryCount>& _dead_entries;
FreeList* _free_list;
EntryRefArray _free_refs;
@@ -28,7 +27,7 @@ private:
void detach();
public:
- BufferFreeList(std::atomic<ElemCount>& dead_elems);
+ BufferFreeList(std::atomic<EntryCount>& dead_entrie);
~BufferFreeList();
BufferFreeList(BufferFreeList&&) = default; // Needed for emplace_back() during setup.
BufferFreeList(const BufferFreeList&) = delete;
@@ -37,10 +36,8 @@ public:
void enable(FreeList& free_list);
void disable();
- void set_array_size(uint32_t value) { _array_size = value; }
bool enabled() const { return _free_list != nullptr; }
bool empty() const { return _free_refs.empty(); }
- uint32_t array_size() const { return _array_size; }
void push_entry(EntryRef ref);
EntryRef pop_entry();
};
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp
index 8d97414626e..0d96e3f6d47 100644
--- a/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp
@@ -6,27 +6,27 @@
namespace vespalib::datastore {
BufferStats::BufferStats()
- : _alloc_elems(0),
- _used_elems(0),
- _hold_elems(0),
- _dead_elems(0),
+ : _alloc_entries(0),
+ _used_entries(0),
+ _hold_entries(0),
+ _dead_entries(0),
_extra_used_bytes(0),
_extra_hold_bytes(0)
{
}
void
-BufferStats::add_to_mem_stats(size_t element_size, MemoryStats& stats) const
+BufferStats::add_to_mem_stats(size_t entry_size, MemoryStats& stats) const
{
size_t extra_used = extra_used_bytes();
- stats._allocElems += capacity();
- stats._usedElems += size();
- stats._deadElems += dead_elems();
- stats._holdElems += hold_elems();
- stats._allocBytes += (capacity() * element_size) + extra_used;
- stats._usedBytes += (size() * element_size) + extra_used;
- stats._deadBytes += dead_elems() * element_size;
- stats._holdBytes += (hold_elems() * element_size) + extra_hold_bytes();
+ stats._alloc_entries += capacity();
+ stats._used_entries += size();
+ stats._dead_entries += dead_entries();
+ stats._hold_entries += hold_entries();
+ stats._allocBytes += (capacity() * entry_size) + extra_used;
+ stats._usedBytes += (size() * entry_size) + extra_used;
+ stats._deadBytes += dead_entries() * entry_size;
+ stats._holdBytes += (hold_entries() * entry_size) + extra_hold_bytes();
}
InternalBufferStats::InternalBufferStats()
@@ -37,20 +37,20 @@ InternalBufferStats::InternalBufferStats()
void
InternalBufferStats::clear()
{
- _alloc_elems.store(0, std::memory_order_relaxed);
- _used_elems.store(0, std::memory_order_relaxed);
- _hold_elems.store(0, std::memory_order_relaxed);
- _dead_elems.store(0, std::memory_order_relaxed);
+ _alloc_entries.store(0, std::memory_order_relaxed);
+ _used_entries.store(0, std::memory_order_relaxed);
+ _hold_entries.store(0, std::memory_order_relaxed);
+ _dead_entries.store(0, std::memory_order_relaxed);
_extra_used_bytes.store(0, std::memory_order_relaxed);
_extra_hold_bytes.store(0, std::memory_order_relaxed);
}
void
-InternalBufferStats::dec_hold_elems(size_t value)
+InternalBufferStats::dec_hold_entries(size_t value)
{
- ElemCount elems = hold_elems();
+ EntryCount elems = hold_entries();
assert(elems >= value);
- _hold_elems.store(elems - value, std::memory_order_relaxed);
+ _hold_entries.store(elems - value, std::memory_order_relaxed);
}
}
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_stats.h b/vespalib/src/vespa/vespalib/datastore/buffer_stats.h
index 66f8b532c41..1974efa97ec 100644
--- a/vespalib/src/vespa/vespalib/datastore/buffer_stats.h
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_stats.h
@@ -13,16 +13,16 @@ namespace vespalib::datastore {
*/
class BufferStats {
protected:
- // The number of elements that are allocated in the buffer.
- std::atomic<ElemCount> _alloc_elems;
- // The number of elements (of the allocated) that are used: _used_elems <= _alloc_elems.
- std::atomic<ElemCount> _used_elems;
- // The number of elements (of the used) that are on hold: _hold_elems <= _used_elems.
- // "On hold" is a transitionary state used when removing elements.
- std::atomic<ElemCount> _hold_elems;
- // The number of elements (of the used) that are dead: _dead_elems <= _used_elems.
- // A dead element was first on hold, and is now available for reuse in the free list (if enabled).
- std::atomic<ElemCount> _dead_elems;
+ // The number of entries that are allocated in the buffer.
+ std::atomic<EntryCount> _alloc_entries;
+ // The number of entries (of the allocated) that are used: _used_entries <= _alloc_entries.
+ std::atomic<EntryCount> _used_entries;
+ // The number of entries (of the used) that are on hold: _hold_entries <= _used_entries.
+ // "On hold" is a transitionary state used when removing entries.
+ std::atomic<EntryCount> _hold_entries;
+ // The number of entries (of the used) that are dead: _dead_entries <= _used_entries.
+ // A dead entry was first on hold, and is now available for reuse in the free list (if enabled).
+ std::atomic<EntryCount> _dead_entries;
// Number of bytes that are heap allocated (and used) by elements that are stored in this buffer.
// For simple types this is always 0.
@@ -34,22 +34,22 @@ protected:
public:
BufferStats();
- size_t size() const { return _used_elems.load(std::memory_order_relaxed); }
- size_t capacity() const { return _alloc_elems.load(std::memory_order_relaxed); }
+ size_t size() const { return _used_entries.load(std::memory_order_relaxed); }
+ size_t capacity() const { return _alloc_entries.load(std::memory_order_relaxed); }
size_t remaining() const { return capacity() - size(); }
- void pushed_back(size_t num_elems) {
- _used_elems.store(size() + num_elems, std::memory_order_relaxed);
+ void pushed_back(size_t num_entries) {
+ _used_entries.store(size() + num_entries, std::memory_order_relaxed);
}
- size_t dead_elems() const { return _dead_elems.load(std::memory_order_relaxed); }
- size_t hold_elems() const { return _hold_elems.load(std::memory_order_relaxed); }
+ size_t dead_entries() const { return _dead_entries.load(std::memory_order_relaxed); }
+ size_t hold_entries() const { return _hold_entries.load(std::memory_order_relaxed); }
size_t extra_used_bytes() const { return _extra_used_bytes.load(std::memory_order_relaxed); }
size_t extra_hold_bytes() const { return _extra_hold_bytes.load(std::memory_order_relaxed); }
void inc_extra_used_bytes(size_t value) { _extra_used_bytes.store(extra_used_bytes() + value, std::memory_order_relaxed); }
- void add_to_mem_stats(size_t element_size, MemoryStats& stats) const;
+ void add_to_mem_stats(size_t entry_size, MemoryStats& stats) const;
};
/**
@@ -59,15 +59,15 @@ class InternalBufferStats : public BufferStats {
public:
InternalBufferStats();
void clear();
- void set_alloc_elems(size_t value) { _alloc_elems.store(value, std::memory_order_relaxed); }
- void set_dead_elems(size_t value) { _dead_elems.store(value, std::memory_order_relaxed); }
- void set_hold_elems(size_t value) { _hold_elems.store(value, std::memory_order_relaxed); }
- void inc_dead_elems(size_t value) { _dead_elems.store(dead_elems() + value, std::memory_order_relaxed); }
- void inc_hold_elems(size_t value) { _hold_elems.store(hold_elems() + value, std::memory_order_relaxed); }
- void dec_hold_elems(size_t value);
+ void set_alloc_entries(size_t value) { _alloc_entries.store(value, std::memory_order_relaxed); }
+ void set_dead_entries(size_t value) { _dead_entries.store(value, std::memory_order_relaxed); }
+ void set_hold_entries(size_t value) { _hold_entries.store(value, std::memory_order_relaxed); }
+ void inc_dead_entries(size_t value) { _dead_entries.store(dead_entries() + value, std::memory_order_relaxed); }
+ void inc_hold_entries(size_t value) { _hold_entries.store(hold_entries() + value, std::memory_order_relaxed); }
+ void dec_hold_entries(size_t value);
void inc_extra_hold_bytes(size_t value) { _extra_hold_bytes.store(extra_hold_bytes() + value, std::memory_order_relaxed); }
- std::atomic<ElemCount>& used_elems_ref() { return _used_elems; }
- std::atomic<ElemCount>& dead_elems_ref() { return _dead_elems; }
+ std::atomic<EntryCount>& used_entries_ref() { return _used_entries; }
+ std::atomic<EntryCount>& dead_entries_ref() { return _dead_entries; }
std::atomic<size_t>& extra_used_bytes_ref() { return _extra_used_bytes; }
std::atomic<size_t>& extra_hold_bytes_ref() { return _extra_hold_bytes; }
};
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp
index 4a9ba2d33a8..0d43ede9e62 100644
--- a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp
@@ -27,85 +27,85 @@ BufferTypeBase::CleanContext::extraBytesCleaned(size_t value)
}
BufferTypeBase::BufferTypeBase(uint32_t arraySize,
- uint32_t minArrays,
- uint32_t maxArrays,
- uint32_t numArraysForNewBuffer,
+ uint32_t min_entries,
+ uint32_t max_entries,
+ uint32_t num_entries_for_new_buffer,
float allocGrowFactor) noexcept
: _arraySize(arraySize),
- _minArrays(std::min(minArrays, maxArrays)),
- _maxArrays(maxArrays),
- _numArraysForNewBuffer(std::min(numArraysForNewBuffer, maxArrays)),
+ _min_entries(std::min(min_entries, max_entries)),
+ _max_entries(max_entries),
+ _num_entries_for_new_buffer(std::min(num_entries_for_new_buffer, max_entries)),
_allocGrowFactor(allocGrowFactor),
_holdBuffers(0),
- _holdUsedElems(0),
+ _hold_used_entries(0),
_aggr_counts(),
_active_buffers()
{
}
BufferTypeBase::BufferTypeBase(uint32_t arraySize,
- uint32_t minArrays,
- uint32_t maxArrays) noexcept
- : BufferTypeBase(arraySize, minArrays, maxArrays, 0u, DEFAULT_ALLOC_GROW_FACTOR)
+ uint32_t min_entries,
+ uint32_t max_entries) noexcept
+ : BufferTypeBase(arraySize, min_entries, max_entries, 0u, DEFAULT_ALLOC_GROW_FACTOR)
{
}
BufferTypeBase::~BufferTypeBase()
{
assert(_holdBuffers == 0);
- assert(_holdUsedElems == 0);
+ assert(_hold_used_entries == 0);
assert(_aggr_counts.empty());
assert(_active_buffers.empty());
}
-ElemCount
-BufferTypeBase::getReservedElements(uint32_t bufferId) const
+EntryCount
+BufferTypeBase::get_reserved_entries(uint32_t bufferId) const
{
- return bufferId == 0 ? _arraySize : 0u;
+ return bufferId == 0 ? 1u : 0u;
}
void
-BufferTypeBase::onActive(uint32_t bufferId, std::atomic<ElemCount>* usedElems, std::atomic<ElemCount>* deadElems, void* buffer)
+BufferTypeBase::on_active(uint32_t bufferId, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries, void* buffer)
{
- _aggr_counts.add_buffer(usedElems, deadElems);
+ _aggr_counts.add_buffer(used_entries, dead_entries);
assert(std::find(_active_buffers.begin(), _active_buffers.end(), bufferId) == _active_buffers.end());
_active_buffers.emplace_back(bufferId);
- size_t reservedElems = getReservedElements(bufferId);
- if (reservedElems != 0u) {
- initializeReservedElements(buffer, reservedElems);
- *usedElems = reservedElems;
- *deadElems = reservedElems;
+ auto reserved_entries = get_reserved_entries(bufferId);
+ if (reserved_entries != 0u) {
+ initialize_reserved_entries(buffer, reserved_entries);
+ *used_entries = reserved_entries;
+ *dead_entries = reserved_entries;
}
}
void
-BufferTypeBase::onHold(uint32_t buffer_id, const std::atomic<ElemCount>* usedElems, const std::atomic<ElemCount>* deadElems)
+BufferTypeBase::on_hold(uint32_t buffer_id, const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries)
{
++_holdBuffers;
auto itr = std::find(_active_buffers.begin(), _active_buffers.end(), buffer_id);
assert(itr != _active_buffers.end());
_active_buffers.erase(itr);
- _aggr_counts.remove_buffer(usedElems, deadElems);
- _holdUsedElems += *usedElems;
+ _aggr_counts.remove_buffer(used_entries, dead_entries);
+ _hold_used_entries += *used_entries;
}
void
-BufferTypeBase::onFree(ElemCount usedElems)
+BufferTypeBase::on_free(EntryCount used_entries)
{
--_holdBuffers;
- assert(_holdUsedElems >= usedElems);
- _holdUsedElems -= usedElems;
+ assert(_hold_used_entries >= used_entries);
+ _hold_used_entries -= used_entries;
}
void
-BufferTypeBase::resume_primary_buffer(uint32_t buffer_id, std::atomic<ElemCount>* used_elems, std::atomic<ElemCount>* dead_elems)
+BufferTypeBase::resume_primary_buffer(uint32_t buffer_id, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries)
{
auto itr = std::find(_active_buffers.begin(), _active_buffers.end(), buffer_id);
assert(itr != _active_buffers.end());
_active_buffers.erase(itr);
_active_buffers.emplace_back(buffer_id);
- _aggr_counts.remove_buffer(used_elems, dead_elems);
- _aggr_counts.add_buffer(used_elems, dead_elems);
+ _aggr_counts.remove_buffer(used_entries, dead_entries);
+ _aggr_counts.add_buffer(used_entries, dead_entries);
}
const alloc::MemoryAllocator*
@@ -115,17 +115,17 @@ BufferTypeBase::get_memory_allocator() const
}
void
-BufferTypeBase::clampMaxArrays(uint32_t maxArrays)
+BufferTypeBase::clamp_max_entries(uint32_t max_entries)
{
- _maxArrays = std::min(_maxArrays, maxArrays);
- _minArrays = std::min(_minArrays, _maxArrays);
- _numArraysForNewBuffer = std::min(_numArraysForNewBuffer, _maxArrays);
+ _max_entries = std::min(_max_entries, max_entries);
+ _min_entries = std::min(_min_entries, _max_entries);
+ _num_entries_for_new_buffer = std::min(_num_entries_for_new_buffer, _max_entries);
}
size_t
-BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool resizing) const
+BufferTypeBase::calc_entries_to_alloc(uint32_t bufferId, EntryCount free_entries_needed, bool resizing) const
{
- size_t reservedElems = getReservedElements(bufferId);
+ size_t reserved_entries = get_reserved_entries(bufferId);
BufferCounts last_bc;
BufferCounts bc;
if (resizing) {
@@ -134,56 +134,53 @@ BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool
}
}
bc = _aggr_counts.all_buffers();
- assert((bc.used_elems % _arraySize) == 0);
- assert((bc.dead_elems % _arraySize) == 0);
- assert(bc.used_elems >= bc.dead_elems);
- size_t neededArrays = (elemsNeeded + (resizing ? last_bc.used_elems : reservedElems) + _arraySize - 1) / _arraySize;
-
- size_t liveArrays = (bc.used_elems - bc.dead_elems) / _arraySize;
- size_t growArrays = (liveArrays * _allocGrowFactor);
- size_t usedArrays = last_bc.used_elems / _arraySize;
- size_t wantedArrays = std::max((resizing ? usedArrays : 0u) + growArrays,
- static_cast<size_t>(_minArrays));
-
- size_t result = wantedArrays;
- if (result < neededArrays) {
- result = neededArrays;
+ assert(bc.used_entries >= bc.dead_entries);
+ size_t needed_entries = static_cast<size_t>(free_entries_needed) + (resizing ? last_bc.used_entries : reserved_entries);
+ size_t live_entries = (bc.used_entries - bc.dead_entries);
+ size_t grow_entries = (live_entries * _allocGrowFactor);
+ size_t used_entries = last_bc.used_entries;
+ size_t wanted_entries = std::max((resizing ? used_entries : 0u) + grow_entries,
+ static_cast<size_t>(_min_entries));
+
+ size_t result = wanted_entries;
+ if (result < needed_entries) {
+ result = needed_entries;
}
- if (result > _maxArrays) {
- result = _maxArrays;
+ if (result > _max_entries) {
+ result = _max_entries;
}
- if (result < neededArrays) {
+ if (result < needed_entries) {
vespalib::asciistream s;
s << "BufferTypeBase::calcArraysToAlloc(" <<
"bufferId=" << bufferId <<
- ",elemsNeeeded=" << elemsNeeded <<
+ ",free_entries_needed=" << free_entries_needed <<
",resizing=" << (resizing ? "true" : "false") << ")" <<
- " wantedArrays=" << wantedArrays <<
+ " wanted_entries=" << wanted_entries <<
", _arraySize=" << _arraySize <<
- ", _maxArrays=" << _maxArrays <<
- ", reservedElems=" << reservedElems <<
- ", liveArrays=" << liveArrays <<
- ", growArrays=" << growArrays <<
- ", usedArrays=" << usedArrays <<
+ ", _max_entries=" << _max_entries <<
+ ", reserved_entries=" << reserved_entries <<
+ ", live_entries=" << live_entries <<
+ ", grow_entries=" << grow_entries <<
+ ", used_entries=" << used_entries <<
", typeid(*this).name=\"" << typeid(*this).name() << "\"" <<
- ", newArrays=" << result <<
- " < neededArrays=" << neededArrays;;
+ ", new_entries=" << result <<
+ " < needed_entries=" << needed_entries;
throw vespalib::OverflowException(s.c_str());
}
return result;
}
uint32_t
-BufferTypeBase::get_scaled_num_arrays_for_new_buffer() const
+BufferTypeBase::get_scaled_num_entries_for_new_buffer() const
{
uint32_t active_buffers_count = get_active_buffers_count();
- if (active_buffers_count <= 1u || _numArraysForNewBuffer == 0u) {
- return _numArraysForNewBuffer;
+ if (active_buffers_count <= 1u || _num_entries_for_new_buffer == 0u) {
+ return _num_entries_for_new_buffer;
}
double scale_factor = std::pow(1.0 + _allocGrowFactor, active_buffers_count - 1);
- double scaled_result = _numArraysForNewBuffer * scale_factor;
- if (scaled_result >= _maxArrays) {
- return _maxArrays;
+ double scaled_result = _num_entries_for_new_buffer * scale_factor;
+ if (scaled_result >= _max_entries) {
+ return _max_entries;
}
return scaled_result;
}
@@ -194,22 +191,22 @@ BufferTypeBase::AggregatedBufferCounts::AggregatedBufferCounts()
}
void
-BufferTypeBase::AggregatedBufferCounts::add_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems)
+BufferTypeBase::AggregatedBufferCounts::add_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries)
{
for (const auto& elem : _counts) {
- assert(elem.used_ptr != used_elems);
- assert(elem.dead_ptr != dead_elems);
+ assert(elem.used_ptr != used_entries);
+ assert(elem.dead_ptr != dead_entries);
}
- _counts.emplace_back(used_elems, dead_elems);
+ _counts.emplace_back(used_entries, dead_entries);
}
void
-BufferTypeBase::AggregatedBufferCounts::remove_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems)
+BufferTypeBase::AggregatedBufferCounts::remove_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries)
{
auto itr = std::find_if(_counts.begin(), _counts.end(),
- [=](const auto& elem){ return elem.used_ptr == used_elems; });
+ [=](const auto& elem){ return elem.used_ptr == used_entries; });
assert(itr != _counts.end());
- assert(itr->dead_ptr == dead_elems);
+ assert(itr->dead_ptr == dead_entries);
_counts.erase(itr);
}
@@ -219,8 +216,8 @@ BufferTypeBase::AggregatedBufferCounts::last_buffer() const
BufferCounts result;
assert(!_counts.empty());
const auto& last = _counts.back();
- result.used_elems += last.used_ptr->load(std::memory_order_relaxed);
- result.dead_elems += last.dead_ptr->load(std::memory_order_relaxed);
+ result.used_entries += last.used_ptr->load(std::memory_order_relaxed);
+ result.dead_entries += last.dead_ptr->load(std::memory_order_relaxed);
return result;
}
@@ -229,8 +226,8 @@ BufferTypeBase::AggregatedBufferCounts::all_buffers() const
{
BufferCounts result;
for (const auto& elem : _counts) {
- result.used_elems += elem.used_ptr->load(std::memory_order_relaxed);
- result.dead_elems += elem.dead_ptr->load(std::memory_order_relaxed);
+ result.used_entries += elem.used_ptr->load(std::memory_order_relaxed);
+ result.dead_entries += elem.dead_ptr->load(std::memory_order_relaxed);
}
return result;
}
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.h b/vespalib/src/vespa/vespalib/datastore/buffer_type.h
index bedbb2c984e..ea52b026228 100644
--- a/vespalib/src/vespa/vespalib/datastore/buffer_type.h
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.h
@@ -10,7 +10,7 @@ namespace vespalib::alloc { class MemoryAllocator; }
namespace vespalib::datastore {
-using ElemCount = uint64_t;
+using EntryCount = uint32_t;
/**
* Abstract class used to manage allocation and de-allocation of a specific data type in underlying memory buffers in a data store.
@@ -22,7 +22,7 @@ using ElemCount = uint64_t;
class BufferTypeBase
{
public:
- using ElemCount = vespalib::datastore::ElemCount;
+ using EntryCount = vespalib::datastore::EntryCount;
class CleanContext {
private:
std::atomic<size_t> &_extraUsedBytes;
@@ -39,53 +39,52 @@ public:
BufferTypeBase & operator=(const BufferTypeBase &rhs) = delete;
BufferTypeBase(BufferTypeBase &&rhs) noexcept = default;
BufferTypeBase & operator=(BufferTypeBase &&rhs) noexcept = default;
- BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept;
- BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays,
- uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept;
+ BufferTypeBase(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept;
+ BufferTypeBase(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries,
+ uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept;
virtual ~BufferTypeBase();
- virtual void destroyElements(void *buffer, ElemCount numElems) = 0;
- virtual void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) = 0;
+ virtual void destroy_entries(void *buffer, EntryCount num_entries) = 0;
+ virtual void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) = 0;
/**
- * Return number of reserved elements at start of buffer, to avoid
- * invalid reference and handle data at negative offset (alignment
- * hacks) as used by dense tensor store.
+ * Return number of reserved entries at start of buffer, to avoid
+ * invalid reference.
*/
- virtual ElemCount getReservedElements(uint32_t bufferId) const;
+ virtual EntryCount get_reserved_entries(uint32_t bufferId) const;
/**
* Initialize reserved elements at start of buffer.
*/
- virtual void initializeReservedElements(void *buffer, ElemCount reservedElements) = 0;
- virtual size_t elementSize() const = 0;
- virtual void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) = 0;
+ virtual void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) = 0;
+ virtual size_t entry_size() const = 0; // Size of entry measured in bytes
+ virtual void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) = 0;
size_t getArraySize() const { return _arraySize; }
- virtual void onActive(uint32_t bufferId, std::atomic<ElemCount>* usedElems, std::atomic<ElemCount>* deadElems, void* buffer);
- void onHold(uint32_t buffer_id, const std::atomic<ElemCount>* usedElems, const std::atomic<ElemCount>* deadElems);
- virtual void onFree(ElemCount usedElems);
- void resume_primary_buffer(uint32_t buffer_id, std::atomic<ElemCount>* used_elems, std::atomic<ElemCount>* dead_elems);
+ virtual void on_active(uint32_t bufferId, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries, void* buffer);
+ void on_hold(uint32_t buffer_id, const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries);
+ virtual void on_free(EntryCount used_entries);
+ void resume_primary_buffer(uint32_t buffer_id, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries);
virtual const alloc::MemoryAllocator* get_memory_allocator() const;
/**
- * Calculate number of arrays to allocate for new buffer given how many elements are needed.
+ * Calculate number of entries to allocate for new buffer given how many free entries are needed.
*/
- virtual size_t calcArraysToAlloc(uint32_t bufferId, ElemCount elementsNeeded, bool resizing) const;
+ virtual size_t calc_entries_to_alloc(uint32_t bufferId, EntryCount free_entries_needed, bool resizing) const;
- void clampMaxArrays(uint32_t maxArrays);
+ void clamp_max_entries(uint32_t max_entries);
uint32_t get_active_buffers_count() const { return _active_buffers.size(); }
const std::vector<uint32_t>& get_active_buffers() const noexcept { return _active_buffers; }
- size_t getMaxArrays() const { return _maxArrays; }
- uint32_t get_scaled_num_arrays_for_new_buffer() const;
- uint32_t get_num_arrays_for_new_buffer() const noexcept { return _numArraysForNewBuffer; }
+ size_t get_max_entries() const { return _max_entries; }
+ uint32_t get_scaled_num_entries_for_new_buffer() const;
+ uint32_t get_num_entries_for_new_buffer() const noexcept { return _num_entries_for_new_buffer; }
protected:
struct BufferCounts {
- ElemCount used_elems;
- ElemCount dead_elems;
- BufferCounts() : used_elems(0), dead_elems(0) {}
- BufferCounts(ElemCount used_elems_in, ElemCount dead_elems_in)
- : used_elems(used_elems_in), dead_elems(dead_elems_in)
+ EntryCount used_entries;
+ EntryCount dead_entries;
+ BufferCounts() : used_entries(0), dead_entries(0) {}
+ BufferCounts(EntryCount used_entries_in, EntryCount dead_entries_in)
+ : used_entries(used_entries_in), dead_entries(dead_entries_in)
{}
};
@@ -94,45 +93,48 @@ protected:
*/
class AggregatedBufferCounts {
private:
- struct Element {
- const std::atomic<ElemCount>* used_ptr;
- const std::atomic<ElemCount>* dead_ptr;
- Element() noexcept : used_ptr(nullptr), dead_ptr(nullptr) {}
- Element(const std::atomic<ElemCount>* used_ptr_in, const std::atomic<ElemCount>* dead_ptr_in) noexcept
+ struct ActiveBufferCounts {
+ const std::atomic<EntryCount>* used_ptr;
+ const std::atomic<EntryCount>* dead_ptr;
+ ActiveBufferCounts() noexcept : used_ptr(nullptr), dead_ptr(nullptr) {}
+ ActiveBufferCounts(const std::atomic<EntryCount>* used_ptr_in, const std::atomic<EntryCount>* dead_ptr_in) noexcept
: used_ptr(used_ptr_in), dead_ptr(dead_ptr_in)
{}
};
- std::vector<Element> _counts;
+ std::vector<ActiveBufferCounts> _counts;
public:
AggregatedBufferCounts();
- void add_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems);
- void remove_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems);
+ void add_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries);
+ void remove_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries);
BufferCounts last_buffer() const;
BufferCounts all_buffers() const;
bool empty() const { return _counts.empty(); }
};
uint32_t _arraySize; // Number of elements in an allocation unit
- uint32_t _minArrays; // Minimum number of arrays to allocate in a buffer
- uint32_t _maxArrays; // Maximum number of arrays to allocate in a buffer
- // Number of arrays needed before allocating a new buffer instead of just resizing the first one
- uint32_t _numArraysForNewBuffer;
+ uint32_t _min_entries; // Minimum number of entries to allocate in a buffer
+ uint32_t _max_entries; // Maximum number of entries to allocate in a buffer
+ // Number of entries needed before allocating a new buffer instead of just resizing the first one
+ uint32_t _num_entries_for_new_buffer;
float _allocGrowFactor;
uint32_t _holdBuffers;
- size_t _holdUsedElems; // Number of used elements in all held buffers for this type.
+ size_t _hold_used_entries; // Number of used entries in all held buffers for this type.
AggregatedBufferCounts _aggr_counts;
std::vector<uint32_t> _active_buffers;
};
/**
- * Concrete class used to manage allocation and de-allocation of elements of type EntryType in data store buffers.
+ * Concrete class used to manage allocation and de-allocation of elements of type ElemType in data store buffers.
*/
-template <typename EntryType, typename EmptyType = EntryType>
+template <typename ElemT, typename EmptyT = ElemT>
class BufferType : public BufferTypeBase
{
+public:
+ using ElemType = ElemT;
+ using EmptyType = EmptyT;
protected:
- static const EntryType& empty_entry() noexcept;
+ static const ElemType& empty_entry() noexcept;
public:
BufferType() noexcept : BufferType(1,1,1) {}
@@ -140,15 +142,15 @@ public:
BufferType & operator=(const BufferType &rhs) = delete;
BufferType(BufferType && rhs) noexcept = default;
BufferType & operator=(BufferType && rhs) noexcept = default;
- BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept;
- BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays,
- uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept;
+ BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept;
+ BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries,
+ uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept;
~BufferType() override;
- void destroyElements(void *buffer, ElemCount numElems) override;
- void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override;
- void initializeReservedElements(void *buffer, ElemCount reservedElements) override;
- void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCxt) override;
- size_t elementSize() const override { return sizeof(EntryType); }
+ void destroy_entries(void *buffer, EntryCount num_entries) override;
+ void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override;
+ void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override;
+ void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCxt) override;
+ size_t entry_size() const override { return sizeof(ElemType) * _arraySize; }
};
extern template class BufferType<char>;
diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp
index 72c8f574a70..60acca5ff39 100644
--- a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp
@@ -6,78 +6,80 @@
namespace vespalib::datastore {
-template <typename EntryType, typename EmptyType>
-BufferType<EntryType, EmptyType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept
- : BufferTypeBase(arraySize, minArrays, maxArrays)
+template <typename ElemT, typename EmptyT>
+BufferType<ElemT, EmptyT>::BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept
+ : BufferTypeBase(arraySize, min_entries, max_entries)
{ }
-template <typename EntryType, typename EmptyType>
-BufferType<EntryType, EmptyType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays,
- uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept
- : BufferTypeBase(arraySize, minArrays, maxArrays, numArraysForNewBuffer, allocGrowFactor)
+template <typename ElemT, typename EmptyT>
+BufferType<ElemT, EmptyT>::BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries,
+ uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept
+ : BufferTypeBase(arraySize, min_entries, max_entries, num_entries_for_new_buffer, allocGrowFactor)
{ }
-template <typename EntryType, typename EmptyType>
-BufferType<EntryType, EmptyType>::~BufferType() = default;
+template <typename ElemT, typename EmptyT>
+BufferType<ElemT, EmptyT>::~BufferType() = default;
-template <typename EntryType, typename EmptyType>
+template <typename ElemT, typename EmptyT>
void
-BufferType<EntryType, EmptyType>::destroyElements(void *buffer, ElemCount numElems)
+BufferType<ElemT, EmptyT>::destroy_entries(void *buffer, EntryCount num_entries)
{
- EntryType *e = static_cast<EntryType *>(buffer);
- for (size_t j = numElems; j != 0; --j) {
- e->~EntryType();
+ auto num_elems = num_entries * getArraySize();
+ ElemType *e = static_cast<ElemType *>(buffer);
+ for (size_t j = num_elems; j != 0; --j) {
+ e->~ElemType();
++e;
}
}
-template <typename EntryType, typename EmptyType>
+template <typename ElemT, typename EmptyT>
void
-BufferType<EntryType, EmptyType>::fallbackCopy(void *newBuffer,
- const void *oldBuffer,
- ElemCount numElems)
+BufferType<ElemT, EmptyT>::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries)
{
- EntryType *d = static_cast<EntryType *>(newBuffer);
- const EntryType *s = static_cast<const EntryType *>(oldBuffer);
- for (size_t j = numElems; j != 0; --j) {
- new (static_cast<void *>(d)) EntryType(*s);
+ auto num_elems = num_entries * getArraySize();
+ ElemType *d = static_cast<ElemType *>(newBuffer);
+ const ElemType *s = static_cast<const ElemType *>(oldBuffer);
+ for (size_t j = num_elems; j != 0; --j) {
+ new (static_cast<void *>(d)) ElemType(*s);
++s;
++d;
}
}
-template <typename EntryType, typename EmptyType>
+template <typename ElemT, typename EmptyT>
void
-BufferType<EntryType, EmptyType>::initializeReservedElements(void *buffer, ElemCount reservedElems)
+BufferType<ElemT, EmptyT>::initialize_reserved_entries(void *buffer, EntryCount reserved_entries)
{
- EntryType *e = static_cast<EntryType *>(buffer);
+ auto reserved_elems = reserved_entries * getArraySize();
+ ElemType *e = static_cast<ElemType *>(buffer);
const auto& empty = empty_entry();
- for (size_t j = reservedElems; j != 0; --j) {
- new (static_cast<void *>(e)) EntryType(empty);
+ for (size_t j = reserved_elems; j != 0; --j) {
+ new (static_cast<void *>(e)) ElemType(empty);
++e;
}
}
-template <typename EntryType, typename EmptyType>
+template <typename ElemT, typename EmptyT>
void
-BufferType<EntryType, EmptyType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext)
+BufferType<ElemT, EmptyT>::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext)
{
- EntryType *e = static_cast<EntryType *>(buffer) + offset;
+ auto num_elems = num_entries * getArraySize();
+ ElemType *e = static_cast<ElemType *>(buffer) + offset * getArraySize();
const auto& empty = empty_entry();
- for (size_t j = numElems; j != 0; --j) {
+ for (size_t j = num_elems; j != 0; --j) {
*e = empty;
++e;
}
}
-template <typename EntryType, typename EmptyType>
-const EntryType&
-BufferType<EntryType, EmptyType>::empty_entry() noexcept
+template <typename ElemT, typename EmptyT>
+const ElemT&
+BufferType<ElemT, EmptyT>::empty_entry() noexcept
{
- // It's possible for EntryType to wrap e.g. an Alloc instance, which has a transitive
+ // It's possible for ElemType to wrap e.g. an Alloc instance, which has a transitive
// dependency on globally constructed allocator object(s). To avoid issues with global
// construction order, initialize the sentinel on the first access.
- static EntryType empty = EmptyType();
+ static ElemType empty = EmptyType();
return empty;
}
diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp
index 47fba1ef697..f312596d6f7 100644
--- a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp
@@ -12,13 +12,13 @@ namespace vespalib::datastore {
BufferState::BufferState()
: _stats(),
- _free_list(_stats.dead_elems_ref()),
+ _free_list(_stats.dead_entries_ref()),
_typeHandler(nullptr),
_buffer(Alloc::alloc(0, MemoryAllocator::HUGEPAGE_SIZE)),
_arraySize(0),
_typeId(0),
_state(State::FREE),
- _disableElemHoldList(false),
+ _disable_entry_hold_list(false),
_compacting(false)
{
}
@@ -28,15 +28,15 @@ BufferState::~BufferState()
assert(getState() == State::FREE);
assert(!_free_list.enabled());
assert(_free_list.empty());
- assert(_stats.hold_elems() == 0);
+ assert(_stats.hold_entries() == 0);
}
namespace {
struct AllocResult {
- size_t elements;
+ size_t entries;
size_t bytes;
- AllocResult(size_t elements_, size_t bytes_) : elements(elements_), bytes(bytes_) {}
+ AllocResult(size_t entries_, size_t bytes_) : entries(entries_), bytes(bytes_) {}
};
size_t
@@ -57,30 +57,30 @@ roundUpToMatchAllocator(size_t sz)
}
AllocResult
-calcAllocation(uint32_t bufferId,
- BufferTypeBase &typeHandler,
- size_t elementsNeeded,
- bool resizing)
+calc_allocation(uint32_t bufferId,
+ BufferTypeBase &typeHandler,
+ size_t free_entries_needed,
+ bool resizing)
{
- size_t allocArrays = typeHandler.calcArraysToAlloc(bufferId, elementsNeeded, resizing);
- size_t allocElements = allocArrays * typeHandler.getArraySize();
- size_t allocBytes = roundUpToMatchAllocator(allocElements * typeHandler.elementSize());
- size_t maxAllocBytes = typeHandler.getMaxArrays() * typeHandler.getArraySize() * typeHandler.elementSize();
+ size_t alloc_entries = typeHandler.calc_entries_to_alloc(bufferId, free_entries_needed, resizing);
+ size_t entry_size = typeHandler.entry_size();
+ size_t allocBytes = roundUpToMatchAllocator(alloc_entries * entry_size);
+ size_t maxAllocBytes = typeHandler.get_max_entries() * entry_size;
if (allocBytes > maxAllocBytes) {
// Ensure that allocated bytes does not exceed the maximum handled by this type.
allocBytes = maxAllocBytes;
}
- size_t adjustedAllocElements = (allocBytes / typeHandler.elementSize());
- return AllocResult(adjustedAllocElements, allocBytes);
+ size_t adjusted_alloc_entries = allocBytes / entry_size;
+ return AllocResult(adjusted_alloc_entries, allocBytes);
}
}
void
-BufferState::onActive(uint32_t bufferId, uint32_t typeId,
- BufferTypeBase *typeHandler,
- size_t elementsNeeded,
- std::atomic<void*>& buffer)
+BufferState::on_active(uint32_t bufferId, uint32_t typeId,
+ BufferTypeBase *typeHandler,
+ size_t free_entries_needed,
+ std::atomic<void*>& buffer)
{
assert(buffer.load(std::memory_order_relaxed) == nullptr);
assert(_buffer.get() == nullptr);
@@ -88,30 +88,29 @@ BufferState::onActive(uint32_t bufferId, uint32_t typeId,
assert(_typeHandler == nullptr);
assert(capacity() == 0);
assert(size() == 0);
- assert(_stats.dead_elems() == 0u);
- assert(_stats.hold_elems() == 0);
+ assert(_stats.dead_entries() == 0u);
+ assert(_stats.hold_entries() == 0);
assert(_stats.extra_used_bytes() == 0);
assert(_stats.extra_hold_bytes() == 0);
assert(_free_list.empty());
- size_t reservedElements = typeHandler->getReservedElements(bufferId);
- (void) reservedElements;
- AllocResult alloc = calcAllocation(bufferId, *typeHandler, elementsNeeded, false);
- assert(alloc.elements >= reservedElements + elementsNeeded);
+ size_t reserved_entries = typeHandler->get_reserved_entries(bufferId);
+ (void) reserved_entries;
+ AllocResult alloc = calc_allocation(bufferId, *typeHandler, free_entries_needed, false);
+ assert(alloc.entries >= reserved_entries + free_entries_needed);
auto allocator = typeHandler->get_memory_allocator();
_buffer = (allocator != nullptr) ? Alloc::alloc_with_allocator(allocator) : Alloc::alloc(0, MemoryAllocator::HUGEPAGE_SIZE);
_buffer.create(alloc.bytes).swap(_buffer);
- assert(_buffer.get() != nullptr || alloc.elements == 0u);
+ assert(_buffer.get() != nullptr || alloc.entries == 0u);
buffer.store(_buffer.get(), std::memory_order_release);
- _stats.set_alloc_elems(alloc.elements);
+ _stats.set_alloc_entries(alloc.entries);
_typeHandler.store(typeHandler, std::memory_order_release);
assert(typeId <= std::numeric_limits<uint16_t>::max());
_typeId = typeId;
_arraySize = typeHandler->getArraySize();
- _free_list.set_array_size(_arraySize);
_state.store(State::ACTIVE, std::memory_order_release);
- typeHandler->onActive(bufferId, &_stats.used_elems_ref(), &_stats.dead_elems_ref(),
- buffer.load(std::memory_order::relaxed));
+ typeHandler->on_active(bufferId, &_stats.used_entries_ref(), &_stats.dead_entries_ref(),
+ buffer.load(std::memory_order::relaxed));
}
void
@@ -121,11 +120,11 @@ BufferState::onHold(uint32_t buffer_id)
assert(getTypeHandler() != nullptr);
_state.store(State::HOLD, std::memory_order_release);
_compacting = false;
- assert(_stats.dead_elems() <= size());
- assert(_stats.hold_elems() <= (size() - _stats.dead_elems()));
- _stats.set_dead_elems(0);
- _stats.set_hold_elems(size());
- getTypeHandler()->onHold(buffer_id, &_stats.used_elems_ref(), &_stats.dead_elems_ref());
+ assert(_stats.dead_entries() <= size());
+ assert(_stats.hold_entries() <= (size() - _stats.dead_entries()));
+ _stats.set_dead_entries(0);
+ _stats.set_hold_entries(size());
+ getTypeHandler()->on_hold(buffer_id, &_stats.used_entries_ref(), &_stats.dead_entries_ref());
_free_list.disable();
}
@@ -135,20 +134,19 @@ BufferState::onFree(std::atomic<void*>& buffer)
assert(buffer.load(std::memory_order_relaxed) == _buffer.get());
assert(getState() == State::HOLD);
assert(_typeHandler != nullptr);
- assert(_stats.dead_elems() <= size());
- assert(_stats.hold_elems() == (size() - _stats.dead_elems()));
- getTypeHandler()->destroyElements(buffer, size());
+ assert(_stats.dead_entries() <= size());
+ assert(_stats.hold_entries() == (size() - _stats.dead_entries()));
+ getTypeHandler()->destroy_entries(buffer, size());
Alloc::alloc().swap(_buffer);
- getTypeHandler()->onFree(size());
+ getTypeHandler()->on_free(size());
buffer.store(nullptr, std::memory_order_release);
_stats.clear();
_state.store(State::FREE, std::memory_order_release);
_typeHandler = nullptr;
_arraySize = 0;
- _free_list.set_array_size(_arraySize);
assert(!_free_list.enabled());
assert(_free_list.empty());
- _disableElemHoldList = false;
+ _disable_entry_hold_list = false;
}
@@ -171,67 +169,67 @@ BufferState::dropBuffer(uint32_t buffer_id, std::atomic<void*>& buffer)
}
void
-BufferState::disable_elem_hold_list()
+BufferState::disable_entry_hold_list()
{
- _disableElemHoldList = true;
+ _disable_entry_hold_list = true;
}
bool
-BufferState::hold_elems(size_t num_elems, size_t extra_bytes)
+BufferState::hold_entries(size_t num_entries, size_t extra_bytes)
{
assert(isActive());
- if (_disableElemHoldList) {
+ if (_disable_entry_hold_list) {
// The elements are directly marked as dead as they are not put on hold.
- _stats.inc_dead_elems(num_elems);
+ _stats.inc_dead_entries(num_entries);
return true;
}
- _stats.inc_hold_elems(num_elems);
+ _stats.inc_hold_entries(num_entries);
_stats.inc_extra_hold_bytes(extra_bytes);
return false;
}
void
-BufferState::free_elems(EntryRef ref, size_t num_elems, size_t ref_offset)
+BufferState::free_entries(EntryRef ref, size_t num_entries, size_t ref_offset)
{
if (isActive()) {
- if (_free_list.enabled() && (num_elems == getArraySize())) {
+ if (_free_list.enabled() && (num_entries == 1)) {
_free_list.push_entry(ref);
}
} else {
assert(isOnHold());
}
- _stats.inc_dead_elems(num_elems);
- _stats.dec_hold_elems(num_elems);
- getTypeHandler()->cleanHold(_buffer.get(), (ref_offset * _arraySize), num_elems,
- BufferTypeBase::CleanContext(_stats.extra_used_bytes_ref(),
- _stats.extra_hold_bytes_ref()));
+ _stats.inc_dead_entries(num_entries);
+ _stats.dec_hold_entries(num_entries);
+ getTypeHandler()->clean_hold(_buffer.get(), ref_offset, num_entries,
+ BufferTypeBase::CleanContext(_stats.extra_used_bytes_ref(),
+ _stats.extra_hold_bytes_ref()));
}
void
-BufferState::fallbackResize(uint32_t bufferId,
- size_t elementsNeeded,
+BufferState::fallback_resize(uint32_t bufferId,
+ size_t free_entries_needed,
std::atomic<void*>& buffer,
Alloc &holdBuffer)
{
assert(getState() == State::ACTIVE);
assert(_typeHandler != nullptr);
assert(holdBuffer.get() == nullptr);
- AllocResult alloc = calcAllocation(bufferId, *_typeHandler, elementsNeeded, true);
- assert(alloc.elements >= size() + elementsNeeded);
- assert(alloc.elements > capacity());
+ AllocResult alloc = calc_allocation(bufferId, *_typeHandler, free_entries_needed, true);
+ assert(alloc.entries >= size() + free_entries_needed);
+ assert(alloc.entries > capacity());
Alloc newBuffer = _buffer.create(alloc.bytes);
- getTypeHandler()->fallbackCopy(newBuffer.get(), buffer.load(std::memory_order_relaxed), size());
+ getTypeHandler()->fallback_copy(newBuffer.get(), buffer.load(std::memory_order_relaxed), size());
holdBuffer.swap(_buffer);
std::atomic_thread_fence(std::memory_order_release);
_buffer = std::move(newBuffer);
buffer.store(_buffer.get(), std::memory_order_release);
- _stats.set_alloc_elems(alloc.elements);
+ _stats.set_alloc_entries(alloc.entries);
}
void
BufferState::resume_primary_buffer(uint32_t buffer_id)
{
- getTypeHandler()->resume_primary_buffer(buffer_id, &_stats.used_elems_ref(), &_stats.dead_elems_ref());
+ getTypeHandler()->resume_primary_buffer(buffer_id, &_stats.used_entries_ref(), &_stats.dead_entries_ref());
}
}
diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.h b/vespalib/src/vespa/vespalib/datastore/bufferstate.h
index 5b98099ed69..f714f8e24d5 100644
--- a/vespalib/src/vespa/vespalib/datastore/bufferstate.h
+++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.h
@@ -23,8 +23,8 @@ namespace vespalib::datastore {
* It is kept in this state until all reader threads are no longer accessing the buffer.
* Finally, it transitions back to FREE via onFree() and memory is de-allocated.
*
- * This class also supports use of free lists, where previously allocated elements in the buffer can be re-used.
- * First the element is put on hold, then on the free list (counted as dead) to be re-used.
+ * This class also supports use of free lists, where previously allocated entries in the buffer can be re-used.
+ * First the entry is put on hold, then on the free list (counted as dead) to be re-used.
*/
class BufferState
{
@@ -45,7 +45,7 @@ private:
uint32_t _arraySize;
uint16_t _typeId;
std::atomic<State> _state;
- bool _disableElemHoldList : 1;
+ bool _disable_entry_hold_list : 1;
bool _compacting : 1;
public:
@@ -62,14 +62,14 @@ public:
/**
* Transition from FREE to ACTIVE state.
*
- * @param bufferId Id of buffer to be active.
- * @param typeId Registered data type id for buffer.
- * @param typeHandler Type handler for registered data type.
- * @param elementsNeeded Number of elements needed to be free in the memory allocated.
- * @param buffer Start of allocated buffer return value.
+ * @param bufferId Id of buffer to be active.
+ * @param typeId Registered data type id for buffer.
+ * @param typeHandler Type handler for registered data type.
+ * @param free_entries_needed Number of entries needed to be free in the memory allocated.
+ * @param buffer Start of allocated buffer return value.
*/
- void onActive(uint32_t bufferId, uint32_t typeId, BufferTypeBase *typeHandler,
- size_t elementsNeeded, std::atomic<void*>& buffer);
+ void on_active(uint32_t bufferId, uint32_t typeId, BufferTypeBase *typeHandler,
+ size_t free_entries_needed, std::atomic<void*>& buffer);
/**
* Transition from ACTIVE to HOLD state.
@@ -82,24 +82,24 @@ public:
void onFree(std::atomic<void*>& buffer);
/**
- * Disable hold of elements, just mark elements as dead without cleanup.
+ * Disable hold of entries, just mark entries as dead without cleanup.
* Typically used when tearing down data structure in a controlled manner.
*/
- void disable_elem_hold_list();
+ void disable_entry_hold_list();
/**
- * Update stats to reflect that the given elements are put on hold.
- * Returns true if element hold list is disabled for this buffer.
+ * Update stats to reflect that the given entries are put on hold.
+ * Returns true if entry hold list is disabled for this buffer.
*/
- bool hold_elems(size_t num_elems, size_t extra_bytes);
+ bool hold_entries(size_t num_entries, size_t extra_bytes);
/**
- * Free the given elements and update stats accordingly.
+ * Free the given entries and update stats accordingly.
*
* The given entry ref is put on the free list (if enabled).
- * Hold cleaning of elements is executed on the buffer type.
+ * Hold cleaning of entries is executed on the buffer type.
*/
- void free_elems(EntryRef ref, size_t num_elems, size_t ref_offset);
+ void free_entries(EntryRef ref, size_t num_entries, size_t ref_offset);
BufferStats& stats() { return _stats; }
const BufferStats& stats() const { return _stats; }
@@ -115,8 +115,7 @@ public:
uint32_t getArraySize() const { return _arraySize; }
bool getCompacting() const { return _compacting; }
void setCompacting() { _compacting = true; }
- uint32_t get_used_arrays() const noexcept { return size() / _arraySize; }
- void fallbackResize(uint32_t bufferId, size_t elementsNeeded, std::atomic<void*>& buffer, Alloc &holdBuffer);
+ void fallback_resize(uint32_t bufferId, size_t free_entries_needed, std::atomic<void*>& buffer, Alloc &holdBuffer);
bool isActive(uint32_t typeId) const {
return (isActive() && (_typeId == typeId));
diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.h b/vespalib/src/vespa/vespalib/datastore/datastore.h
index 01b81d0fa58..f81348ce287 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastore.h
+++ b/vespalib/src/vespa/vespalib/datastore/datastore.h
@@ -27,7 +27,7 @@ template <typename RefT = EntryRefT<22> >
class DataStoreT : public DataStoreBase
{
private:
- void free_elem_internal(EntryRef ref, size_t numElems);
+ void free_entry_internal(EntryRef ref, size_t num_entries);
public:
using RefType = RefT;
@@ -38,12 +38,12 @@ public:
~DataStoreT() override;
/**
- * Hold element(s).
+ * Hold entries.
*/
- void holdElem(EntryRef ref, size_t numElems) {
- holdElem(ref, numElems, 0);
- }
- void holdElem(EntryRef ref, size_t numElems, size_t extraBytes);
+ void hold_entry(EntryRef ref) { hold_entries(ref, 1, 0); }
+ void hold_entry(EntryRef ref, size_t extra_bytes) { hold_entries(ref, 1, extra_bytes); }
+ void hold_entries(EntryRef ref, size_t num_entries) { hold_entries(ref, num_entries, 0); }
+ void hold_entries(EntryRef ref, size_t num_entries, size_t extraBytes);
void reclaim_entry_refs(generation_t oldest_used_gen) override;
@@ -75,7 +75,7 @@ class DataStore : public DataStoreT<RefT>
{
protected:
using ParentType = DataStoreT<RefT>;
- using ParentType::ensureBufferCapacity;
+ using ParentType::ensure_buffer_capacity;
using ParentType::getEntry;
using ParentType::dropBuffers;
using ParentType::init_primary_buffers;
diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.hpp b/vespalib/src/vespa/vespalib/datastore/datastore.hpp
index bfb63954875..b21a5954eee 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastore.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/datastore.hpp
@@ -22,21 +22,21 @@ DataStoreT<RefT>::~DataStoreT() = default;
template <typename RefT>
void
-DataStoreT<RefT>::free_elem_internal(EntryRef ref, size_t numElems)
+DataStoreT<RefT>::free_entry_internal(EntryRef ref, size_t num_entries)
{
RefType intRef(ref);
BufferState &state = getBufferState(intRef.bufferId());
- state.free_elems(ref, numElems, intRef.offset());
+ state.free_entries(ref, num_entries, intRef.offset());
}
template <typename RefT>
void
-DataStoreT<RefT>::holdElem(EntryRef ref, size_t numElems, size_t extraBytes)
+DataStoreT<RefT>::hold_entries(EntryRef ref, size_t num_entries, size_t extraBytes)
{
RefType intRef(ref);
BufferState &state = getBufferState(intRef.bufferId());
- if (!state.hold_elems(numElems, extraBytes)) {
- _entry_ref_hold_list.insert({ref, numElems});
+ if (!state.hold_entries(num_entries, extraBytes)) {
+ _entry_ref_hold_list.insert({ref, num_entries});
}
}
@@ -45,7 +45,7 @@ void
DataStoreT<RefT>::reclaim_entry_refs(generation_t oldest_used_gen)
{
_entry_ref_hold_list.reclaim(oldest_used_gen, [this](const auto& elem) {
- free_elem_internal(elem.ref, elem.num_elems);
+ free_entry_internal(elem.ref, elem.num_entries);
});
}
@@ -54,7 +54,7 @@ void
DataStoreT<RefT>::reclaim_all_entry_refs()
{
_entry_ref_hold_list.reclaim_all([this](const auto& elem) {
- free_elem_internal(elem.ref, elem.num_elems);
+ free_entry_internal(elem.ref, elem.num_entries);
});
}
diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
index a40aa713bca..75ffe855a32 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
@@ -40,18 +40,18 @@ constexpr size_t TOO_DEAD_SLACK = 0x4000u;
bool
primary_buffer_too_dead(const BufferState &state)
{
- size_t deadElems = state.stats().dead_elems();
- size_t deadBytes = deadElems * state.getArraySize();
- return ((deadBytes >= TOO_DEAD_SLACK) && (deadElems * 2 >= state.size()));
+ size_t dead_entries = state.stats().dead_entries();
+ size_t deadBytes = dead_entries * state.getTypeHandler()->entry_size();
+ return ((deadBytes >= TOO_DEAD_SLACK) && (dead_entries * 2 >= state.size()));
}
}
-DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t usedElems,
+DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t used_entries,
BufferTypeBase *typeHandler, uint32_t typeId)
: GenerationHeldBase(bytesSize),
_buffer(std::move(buffer)),
- _usedElems(usedElems),
+ _used_entries(used_entries),
_typeHandler(typeHandler),
_typeId(typeId)
{
@@ -59,7 +59,7 @@ DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc &
DataStoreBase::FallbackHold::~FallbackHold()
{
- _typeHandler->destroyElements(_buffer.get(), _usedElems);
+ _typeHandler->destroy_entries(_buffer.get(), _used_entries);
}
class DataStoreBase::BufferHold : public GenerationHeldBase {
@@ -80,7 +80,7 @@ public:
}
};
-DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t maxArrays)
+DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t max_entries)
: _entry_ref_hold_list(),
_buffers(numBuffers),
_primary_buffer_ids(),
@@ -89,12 +89,12 @@ DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t m
_free_lists(),
_compaction_count(0u),
_genHolder(),
- _maxArrays(maxArrays),
+ _max_entries(max_entries),
_bufferIdLimit(0u),
_hold_buffer_count(0u),
_offset_bits(offset_bits),
_freeListsEnabled(false),
- _disableElemHoldList(false),
+ _disable_entry_hold_list(false),
_initializing(false)
{
}
@@ -105,19 +105,19 @@ DataStoreBase::~DataStoreBase()
}
void
-DataStoreBase::switch_primary_buffer(uint32_t typeId, size_t elemsNeeded)
+DataStoreBase::switch_primary_buffer(uint32_t typeId, size_t entries_needed)
{
size_t buffer_id = getFirstFreeBufferId();
if (buffer_id >= getMaxNumBuffers()) {
LOG_ABORT(vespalib::make_string("switch_primary_buffer(%u, %zu): did not find a free buffer",
- typeId, elemsNeeded).c_str());
+ typeId, entries_needed).c_str());
}
- onActive(buffer_id, typeId, elemsNeeded);
+ on_active(buffer_id, typeId, entries_needed);
_primary_buffer_ids[typeId] = buffer_id;
}
bool
-DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed)
+DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t entries_needed)
{
auto type_handler = _typeHandlers[type_id];
uint32_t buffer_id = primary_buffer_id(type_id);
@@ -126,7 +126,7 @@ DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed
if (active_buffers_count < min_active_buffers) {
return false;
}
- if (type_handler->get_num_arrays_for_new_buffer() == 0u) {
+ if (type_handler->get_num_entries_for_new_buffer() == 0u) {
return false;
}
assert(!getBufferState(buffer_id).getCompacting());
@@ -146,8 +146,7 @@ DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed
if (checked_active_buffers < min_active_buffers) {
return false;
}
- auto array_size = type_handler->getArraySize();
- if (elems_needed + min_used > type_handler->getMaxArrays() * array_size) {
+ if (entries_needed + min_used > type_handler->get_max_entries()) {
return false;
}
if (min_buffer_id != buffer_id) {
@@ -181,24 +180,22 @@ DataStoreBase::getBufferState(uint32_t buffer_id) noexcept {
}
void
-DataStoreBase::switch_or_grow_primary_buffer(uint32_t typeId, size_t elemsNeeded)
+DataStoreBase::switch_or_grow_primary_buffer(uint32_t typeId, size_t entries_needed)
{
auto typeHandler = _typeHandlers[typeId];
- uint32_t arraySize = typeHandler->getArraySize();
- size_t numArraysForNewBuffer = typeHandler->get_scaled_num_arrays_for_new_buffer();
- size_t numEntriesForNewBuffer = numArraysForNewBuffer * arraySize;
+ size_t num_entries_for_new_buffer = typeHandler->get_scaled_num_entries_for_new_buffer();
uint32_t bufferId = primary_buffer_id(typeId);
- if (elemsNeeded + getBufferState(bufferId).size() >= numEntriesForNewBuffer) {
- if (consider_grow_active_buffer(typeId, elemsNeeded)) {
+ if (entries_needed + getBufferState(bufferId).size() >= num_entries_for_new_buffer) {
+ if (consider_grow_active_buffer(typeId, entries_needed)) {
bufferId = primary_buffer_id(typeId);
- if (elemsNeeded > getBufferState(bufferId).remaining()) {
- fallbackResize(bufferId, elemsNeeded);
+ if (entries_needed > getBufferState(bufferId).remaining()) {
+ fallback_resize(bufferId, entries_needed);
}
} else {
- switch_primary_buffer(typeId, elemsNeeded);
+ switch_primary_buffer(typeId, entries_needed);
}
} else {
- fallbackResize(bufferId, elemsNeeded);
+ fallback_resize(bufferId, entries_needed);
}
}
@@ -209,7 +206,7 @@ DataStoreBase::init_primary_buffers()
for (uint32_t typeId = 0; typeId < numTypes; ++typeId) {
size_t buffer_id = getFirstFreeBufferId();
assert(buffer_id <= get_bufferid_limit_relaxed());
- onActive(buffer_id, typeId, 0u);
+ on_active(buffer_id, typeId, 0u);
_primary_buffer_ids[typeId] = buffer_id;
}
}
@@ -219,7 +216,7 @@ DataStoreBase::addType(BufferTypeBase *typeHandler)
{
uint32_t typeId = _primary_buffer_ids.size();
assert(typeId == _typeHandlers.size());
- typeHandler->clampMaxArrays(_maxArrays);
+ typeHandler->clamp_max_entries(_max_entries);
_primary_buffer_ids.push_back(0);
_typeHandlers.push_back(typeHandler);
_free_lists.emplace_back();
@@ -328,12 +325,12 @@ DataStoreBase::disableFreeLists()
}
void
-DataStoreBase::disableElemHoldList()
+DataStoreBase::disable_entry_hold_list()
{
for_each_buffer([](BufferState & state) {
- if (!state.isFree()) state.disable_elem_hold_list();
+ if (!state.isFree()) state.disable_entry_hold_list();
});
- _disableElemHoldList = true;
+ _disable_entry_hold_list = true;
}
MemoryStats
@@ -351,13 +348,13 @@ DataStoreBase::getMemStats() const
if ((state == BufferState::State::FREE) || (typeHandler == nullptr)) {
++stats._freeBuffers;
} else if (state == BufferState::State::ACTIVE) {
- size_t elementSize = typeHandler->elementSize();
+ size_t entry_size = typeHandler->entry_size();
++stats._activeBuffers;
- bState->stats().add_to_mem_stats(elementSize, stats);
+ bState->stats().add_to_mem_stats(entry_size, stats);
} else if (state == BufferState::State::HOLD) {
- size_t elementSize = typeHandler->elementSize();
+ size_t entry_size = typeHandler->entry_size();
++stats._holdBuffers;
- bState->stats().add_to_mem_stats(elementSize, stats);
+ bState->stats().add_to_mem_stats(entry_size, stats);
} else {
LOG_ABORT("should not be reached");
}
@@ -373,32 +370,30 @@ vespalib::AddressSpace
DataStoreBase::getAddressSpaceUsage() const
{
uint32_t buffer_id_limit = get_bufferid_limit_acquire();
- size_t usedArrays = 0;
- size_t deadArrays = 0;
- size_t limitArrays = size_t(_maxArrays) * (getMaxNumBuffers() - buffer_id_limit);
+ size_t used_entries = 0;
+ size_t dead_entries = 0;
+ size_t limit_entries = size_t(_max_entries) * (getMaxNumBuffers() - buffer_id_limit);
for (uint32_t bufferId = 0; bufferId < buffer_id_limit; ++bufferId) {
const BufferState * bState = _buffers[bufferId].get_state_acquire();
assert(bState != nullptr);
if (bState->isFree()) {
- limitArrays += _maxArrays;
+ limit_entries += _max_entries;
} else if (bState->isActive()) {
- uint32_t arraySize = bState->getArraySize();
- usedArrays += bState->size() / arraySize;
- deadArrays += bState->stats().dead_elems() / arraySize;
- limitArrays += bState->capacity() / arraySize;
+ used_entries += bState->size();
+ dead_entries += bState->stats().dead_entries();
+ limit_entries += bState->capacity();
} else if (bState->isOnHold()) {
- uint32_t arraySize = bState->getArraySize();
- usedArrays += bState->size() / arraySize;
- limitArrays += bState->capacity() / arraySize;
+ used_entries += bState->size();
+ limit_entries += bState->capacity();
} else {
LOG_ABORT("should not be reached");
}
}
- return {usedArrays, deadArrays, limitArrays};
+ return {used_entries, dead_entries, limit_entries};
}
void
-DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded)
+DataStoreBase::on_active(uint32_t bufferId, uint32_t typeId, size_t entries_needed)
{
assert(typeId < _typeHandlers.size());
assert(bufferId <= _bufferIdLimit);
@@ -407,8 +402,8 @@ DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded)
BufferState *state = bufferMeta.get_state_relaxed();
if (state == nullptr) {
BufferState & newState = _stash.create<BufferState>();
- if (_disableElemHoldList) {
- newState.disable_elem_hold_list();
+ if (_disable_entry_hold_list) {
+ newState.disable_entry_hold_list();
}
if ( ! _freeListsEnabled) {
newState.disable_free_list();
@@ -418,7 +413,7 @@ DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded)
_bufferIdLimit.store(bufferId + 1, std::memory_order_release);
}
assert(state->isFree());
- state->onActive(bufferId, typeId, _typeHandlers[typeId], elemsNeeded, bufferMeta.get_atomic_buffer());
+ state->on_active(bufferId, typeId, _typeHandlers[typeId], entries_needed, bufferMeta.get_atomic_buffer());
bufferMeta.setTypeId(typeId);
bufferMeta.setArraySize(state->getArraySize());
if (_freeListsEnabled && state->isActive() && !state->getCompacting()) {
@@ -436,17 +431,17 @@ DataStoreBase::finishCompact(const std::vector<uint32_t> &toHold)
}
void
-DataStoreBase::fallbackResize(uint32_t bufferId, size_t elemsNeeded)
+DataStoreBase::fallback_resize(uint32_t bufferId, size_t entries_needed)
{
BufferState &state = getBufferState(bufferId);
BufferState::Alloc toHoldBuffer;
- size_t oldUsedElems = state.size();
- size_t oldAllocElems = state.capacity();
- size_t elementSize = state.getTypeHandler()->elementSize();
- state.fallbackResize(bufferId, elemsNeeded, _buffers[bufferId].get_atomic_buffer(), toHoldBuffer);
- auto hold = std::make_unique<FallbackHold>(oldAllocElems * elementSize,
+ size_t old_used_entries = state.size();
+ size_t old_alloc_entries = state.capacity();
+ size_t entry_size = state.getTypeHandler()->entry_size();
+ state.fallback_resize(bufferId, entries_needed, _buffers[bufferId].get_atomic_buffer(), toHoldBuffer);
+ auto hold = std::make_unique<FallbackHold>(old_alloc_entries * entry_size,
std::move(toHoldBuffer),
- oldUsedElems,
+ old_used_entries,
state.getTypeHandler(),
state.getTypeId());
if (!_initializing) {
@@ -465,7 +460,7 @@ DataStoreBase::markCompacting(uint32_t bufferId)
}
assert(!state.getCompacting());
state.setCompacting();
- state.disable_elem_hold_list();
+ state.disable_entry_hold_list();
state.disable_free_list();
inc_compaction_count();
}
@@ -492,15 +487,15 @@ DataStoreBase::start_compact_worst_buffers(CompactionSpec compaction_spec, const
free_buffers++;
} else if (state->isActive()) {
auto typeHandler = state->getTypeHandler();
- uint32_t arraySize = typeHandler->getArraySize();
- uint32_t reservedElements = typeHandler->getReservedElements(bufferId);
- size_t used_elems = state->size();
- size_t deadElems = state->stats().dead_elems() - reservedElements;
+ uint32_t reserved_entries = typeHandler->get_reserved_entries(bufferId);
+ size_t used_entries = state->size();
+ size_t dead_entries = state->stats().dead_entries() - reserved_entries;
+ size_t entry_size = typeHandler->entry_size();
if (compaction_spec.compact_memory()) {
- elem_buffers.add(bufferId, used_elems, deadElems);
+ elem_buffers.add(bufferId, used_entries * entry_size, dead_entries * entry_size);
}
if (compaction_spec.compact_address_space()) {
- array_buffers.add(bufferId, used_elems / arraySize, deadElems / arraySize);
+ array_buffers.add(bufferId, used_entries, dead_entries);
}
}
}
diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h
index 9cab7a2e375..e5a38e3fd41 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h
+++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h
@@ -35,15 +35,16 @@ public:
void init_primary_buffers();
/**
- * Ensure that the primary buffer for the given type has a given number of elements free at end.
+ * Ensure that the primary buffer for the given type has a given number of entries free at end.
* Switch to new buffer if current buffer is too full.
*
- * @param typeId Registered data type for buffer.
- * @param elemsNeeded Number of elements needed to be free.
+ * @param typeId Registered data type for buffer.
+ * @param entries_needed Number of entries needed to be free.
*/
- void ensureBufferCapacity(uint32_t typeId, size_t elemsNeeded) {
- if (elemsNeeded > getBufferState(primary_buffer_id(typeId)).remaining()) [[unlikely]] {
- switch_or_grow_primary_buffer(typeId, elemsNeeded);
+ void ensure_buffer_capacity(uint32_t typeId, size_t entries_needed) {
+ auto &state = getBufferState(primary_buffer_id(typeId));
+ if (entries_needed > state.remaining()) [[unlikely]] {
+ switch_or_grow_primary_buffer(typeId, entries_needed);
}
}
@@ -58,10 +59,10 @@ public:
* Switch to a new primary buffer, typically in preparation for compaction
* or when the current primary buffer no longer has free space.
*
- * @param typeId Registered data type for buffer.
- * @param elemsNeeded Number of elements needed to be free.
+ * @param typeId Registered data type for buffer.
+ * @param entries_needed Number of entries needed to be free.
*/
- void switch_primary_buffer(uint32_t typeId, size_t elemsNeeded);
+ void switch_primary_buffer(uint32_t typeId, size_t entries_needed);
vespalib::MemoryUsage getMemoryUsage() const;
vespalib::MemoryUsage getDynamicMemoryUsage() const;
@@ -127,7 +128,7 @@ public:
/**
* Enable free list management.
- * This only works for fixed size elements.
+ * This only works for fixed size entries.
*/
void enableFreeLists();
@@ -135,7 +136,7 @@ public:
* Disable free list management.
*/
void disableFreeLists();
- void disableElemHoldList();
+ void disable_entry_hold_list();
bool has_free_lists_enabled() const { return _freeListsEnabled; }
@@ -177,25 +178,25 @@ public:
bool has_held_buffers() const noexcept { return _hold_buffer_count != 0u; }
/**
- * Trim elem hold list, freeing elements that no longer needs to be held.
+ * Trim entry hold list, freeing entries that no longer needs to be held.
*
* @param oldest_used_gen the oldest generation that is still used.
*/
virtual void reclaim_entry_refs(generation_t oldest_used_gen) = 0;
protected:
- DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t maxArrays);
+ DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t max_entries);
virtual ~DataStoreBase();
void* getBuffer(uint32_t bufferId) { return _buffers[bufferId].get_buffer_relaxed(); }
struct EntryRefHoldElem {
EntryRef ref;
- size_t num_elems;
+ size_t num_entries;
- EntryRefHoldElem(EntryRef ref_in, size_t num_elems_in)
+ EntryRefHoldElem(EntryRef ref_in, size_t num_entries_in)
: ref(ref_in),
- num_elems(num_elems_in)
+ num_entries(num_entries_in)
{}
};
@@ -215,11 +216,11 @@ private:
{
public:
BufferState::Alloc _buffer;
- size_t _usedElems;
+ size_t _used_entries;
BufferTypeBase *_typeHandler;
uint32_t _typeId;
- FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t usedElems,
+ FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t used_entries,
BufferTypeBase *typeHandler, uint32_t typeId);
~FallbackHold() override;
@@ -227,8 +228,8 @@ private:
class BufferHold;
- bool consider_grow_active_buffer(uint32_t type_id, size_t elems_needed);
- void switch_or_grow_primary_buffer(uint32_t typeId, size_t elemsNeeded);
+ bool consider_grow_active_buffer(uint32_t type_id, size_t entries_needed);
+ void switch_or_grow_primary_buffer(uint32_t typeId, size_t entries_needed);
void markCompacting(uint32_t bufferId);
/**
* Hold of buffer has ended.
@@ -238,14 +239,14 @@ private:
/**
* Switch buffer state to active for the given buffer.
*
- * @param bufferId Id of buffer to be active.
- * @param typeId Registered data type for buffer.
- * @param elemsNeeded Number of elements needed to be free.
+ * @param bufferId Id of buffer to be active.
+ * @param typeId Registered data type for buffer.
+ * @param entries_needed Number of entries needed to be free.
*/
- void onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded);
+ void on_active(uint32_t bufferId, uint32_t typeId, size_t entries_needed);
void inc_hold_buffer_count();
- void fallbackResize(uint32_t bufferId, size_t elementsNeeded);
+ void fallback_resize(uint32_t bufferId, size_t entries_needed);
uint32_t getFirstFreeBufferId();
template<typename FuncType>
@@ -261,7 +262,7 @@ private:
std::vector<BufferAndMeta> _buffers; // For fast mapping with known types
// Provides a mapping from typeId -> primary buffer for that type.
- // The primary buffer is used for allocations of new element(s) if no available slots are found in free lists.
+ // The primary buffer is used for allocations of new entries if no available slots are found in free lists.
std::vector<uint32_t> _primary_buffer_ids;
Stash _stash;
@@ -269,12 +270,12 @@ private:
std::vector<FreeList> _free_lists;
mutable std::atomic<uint64_t> _compaction_count;
vespalib::GenerationHolder _genHolder;
- const uint32_t _maxArrays;
+ const uint32_t _max_entries;
std::atomic<uint32_t> _bufferIdLimit;
uint32_t _hold_buffer_count;
const uint8_t _offset_bits;
bool _freeListsEnabled;
- bool _disableElemHoldList;
+ bool _disable_entry_hold_list;
bool _initializing;
};
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list.h b/vespalib/src/vespa/vespalib/datastore/free_list.h
index 20d4a6b96df..cd475af3104 100644
--- a/vespalib/src/vespa/vespalib/datastore/free_list.h
+++ b/vespalib/src/vespa/vespalib/datastore/free_list.h
@@ -30,7 +30,6 @@ public:
bool empty() const { return _free_lists.empty(); }
size_t size() const { return _free_lists.size(); }
- uint32_t array_size() const { return _free_lists.back()->array_size(); }
EntryRef pop_entry() {
return _free_lists.back()->pop_entry();
}
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h
index cf899a76712..dc2d1ea3c34 100644
--- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h
+++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h
@@ -29,7 +29,7 @@ public:
HandleType alloc(Args && ... args);
HandleType allocArray(ConstArrayRef array);
- HandleType allocArray(size_t size);
+ HandleType allocArray();
};
}
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp
index b793e4f77a2..4e69db08a3c 100644
--- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp
@@ -71,8 +71,9 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(ConstArrayRef array)
if (free_list.empty()) {
return ParentType::allocArray(array);
}
- assert(free_list.array_size() == array.size());
RefT ref = free_list.pop_entry();
+ auto& state = _store.getBufferState(ref.bufferId());
+ assert(state.getArraySize() == array.size());
EntryT *buf = _store.template getEntryArray<EntryT>(ref, array.size());
for (size_t i = 0; i < array.size(); ++i) {
*(buf + i) = array[i];
@@ -82,14 +83,15 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(ConstArrayRef array)
template <typename EntryT, typename RefT, typename ReclaimerT>
typename Allocator<EntryT, RefT>::HandleType
-FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(size_t size)
+FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray()
{
auto& free_list = _store.getFreeList(_typeId);
if (free_list.empty()) {
- return ParentType::allocArray(size);
+ return ParentType::allocArray();
}
- assert(free_list.array_size() == size);
RefT ref = free_list.pop_entry();
+ auto& state = _store.getBufferState(ref.bufferId());
+ auto size = state.getArraySize();
EntryT *buf = _store.template getEntryArray<EntryT>(ref, size);
return HandleType(ref, buf);
}
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h
index 1b71c22f0ce..29684267546 100644
--- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h
+++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h
@@ -27,7 +27,7 @@ private:
public:
FreeListRawAllocator(DataStoreBase &store, uint32_t typeId);
- HandleType alloc(size_t numElems);
+ HandleType alloc(size_t num_entries);
};
}
diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp
index af832955cb7..7680cd8a9a5 100644
--- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp
@@ -14,16 +14,18 @@ FreeListRawAllocator<EntryT, RefT>::FreeListRawAllocator(DataStoreBase &store, u
template <typename EntryT, typename RefT>
typename FreeListRawAllocator<EntryT, RefT>::HandleType
-FreeListRawAllocator<EntryT, RefT>::alloc(size_t numElems)
+FreeListRawAllocator<EntryT, RefT>::alloc(size_t num_entries)
{
auto& free_list = _store.getFreeList(_typeId);
if (free_list.empty()) {
- return ParentType::alloc(numElems);
+ return ParentType::alloc(num_entries);
}
- assert(free_list.array_size() == numElems);
+ assert(num_entries == 1);
RefT ref = free_list.pop_entry();
+ auto& state = _store.getBufferState(ref.bufferId());
+ auto array_size = state.getArraySize();
// We must scale the offset according to array size as it was divided when the entry ref was created.
- EntryT *entry = _store.template getEntryArray<EntryT>(ref, numElems);
+ EntryT *entry = _store.template getEntryArray<EntryT>(ref, array_size);
return HandleType(ref, entry);
}
diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h
index 600925969a3..e2718b94cd2 100644
--- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h
+++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h
@@ -14,11 +14,11 @@ namespace vespalib::datastore {
/*
* Class representing buffer type for large arrays in ArrayStore
*/
-template <typename EntryT>
-class LargeArrayBufferType : public BufferType<Array<EntryT>>
+template <typename ElemT>
+class LargeArrayBufferType : public BufferType<Array<ElemT>>
{
using AllocSpec = ArrayStoreConfig::AllocSpec;
- using ArrayType = Array<EntryT>;
+ using ArrayType = Array<ElemT>;
using ParentType = BufferType<ArrayType>;
using ParentType::empty_entry;
using CleanContext = typename ParentType::CleanContext;
@@ -31,7 +31,7 @@ public:
{
}
~LargeArrayBufferType() override;
- void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override;
+ void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override;
const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override;
};
diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp
index 3042bbff73f..72a2662991b 100644
--- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp
@@ -7,32 +7,32 @@
namespace vespalib::datastore {
-template <typename EntryT>
-LargeArrayBufferType<EntryT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept
- : BufferType<Array<EntryT>>(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor),
+template <typename ElemT>
+LargeArrayBufferType<ElemT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept
+ : BufferType<Array<ElemT>>(1u, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor),
_memory_allocator(std::move(memory_allocator))
{
}
-template <typename EntryT>
-LargeArrayBufferType<EntryT>::~LargeArrayBufferType() = default;
+template <typename ElemT>
+LargeArrayBufferType<ElemT>::~LargeArrayBufferType() = default;
-template <typename EntryT>
+template <typename ElemT>
void
-LargeArrayBufferType<EntryT>::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx)
+LargeArrayBufferType<ElemT>::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx)
{
ArrayType* elem = static_cast<ArrayType*>(buffer) + offset;
const auto& empty = empty_entry();
- for (size_t i = 0; i < numElems; ++i) {
- cleanCtx.extraBytesCleaned(sizeof(EntryT) * elem->size());
+ for (size_t i = 0; i < num_entries; ++i) {
+ cleanCtx.extraBytesCleaned(sizeof(ElemT) * elem->size());
*elem = empty;
++elem;
}
}
-template <typename EntryT>
+template <typename ElemT>
const vespalib::alloc::MemoryAllocator*
-LargeArrayBufferType<EntryT>::get_memory_allocator() const
+LargeArrayBufferType<ElemT>::get_memory_allocator() const
{
return _memory_allocator.get();
}
diff --git a/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp b/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp
index 8e060b4cfb4..5cb04796c5b 100644
--- a/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp
@@ -5,10 +5,10 @@
namespace vespalib::datastore {
MemoryStats::MemoryStats()
- : _allocElems(0),
- _usedElems(0),
- _deadElems(0),
- _holdElems(0),
+ : _alloc_entries(0),
+ _used_entries(0),
+ _dead_entries(0),
+ _hold_entries(0),
_allocBytes(0),
_usedBytes(0),
_deadBytes(0),
@@ -22,10 +22,10 @@ MemoryStats::MemoryStats()
MemoryStats&
MemoryStats::operator+=(const MemoryStats& rhs)
{
- _allocElems += rhs._allocElems;
- _usedElems += rhs._usedElems;
- _deadElems += rhs._deadElems;
- _holdElems += rhs._holdElems;
+ _alloc_entries += rhs._alloc_entries;
+ _used_entries += rhs._used_entries;
+ _dead_entries += rhs._dead_entries;
+ _hold_entries += rhs._hold_entries;
_allocBytes += rhs._allocBytes;
_usedBytes += rhs._usedBytes;
_deadBytes += rhs._deadBytes;
diff --git a/vespalib/src/vespa/vespalib/datastore/memory_stats.h b/vespalib/src/vespa/vespalib/datastore/memory_stats.h
index 18d7dd77559..72a570dd625 100644
--- a/vespalib/src/vespa/vespalib/datastore/memory_stats.h
+++ b/vespalib/src/vespa/vespalib/datastore/memory_stats.h
@@ -13,10 +13,10 @@ namespace vespalib::datastore {
class MemoryStats
{
public:
- size_t _allocElems;
- size_t _usedElems;
- size_t _deadElems;
- size_t _holdElems;
+ size_t _alloc_entries;
+ size_t _used_entries;
+ size_t _dead_entries;
+ size_t _hold_entries;
size_t _allocBytes;
size_t _usedBytes;
size_t _deadBytes;
diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.h b/vespalib/src/vespa/vespalib/datastore/raw_allocator.h
index c10c8152e72..e7a59fadcf8 100644
--- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.h
+++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.h
@@ -25,10 +25,10 @@ protected:
public:
RawAllocator(DataStoreBase &store, uint32_t typeId);
- HandleType alloc(size_t numElems) {
- return alloc(numElems, 0);
+ HandleType alloc(size_t num_entries) {
+ return alloc(num_entries, 0);
}
- HandleType alloc(size_t numElems, size_t extraElems);
+ HandleType alloc(size_t num_entries, size_t extra_entries);
};
}
diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp
index 04d99588218..9de361a8b19 100644
--- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp
@@ -16,19 +16,15 @@ RawAllocator<EntryT, RefT>::RawAllocator(DataStoreBase &store, uint32_t typeId)
template <typename EntryT, typename RefT>
typename RawAllocator<EntryT, RefT>::HandleType
-RawAllocator<EntryT, RefT>::alloc(size_t numElems, size_t extraElems)
+RawAllocator<EntryT, RefT>::alloc(size_t num_entries, size_t extra_entries)
{
- _store.ensureBufferCapacity(_typeId, numElems + extraElems);
+ _store.ensure_buffer_capacity(_typeId, num_entries + extra_entries);
uint32_t buffer_id = _store.primary_buffer_id(_typeId);
BufferState &state = _store.getBufferState(buffer_id);
assert(state.isActive());
- size_t oldBufferSize = state.size();
- // Must perform scaling ourselves, according to array size
- size_t arraySize = state.getArraySize();
- assert((numElems % arraySize) == 0u);
- RefT ref((oldBufferSize / arraySize), buffer_id);
- EntryT *buffer = _store.getEntryArray<EntryT>(ref, arraySize);
- state.stats().pushed_back(numElems);
+ RefT ref(state.size(), buffer_id);
+ EntryT *buffer = _store.getEntryArray<EntryT>(ref, state.getArraySize());
+ state.stats().pushed_back(num_entries);
return HandleType(ref, buffer);
}
diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h
index 676a9d3790f..6bb6601839d 100644
--- a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h
+++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h
@@ -13,8 +13,8 @@ namespace vespalib::datastore {
/*
* Class representing buffer type for small arrays in ArrayStore
*/
-template <typename EntryT>
-class SmallArrayBufferType : public BufferType<EntryT>
+template <typename ElemT>
+class SmallArrayBufferType : public BufferType<ElemT>
{
using AllocSpec = ArrayStoreConfig::AllocSpec;
std::shared_ptr<alloc::MemoryAllocator> _memory_allocator;
diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp
index 414804417eb..c9033936bd6 100644
--- a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp
@@ -6,19 +6,19 @@
namespace vespalib::datastore {
-template <typename EntryT>
-SmallArrayBufferType<EntryT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept
- : BufferType<EntryT>(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor),
+template <typename ElemT>
+SmallArrayBufferType<ElemT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept
+ : BufferType<ElemT>(array_size, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor),
_memory_allocator(std::move(memory_allocator))
{
}
-template <typename EntryT>
-SmallArrayBufferType<EntryT>::~SmallArrayBufferType() = default;
+template <typename ElemT>
+SmallArrayBufferType<ElemT>::~SmallArrayBufferType() = default;
-template <typename EntryT>
+template <typename ElemT>
const vespalib::alloc::MemoryAllocator*
-SmallArrayBufferType<EntryT>::get_memory_allocator() const
+SmallArrayBufferType<ElemT>::get_memory_allocator() const
{
return _memory_allocator.get();
}
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
index 52b0798543f..0efaf04b26e 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
@@ -106,7 +106,7 @@ private:
_mapping.resize(data_store.get_bufferid_limit_relaxed());
for (const auto bufferId : _compacting_buffers->get_buffer_ids()) {
BufferState &state = data_store.getBufferState(bufferId);
- _mapping[bufferId].resize(state.get_used_arrays());
+ _mapping[bufferId].resize(state.size());
}
}
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp
index 8ad11b18218..49d2631e73f 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp
@@ -43,7 +43,7 @@ template <typename EntryT, typename RefT>
void
UniqueStoreAllocator<EntryT, RefT>::hold(EntryRef ref)
{
- _store.holdElem(ref, 1);
+ _store.hold_entry(ref);
}
template <typename EntryT, typename RefT>
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp
index 32513d09c72..4a517521d77 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp
@@ -44,7 +44,7 @@ UniqueStoreEnumerator<RefT>::allocate_enum_values(DataStoreBase & store)
{
_enumValues.resize(store.get_bufferid_limit_relaxed());
store.for_each_active_buffer([this](uint32_t buffer_id, const BufferState & state) {
- _enumValues[buffer_id].resize(state.get_used_arrays());
+ _enumValues[buffer_id].resize(state.size());
});
}
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp
index 1d3ba27d6bf..9f2105b7f08 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp
@@ -42,28 +42,28 @@ UniqueStoreSmallStringBufferType::UniqueStoreSmallStringBufferType(uint32_t arra
UniqueStoreSmallStringBufferType::~UniqueStoreSmallStringBufferType() = default;
void
-UniqueStoreSmallStringBufferType::destroyElements(void *, ElemCount)
+UniqueStoreSmallStringBufferType::destroy_entries(void *, EntryCount)
{
static_assert(std::is_trivially_destructible<UniqueStoreSmallStringEntry>::value,
"UniqueStoreSmallStringEntry must be trivially destructable");
}
void
-UniqueStoreSmallStringBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems)
+UniqueStoreSmallStringBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries)
{
static_assert(std::is_trivially_copyable<UniqueStoreSmallStringEntry>::value,
"UniqueStoreSmallStringEntry must be trivially copyable");
- if (numElems > 0) {
- memcpy(newBuffer, oldBuffer, numElems);
+ if (num_entries > 0) {
+ memcpy(newBuffer, oldBuffer, num_entries * getArraySize());
}
}
void
-UniqueStoreSmallStringBufferType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext)
+UniqueStoreSmallStringBufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext)
{
- void *e = static_cast<char *>(buffer) + offset;
- void *e_end = static_cast<char *>(e) + numElems;
size_t array_size = getArraySize();
+ void *e = static_cast<char *>(buffer) + offset * array_size;
+ void *e_end = static_cast<char *>(e) + num_entries * array_size;
while (e < e_end) {
static_cast<UniqueStoreSmallStringEntry *>(e)->clean_hold(array_size);
e = static_cast<char *>(e) + array_size;
@@ -86,10 +86,10 @@ UniqueStoreExternalStringBufferType::UniqueStoreExternalStringBufferType(uint32_
UniqueStoreExternalStringBufferType::~UniqueStoreExternalStringBufferType() = default;
void
-UniqueStoreExternalStringBufferType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx)
+UniqueStoreExternalStringBufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx)
{
UniqueStoreEntry<std::string> *elem = static_cast<UniqueStoreEntry<std::string> *>(buffer) + offset;
- for (size_t i = 0; i < numElems; ++i) {
+ for (size_t i = 0; i < num_entries; ++i) {
cleanCtx.extraBytesCleaned(elem->value().size() + 1);
std::string().swap(elem->value());
++elem;
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h
index a85b73f423d..d3348950891 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h
@@ -62,9 +62,9 @@ class UniqueStoreSmallStringBufferType : public BufferType<char> {
public:
UniqueStoreSmallStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator);
~UniqueStoreSmallStringBufferType() override;
- void destroyElements(void *, ElemCount) override;
- void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override;
- void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) override;
+ void destroy_entries(void *, EntryCount) override;
+ void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount numElems) override;
+ void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) override;
const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override;
};
@@ -76,7 +76,7 @@ class UniqueStoreExternalStringBufferType : public BufferType<UniqueStoreEntry<s
public:
UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator);
~UniqueStoreExternalStringBufferType() override;
- void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override;
+ void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override;
const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override;
};
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp
index 65cab4850ba..4ff8e1e1ab4 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp
@@ -42,7 +42,7 @@ UniqueStoreStringAllocator<RefT>::allocate(const char *value)
uint32_t type_id = string_allocator::get_type_id(value_len);
if (type_id != 0) {
size_t array_size = string_allocator::array_sizes[type_id - 1];
- auto handle = _store.template freeListRawAllocator<char>(type_id).alloc(array_size);
+ auto handle = _store.template freeListRawAllocator<char>(type_id).alloc(1);
new (static_cast<void *>(handle.data)) UniqueStoreSmallStringEntry(value, value_len, array_size);
return handle.ref;
} else {
@@ -61,11 +61,10 @@ UniqueStoreStringAllocator<RefT>::hold(EntryRef ref)
RefT iRef(ref);
uint32_t type_id = _store.getTypeId(iRef.bufferId());
if (type_id != 0) {
- size_t array_size = string_allocator::array_sizes[type_id - 1];
- _store.holdElem(ref, array_size);
+ _store.hold_entry(ref);
} else {
auto &value = _store.template getEntry<WrappedExternalEntryType>(iRef)->value();
- _store.holdElem(ref, 1, value.size() + 1);
+ _store.hold_entry(ref, value.size() + 1);
}
}
@@ -79,7 +78,7 @@ UniqueStoreStringAllocator<RefT>::move_on_compact(EntryRef ref)
static_assert(std::is_trivially_copyable<UniqueStoreSmallStringEntry>::value,
"UniqueStoreSmallStringEntry must be trivially copyable");
size_t array_size = string_allocator::array_sizes[type_id - 1];
- auto handle = _store.template rawAllocator<char>(type_id).alloc(array_size);
+ auto handle = _store.template rawAllocator<char>(type_id).alloc(1);
auto orig = _store.template getEntryArray<char>(iRef, array_size);
memcpy(handle.data, orig, array_size);
return handle.ref;
diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt
index 663b3b65638..8ee3957af32 100644
--- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt
@@ -100,3 +100,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT
zstdcompressor.cpp
DEPENDS
)
+
+# Don't convert call to jump when returning a value from a function with
+# a compatible stack.
+set_source_files_properties(signalhandler.cpp PROPERTIES COMPILE_OPTIONS "-fno-optimize-sibling-calls")
diff --git a/vespalib/src/vespa/vespalib/util/time.cpp b/vespalib/src/vespa/vespalib/util/time.cpp
index cba26f24059..dd4972b1c21 100644
--- a/vespalib/src/vespa/vespalib/util/time.cpp
+++ b/vespalib/src/vespa/vespalib/util/time.cpp
@@ -93,7 +93,7 @@ Timer::waitAtLeast(duration dur, bool busyWait) {
}
-#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12)
+#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12)
// Temporary workaround until libc++ supports stream operators for duration
// Temporary workaround while using libstdc++ 11
diff --git a/vespalib/src/vespa/vespalib/util/time.h b/vespalib/src/vespa/vespalib/util/time.h
index b893661832f..10077a0b49a 100644
--- a/vespalib/src/vespa/vespalib/util/time.h
+++ b/vespalib/src/vespa/vespalib/util/time.h
@@ -101,7 +101,7 @@ duration adjustTimeoutByHz(duration timeout, long hz);
}
-#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12)
+#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12)
// Temporary workaround until libc++ supports stream operators for duration
// Temporary workaround while using libstdc++ 11
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java
index af42e30422b..0b2595b6af8 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java
@@ -2,20 +2,28 @@
package com.yahoo.vespa.zookeeper;
import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.cloud.config.ZookeeperServerConfig.Server;
import com.yahoo.security.tls.ConfigFileBasedTlsContext;
import com.yahoo.security.tls.MixedMode;
import com.yahoo.security.tls.TlsContext;
import com.yahoo.security.tls.TransportSecurityUtils;
+
import java.io.FileWriter;
import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.logging.Level;
import java.util.stream.Collectors;
+import static com.yahoo.stream.CustomCollectors.toLinkedMap;
import static com.yahoo.vespa.defaults.Defaults.getDefaults;
public class Configurator {
@@ -64,7 +72,6 @@ public class Configurator {
// override of Vespa TLS config for unit testing
void writeConfigToDisk(VespaTlsConfig vespaTlsConfig) {
configFilePath.toFile().getParentFile().mkdirs();
-
try {
writeZooKeeperConfigFile(zookeeperServerConfig, vespaTlsConfig);
writeMyIdFile(zookeeperServerConfig);
@@ -75,36 +82,75 @@ public class Configurator {
private void writeZooKeeperConfigFile(ZookeeperServerConfig config,
VespaTlsConfig vespaTlsConfig) throws IOException {
+ String dynamicConfigPath = config.dynamicReconfiguration() ? parseConfigFile(configFilePath).get("dynamicConfigFile") : null;
+ Map<String, String> dynamicConfig = dynamicConfigPath != null ? parseConfigFile(Paths.get(dynamicConfigPath)) : Map.of();
try (FileWriter writer = new FileWriter(configFilePath.toFile())) {
- writer.write(transformConfigToString(config, vespaTlsConfig));
+ writer.write(transformConfigToString(config, vespaTlsConfig, dynamicConfig));
}
}
- private String transformConfigToString(ZookeeperServerConfig config, VespaTlsConfig vespaTlsConfig) {
- StringBuilder sb = new StringBuilder();
- sb.append("tickTime=").append(config.tickTime()).append("\n");
- sb.append("initLimit=").append(config.initLimit()).append("\n");
- sb.append("syncLimit=").append(config.syncLimit()).append("\n");
- sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n");
- sb.append("snapCount=").append(config.snapshotCount()).append("\n");
- sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n");
- sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n");
- sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n");
+ private String transformConfigToString(ZookeeperServerConfig config, VespaTlsConfig vespaTlsConfig, Map<String, String> dynamicConfig) {
+ Map<String, String> configEntries = new LinkedHashMap<>();
+ configEntries.put("tickTime", Integer.toString(config.tickTime()));
+ configEntries.put("initLimit", Integer.toString(config.initLimit()));
+ configEntries.put("syncLimit", Integer.toString(config.syncLimit()));
+ configEntries.put("maxClientCnxns", Integer.toString(config.maxClientConnections()));
+ configEntries.put("snapCount", Integer.toString(config.snapshotCount()));
+ configEntries.put("dataDir", getDefaults().underVespaHome(config.dataDir()));
+ configEntries.put("autopurge.purgeInterval", Integer.toString(config.autopurge().purgeInterval()));
+ configEntries.put("autopurge.snapRetainCount", Integer.toString(config.autopurge().snapRetainCount()));
// See http://zookeeper.apache.org/doc/r3.6.3/zookeeperAdmin.html#sc_zkCommands
// Includes all available commands in 3.6, except 'wchc' and 'wchp'
- sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
- sb.append("admin.enableServer=false").append("\n");
+ configEntries.put("4lw.commands.whitelist", "conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs");
+ configEntries.put("admin.enableServer", "false");
// Use custom connection factory for TLS on client port - see class' Javadoc for rationale
- sb.append("serverCnxnFactory=org.apache.zookeeper.server.VespaNettyServerCnxnFactory").append("\n");
- sb.append("quorumListenOnAllIPs=true").append("\n");
- sb.append("standaloneEnabled=false").append("\n");
- sb.append("reconfigEnabled=").append(config.dynamicReconfiguration()).append("\n");
- sb.append("skipACL=yes").append("\n");
- ensureThisServerIsRepresented(config.myid(), config.server());
- config.server().forEach(server -> sb.append(serverSpec(server, server.joining())).append("\n"));
- sb.append(new TlsQuorumConfig().createConfig(vespaTlsConfig));
- sb.append(new TlsClientServerConfig().createConfig(vespaTlsConfig));
- return sb.toString();
+ configEntries.put("serverCnxnFactory", "org.apache.zookeeper.server.VespaNettyServerCnxnFactory");
+ configEntries.put("quorumListenOnAllIPs", "true");
+ configEntries.put("standaloneEnabled", "false");
+ configEntries.put("reconfigEnabled", Boolean.toString(config.dynamicReconfiguration()));
+ configEntries.put("skipACL", "yes");
+
+ addServerSpecs(configEntries, config, dynamicConfig);
+
+ new TlsQuorumConfig().createConfig(configEntries, vespaTlsConfig);
+ new TlsClientServerConfig().createConfig(configEntries, vespaTlsConfig);
+ return transformConfigToString(configEntries);
+ }
+
+ void addServerSpecs(Map<String, String> configEntries, ZookeeperServerConfig config, Map<String, String> dynamicConfig) {
+ int myIndex = ensureThisServerIsRepresented(config.myid(), config.server());
+
+ // If dynamic config refers to servers that are not in the current config, we must ignore it.
+ Set<String> currentServers = config.server().stream().map(Server::hostname).collect(Collectors.toSet());
+ if (dynamicConfig.values().stream().anyMatch(spec -> ! currentServers.contains(spec.split(":", 2)[0]))) {
+ log.log(Level.WARNING, "Existing dynamic config refers to unknown servers, ignoring it");
+ dynamicConfig = Map.of();
+ }
+
+ // If we have no existing, valid, dynamic config, we use all known servers as a starting point.
+ if (dynamicConfig.isEmpty()) {
+ configEntries.putAll(getServerConfig(config.server(), config.server(myIndex).joining() ? config.myid() : -1));
+ }
+ // Otherwise, we use the existing, dynamic config as a starting point, and add this as a joiner if not present.
+ else {
+ Map.Entry<String, String> thisAsAJoiner = getServerConfig(config.server().subList(myIndex, myIndex + 1), config.myid()).entrySet().iterator().next();
+ dynamicConfig.putIfAbsent(thisAsAJoiner.getKey(), thisAsAJoiner.getValue());
+ configEntries.putAll(dynamicConfig);
+ }
+
+ }
+ static Map<String, String> getServerConfig(List<ZookeeperServerConfig.Server> serversConfig, int joinerId) {
+ Map<String, String> configEntries = new LinkedHashMap<>();
+ for (Server server : serversConfig) {
+ configEntries.put("server." + server.id(), serverSpec(server, server.id() == joinerId));
+ }
+ return configEntries;
+ }
+
+ static String transformConfigToString(Map<String, String> config) {
+ return config.entrySet().stream()
+ .map(entry -> entry.getKey() + "=" + entry.getValue())
+ .collect(Collectors.joining("\n", "", "\n"));
}
private void writeMyIdFile(ZookeeperServerConfig config) throws IOException {
@@ -113,25 +159,17 @@ public class Configurator {
}
}
- private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) {
- boolean found = false;
- for (ZookeeperServerConfig.Server server : servers) {
- if (myid == server.id()) {
- found = true;
- break;
- }
- }
- if (!found) {
- throw new RuntimeException("No id in zookeeper server list that corresponds to my id (" + myid + ")");
+ private static int ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) {
+ for (int i = 0; i < servers.size(); i++) {
+ Server server = servers.get(i);
+ if (myid == server.id()) return i;
}
+ throw new RuntimeException("No id in zookeeper server list that corresponds to my id (" + myid + ")");
}
static String serverSpec(ZookeeperServerConfig.Server server, boolean joining) {
StringBuilder sb = new StringBuilder();
- sb.append("server.")
- .append(server.id())
- .append("=")
- .append(server.hostname())
+ sb.append(server.hostname())
.append(":")
.append(server.quorumPort())
.append(":")
@@ -150,6 +188,19 @@ public class Configurator {
return sb.toString();
}
+ static Map<String, String> parseConfigFile(Path configFilePath) {
+ try {
+ return Files.exists(configFilePath) ? Files.readAllLines(configFilePath).stream()
+ .filter(line -> ! line.startsWith("#"))
+ .map(line -> line.split("=", 2))
+ .collect(toLinkedMap(parts -> parts[0], parts -> parts[1]))
+ : Map.of();
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException("error reading zookeeper config", e);
+ }
+ }
+
static List<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) {
return zookeeperServerConfig.server().stream()
.map(ZookeeperServerConfig.Server::hostname)
@@ -165,15 +216,15 @@ public class Configurator {
private interface TlsConfig {
String configFieldPrefix();
- default void appendSharedTlsConfig(StringBuilder builder, VespaTlsConfig vespaTlsConfig) {
+ default void appendSharedTlsConfig(Map<String, String> configEntries, VespaTlsConfig vespaTlsConfig) {
vespaTlsConfig.context().ifPresent(ctx -> {
VespaSslContextProvider.set(ctx);
- builder.append(configFieldPrefix()).append(".context.supplier.class=").append(VespaSslContextProvider.class.getName()).append("\n");
+ configEntries.put(configFieldPrefix() + ".context.supplier.class", VespaSslContextProvider.class.getName());
String enabledCiphers = Arrays.stream(ctx.parameters().getCipherSuites()).sorted().collect(Collectors.joining(","));
- builder.append(configFieldPrefix()).append(".ciphersuites=").append(enabledCiphers).append("\n");
+ configEntries.put(configFieldPrefix() + ".ciphersuites", enabledCiphers);
String enabledProtocols = Arrays.stream(ctx.parameters().getProtocols()).sorted().collect(Collectors.joining(","));
- builder.append(configFieldPrefix()).append(".enabledProtocols=").append(enabledProtocols).append("\n");
- builder.append(configFieldPrefix()).append(".clientAuth=NEED\n");
+ configEntries.put(configFieldPrefix() + ".enabledProtocols", enabledProtocols);
+ configEntries.put(configFieldPrefix() + ".clientAuth", "NEED");
});
}
@@ -185,16 +236,13 @@ public class Configurator {
static class TlsClientServerConfig implements TlsConfig {
- public String createConfig(VespaTlsConfig vespaTlsConfig) {
- StringBuilder sb = new StringBuilder()
- .append("client.portUnification=").append(enablePortUnification(vespaTlsConfig)).append("\n");
+ public void createConfig(Map<String, String> configEntries, VespaTlsConfig vespaTlsConfig) {
+ configEntries.put("client.portUnification", String.valueOf(enablePortUnification(vespaTlsConfig)));
// ZooKeeper Dynamic Reconfiguration requires the "non-secure" client port to exist
// This is a hack to override the secure parameter through our connection factory wrapper
// https://issues.apache.org/jira/browse/ZOOKEEPER-3577
VespaNettyServerCnxnFactory_isSecure = vespaTlsConfig.tlsEnabled() && vespaTlsConfig.mixedMode() == MixedMode.DISABLED;
- appendSharedTlsConfig(sb, vespaTlsConfig);
-
- return sb.toString();
+ appendSharedTlsConfig(configEntries, vespaTlsConfig);
}
@Override
@@ -205,12 +253,10 @@ public class Configurator {
static class TlsQuorumConfig implements TlsConfig {
- public String createConfig(VespaTlsConfig vespaTlsConfig) {
- StringBuilder sb = new StringBuilder()
- .append("sslQuorum=").append(vespaTlsConfig.tlsEnabled()).append("\n")
- .append("portUnification=").append(enablePortUnification(vespaTlsConfig)).append("\n");
- appendSharedTlsConfig(sb, vespaTlsConfig);
- return sb.toString();
+ public void createConfig(Map<String, String> configEntries, VespaTlsConfig vespaTlsConfig) {
+ configEntries.put("sslQuorum", String.valueOf(vespaTlsConfig.tlsEnabled()));
+ configEntries.put("portUnification", String.valueOf(enablePortUnification(vespaTlsConfig)));
+ appendSharedTlsConfig(configEntries, vespaTlsConfig);
}
@Override
diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
index eb84b13d4d6..15431550d82 100644
--- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
+++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java
@@ -6,15 +6,15 @@ import com.yahoo.component.AbstractComponent;
import com.yahoo.component.annotation.Inject;
import com.yahoo.protect.Process;
import com.yahoo.yolean.Exceptions;
+
import java.time.Duration;
import java.time.Instant;
-import java.util.List;
import java.util.Objects;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
-import static com.yahoo.vespa.zookeeper.Configurator.serverSpec;
+import static java.util.stream.Collectors.joining;
/**
* Starts zookeeper server and supports reconfiguring zookeeper cluster. Keep this as a component
@@ -26,9 +26,11 @@ public class Reconfigurer extends AbstractComponent {
private static final Logger log = java.util.logging.Logger.getLogger(Reconfigurer.class.getName());
- private static final Duration TIMEOUT = Duration.ofMinutes(15);
+ static final Duration TIMEOUT = Duration.ofMinutes(15);
private final ExponentialBackoff backoff = new ExponentialBackoff(Duration.ofMillis(50), Duration.ofSeconds(10));
+ private final Duration timeout;
+ private final boolean haltOnFailure;
private final VespaZooKeeperAdmin vespaZooKeeperAdmin;
private final Sleeper sleeper;
@@ -38,12 +40,14 @@ public class Reconfigurer extends AbstractComponent {
@Inject
public Reconfigurer(VespaZooKeeperAdmin vespaZooKeeperAdmin) {
- this(vespaZooKeeperAdmin, new Sleeper());
+ this(vespaZooKeeperAdmin, new Sleeper(), true, TIMEOUT);
}
- Reconfigurer(VespaZooKeeperAdmin vespaZooKeeperAdmin, Sleeper sleeper) {
+ public Reconfigurer(VespaZooKeeperAdmin vespaZooKeeperAdmin, Sleeper sleeper, boolean haltOnFailure, Duration timeout) {
this.vespaZooKeeperAdmin = Objects.requireNonNull(vespaZooKeeperAdmin);
this.sleeper = Objects.requireNonNull(sleeper);
+ this.haltOnFailure = haltOnFailure;
+ this.timeout = timeout;
}
@Override
@@ -86,14 +90,15 @@ public class Reconfigurer extends AbstractComponent {
// TODO jonmv: wrap Curator in Provider, for Curator shutdown
private void reconfigure(ZookeeperServerConfig newConfig) {
Instant reconfigTriggered = Instant.now();
- String newServers = String.join(",", servers(newConfig));
+ String newServers = servers(newConfig);
log.log(Level.INFO, "Will reconfigure ZooKeeper cluster." +
"\nServers in active config:" + servers(activeConfig) +
- "\nServers in new config:" + servers(newConfig));
+ "\nServers in new config:" + newServers);
String connectionSpec = vespaZooKeeperAdmin.localConnectionSpec(activeConfig);
Instant now = Instant.now();
- Duration reconfigTimeout = reconfigTimeout();
- Instant end = now.plus(reconfigTimeout);
+ // For reconfig to succeed, the current and resulting ensembles must have a majority. When an ensemble grows and
+ // the joining servers outnumber the existing ones, we have to wait for enough of them to start to have a majority.
+ Instant end = now.plus(timeout);
// Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed
for (int attempt = 1; ; attempt++) {
try {
@@ -116,29 +121,20 @@ public class Reconfigurer extends AbstractComponent {
}
else {
log.log(Level.SEVERE, "Reconfiguration attempt " + attempt + " failed, and was failing for " +
- reconfigTimeout + "; giving up now: " + Exceptions.toMessageString(e));
- shutdownAndDie(reconfigTimeout);
+ timeout + "; giving up now: " + Exceptions.toMessageString(e));
+ shutdown();
+ if (haltOnFailure)
+ Process.logAndDie("Reconfiguration did not complete within timeout " + timeout + ". Forcing container shutdown.");
+ else
+ throw e;
}
}
}
}
- private void shutdownAndDie(Duration reconfigTimeout) {
- shutdown();
- Process.logAndDie("Reconfiguration did not complete within timeout " + reconfigTimeout + ". Forcing container shutdown.");
- }
-
- private static Duration reconfigTimeout() {
- // For reconfig to succeed, the current and resulting ensembles must have a majority. When an ensemble grows and
- // the joining servers outnumber the existing ones, we have to wait for enough of them to start to have a majority.
- return TIMEOUT;
- }
-
- private static List<String> servers(ZookeeperServerConfig config) {
- return config.server().stream()
- .filter(server -> ! server.retired())
- .map(server -> serverSpec(server, false))
- .toList();
+ private static String servers(ZookeeperServerConfig config) {
+ return Configurator.getServerConfig(config.server().stream().filter(server -> ! server.retired()).toList(), -1)
+ .entrySet().stream().map(entry -> entry.getKey() + "=" + entry.getValue()).collect(joining(","));
}
}
diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java
index 5d0031d5b55..08acbf2b838 100644
--- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java
+++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java
@@ -24,7 +24,10 @@ import java.math.BigInteger;
import java.nio.file.Files;
import java.security.KeyPair;
import java.security.cert.X509Certificate;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import static com.yahoo.security.KeyAlgorithm.EC;
@@ -62,12 +65,12 @@ public class ConfiguratorTest {
}
@Test
- public void config_is_written_correctly_with_multiple_servers() {
+ public void config_is_written_correctly_with_multiple_servers() throws IOException {
three_config_servers(false);
}
@Test
- public void config_is_written_correctly_with_multiple_servers_on_hosted_vespa() {
+ public void config_is_written_correctly_with_multiple_servers_on_hosted_vespa() throws IOException {
three_config_servers(true);
}
@@ -117,13 +120,49 @@ public class ConfiguratorTest {
assertEquals("" + max_buffer, System.getProperty(ZOOKEEPER_JUTE_MAX_BUFFER));
}
- private void three_config_servers(boolean hosted) {
+ @Test
+ public void test_parsing_config() throws IOException {
ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
builder.server(newServer(0, "foo", 123, 321, false));
builder.server(newServer(1, "bar", 234, 432, false));
builder.server(newServer(2, "baz", 345, 543, true));
builder.myidFile(idFile.getAbsolutePath());
+ builder.myid(2);
+ builder.tickTime(1234);
+ builder.dynamicReconfiguration(true);
+ Configurator configurator = new Configurator(builder.build());
+ configurator.writeConfigToDisk(VespaTlsConfig.tlsDisabled());
+ validateIdFile(idFile, "2\n");
+
+ assertEquals(Files.readString(cfgFile.toPath()),
+ Configurator.transformConfigToString(Configurator.parseConfigFile(cfgFile.toPath())));
+
+ Map<String, String> originalConfig = Configurator.parseConfigFile(cfgFile.toPath());
+ Map<String, String> staticConfig = new LinkedHashMap<>(originalConfig);
+ // Dynamic config says this is not a joiner.
+ Map<String, String> dynamicConfig = Configurator.getServerConfig(builder.build().server(), -1);
+ staticConfig.keySet().removeAll(dynamicConfig.keySet());
+ assertEquals(originalConfig.size(), dynamicConfig.size() + staticConfig.size());
+ File dynFile = folder.newFile();
+ staticConfig.put("dynamicConfigFile", dynFile.getAbsolutePath());
+ Files.write(cfgFile.toPath(), Configurator.transformConfigToString(staticConfig).getBytes());
+ Files.write(dynFile.toPath(), Configurator.transformConfigToString(dynamicConfig).getBytes());
+
+ configurator.writeConfigToDisk(VespaTlsConfig.tlsDisabled());
+ // Next generation of config should not mark this as a joiner either.
+ originalConfig.putAll(Configurator.getServerConfig(builder.build().server().subList(2, 3), -1));
+ assertEquals(Configurator.transformConfigToString(originalConfig),
+ Files.readString(cfgFile.toPath()));
+ }
+
+ private void three_config_servers(boolean hosted) throws IOException {
+ ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
+ builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
+ builder.server(newServer(0, "foo", 123, 321, false));
+ builder.server(newServer(1, "bar", 234, 432, true));
+ builder.server(newServer(2, "baz", 345, 543, false));
+ builder.myidFile(idFile.getAbsolutePath());
builder.myid(1);
builder.tickTime(1234);
builder.dynamicReconfiguration(hosted);
@@ -205,8 +244,8 @@ public class ConfiguratorTest {
String expected =
commonConfig(hosted) +
"server.0=foo:321:123;2181\n" +
- "server.1=bar:432:234;2181\n" +
- "server.2=baz:543:345:observer;2181\n" +
+ "server.1=bar:432:234:observer;2181\n" +
+ "server.2=baz:543:345;2181\n" +
"sslQuorum=false\n" +
"portUnification=false\n" +
"client.portUnification=false\n";
diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
index 697fba3b4c4..ea8dcac945c 100644
--- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
+++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java
@@ -144,11 +144,13 @@ public class ReconfigurerTest {
TestableReconfigurer(TestableVespaZooKeeperAdmin zooKeeperAdmin) {
super(zooKeeperAdmin, new Sleeper() {
- @Override
- public void sleep(Duration duration) {
- // Do nothing
- }
- });
+ @Override
+ public void sleep(Duration duration) {
+ // Do nothing
+ }
+ },
+ false,
+ Reconfigurer.TIMEOUT);
this.zooKeeperAdmin = zooKeeperAdmin;
HostName.setHostNameForTestingOnly("node1");
}